diff --git a/.github/actions/test-all-eamxx/action.yml b/.github/actions/test-all-eamxx/action.yml index dc738bd3567a..8157131b2005 100644 --- a/.github/actions/test-all-eamxx/action.yml +++ b/.github/actions/test-all-eamxx/action.yml @@ -118,6 +118,6 @@ runs: with: name: log-files-${{ inputs.build_type }}-${{ inputs.machine }} path: | - components/eamxx/ctest-build/**/Testing/Temporary/Last*.log components/eamxx/ctest-build/**/ctest_resource_file.json components/eamxx/ctest-build/**/CMakeCache.txt + components/eamxx/ctest-build/**/Testing/** diff --git a/cime b/cime index ed580a83616a..4e686d2f1c57 160000 --- a/cime +++ b/cime @@ -1 +1 @@ -Subproject commit ed580a83616ac400b9e1f42ba275eab457988110 +Subproject commit 4e686d2f1c57996d5a38d09e132c4010c68fc93c diff --git a/cime_config/machines/cmake_macros/aurora_oneapi-ifxgpu.cmake b/cime_config/machines/cmake_macros/aurora_oneapi-ifxgpu.cmake index 06f23b478c61..5c3bdc42bdf1 100644 --- a/cime_config/machines/cmake_macros/aurora_oneapi-ifxgpu.cmake +++ b/cime_config/machines/cmake_macros/aurora_oneapi-ifxgpu.cmake @@ -4,7 +4,7 @@ if (compile_threaded) string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64") endif() -string(APPEND KOKKOS_OPTIONS " -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_SERIAL=On -DKokkos_ARCH_INTEL_PVC=On -DKokkos_ENABLE_SYCL=On -DKokkos_ENABLE_EXPLICIT_INSTANTIATION=Off -DCMAKE_POSITION_INDEPENDENT_CODE=ON") +string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_SERIAL=On -DKokkos_ARCH_INTEL_PVC=On -DKokkos_ENABLE_SYCL=On -DKokkos_ENABLE_EXPLICIT_INSTANTIATION=Off -DCMAKE_POSITION_INDEPENDENT_CODE=ON") string(APPEND SYCL_FLAGS " -fsycl -fsycl-targets=spir64_gen -mlong-double-64 ") string(APPEND OMEGA_SYCL_EXE_LINKER_FLAGS " -Xsycl-target-backend \"-device pvc\" ") diff --git a/cime_config/machines/cmake_macros/frontier_craycray-mphipcc.cmake b/cime_config/machines/cmake_macros/frontier_craycray-mphipcc.cmake index a464bda80d9f..3bd4ce4c40a6 100644 --- a/cime_config/machines/cmake_macros/frontier_craycray-mphipcc.cmake +++ b/cime_config/machines/cmake_macros/frontier_craycray-mphipcc.cmake @@ -17,8 +17,6 @@ if (compile_threaded) string(APPEND CMAKE_EXE_LINKER_FLAGS " -fopenmp") endif() -string(APPEND CMAKE_CXX_FLAGS " -std=c++17") - string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero -f free") string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa") string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64") diff --git a/cime_config/machines/cmake_macros/frontier_craycray.cmake b/cime_config/machines/cmake_macros/frontier_craycray.cmake index 3d8990f2d2e4..cf008431c88c 100644 --- a/cime_config/machines/cmake_macros/frontier_craycray.cmake +++ b/cime_config/machines/cmake_macros/frontier_craycray.cmake @@ -17,8 +17,6 @@ if (compile_threaded) string(APPEND CMAKE_EXE_LINKER_FLAGS " -fopenmp") endif() -string(APPEND CMAKE_CXX_FLAGS " -std=c++17") - string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero -f free") # Crusher: this resolves a crash in mct in docn init diff --git a/cime_config/machines/cmake_macros/gnugpu.cmake b/cime_config/machines/cmake_macros/gnugpu.cmake index 3313140dc6a7..d0a6c2d07659 100644 --- a/cime_config/machines/cmake_macros/gnugpu.cmake +++ b/cime_config/machines/cmake_macros/gnugpu.cmake @@ -1,5 +1,14 @@ -string(APPEND CMAKE_C_FLAGS " -mcmodel=medium") -string(APPEND CMAKE_Fortran_FLAGS " -mcmodel=medium -fconvert=big-endian -ffree-line-length-none -ffixed-line-length-none") +if (CMAKE_SYSTEM_PROCESSOR MATCHES "^aarch64") + string(APPEND CMAKE_C_FLAGS " -mcmodel=small") + string(APPEND CMAKE_Fortran_FLAGS " -mcmodel=small") +elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "arm64") + string(APPEND CMAKE_C_FLAGS " -mcmodel=large") + string(APPEND CMAKE_Fortran_FLAGS " -mcmodel=large") +else() + string(APPEND CMAKE_C_FLAGS " -mcmodel=medium") + string(APPEND CMAKE_Fortran_FLAGS " -mcmodel=medium") +endif() +string(APPEND CMAKE_Fortran_FLAGS " -fconvert=big-endian -ffree-line-length-none -ffixed-line-length-none") if (CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER_EQUAL 10) string(APPEND CMAKE_Fortran_FLAGS " -fallow-argument-mismatch") endif() diff --git a/cime_config/machines/cmake_macros/vista-gg_gnu.cmake b/cime_config/machines/cmake_macros/vista-gg_gnu.cmake new file mode 100644 index 000000000000..e673272f9823 --- /dev/null +++ b/cime_config/machines/cmake_macros/vista-gg_gnu.cmake @@ -0,0 +1,13 @@ +string(APPEND CPPDEFS " -DLINUX") +if (COMP_NAME STREQUAL gptl) + string(APPEND CPPDEFS " -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY -DHAVE_MPI") +endif() +string(APPEND CMAKE_C_FLAGS_RELEASE " -O2 -g") +string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2 -g") + +set(MPICC "mpicc") +set(MPICXX "mpicxx") +set(MPIFC "mpif90") +set(SCC "gcc") +set(SCXX "g++") +set(SFC "gfortran") diff --git a/cime_config/machines/cmake_macros/vista-gg_nvidia.cmake b/cime_config/machines/cmake_macros/vista-gg_nvidia.cmake new file mode 100644 index 000000000000..fb24721f1640 --- /dev/null +++ b/cime_config/machines/cmake_macros/vista-gg_nvidia.cmake @@ -0,0 +1,20 @@ +string(APPEND CPPDEFS " -DLINUX") +if (COMP_NAME STREQUAL gptl) + string(APPEND CPPDEFS " -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY -DHAVE_MPI") +endif() +string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") +string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") +string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -g") + +set(HOMME_QUAD_PREC FALSE CACHE BOOL "") # nvidia does not seem to support QUAD + +if (compile_threaded) + string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_OPENMP=Off") # work-around for nvidia as kokkos is not passing "-mp" for threaded build +endif() + +set(MPICC "mpicc") +set(MPICXX "mpicxx") +set(MPIFC "mpif90") +set(SCC "gcc") +set(SCXX "g++") +set(SFC "gfortran") diff --git a/cime_config/machines/cmake_macros/vista-gh_gnu.cmake b/cime_config/machines/cmake_macros/vista-gh_gnu.cmake new file mode 100644 index 000000000000..0fcf1205df51 --- /dev/null +++ b/cime_config/machines/cmake_macros/vista-gh_gnu.cmake @@ -0,0 +1,15 @@ +string(APPEND CPPDEFS " -DLINUX") +if (COMP_NAME STREQUAL gptl) + string(APPEND CPPDEFS " -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY -DHAVE_MPI") +endif() +string(APPEND CMAKE_C_FLAGS_RELEASE " -O2 -g") +string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2 -g") + +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/opt/apps/gcc/14.2.0/lib64 -lstdc++") # workaround for pnetcdf thinking it needs gcc14 abi + +set(MPICC "mpicc") +set(MPICXX "mpicxx") +set(MPIFC "mpif90") +set(SCC "gcc") +set(SCXX "g++") +set(SFC "gfortran") diff --git a/cime_config/machines/cmake_macros/vista-gh_gnugpu.cmake b/cime_config/machines/cmake_macros/vista-gh_gnugpu.cmake new file mode 100644 index 000000000000..fca3f27da811 --- /dev/null +++ b/cime_config/machines/cmake_macros/vista-gh_gnugpu.cmake @@ -0,0 +1,21 @@ +set(USE_CUDA "TRUE") +string(APPEND CPPDEFS " -DGPU") +string(APPEND CPPDEFS " -DLINUX") +if (COMP_NAME STREQUAL gptl) + string(APPEND CPPDEFS " -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY -DHAVE_MPI") + # -DHAVE_NANOTIME -- cant use this as the assembly instructions that wont work on ARM +endif() +string(APPEND CMAKE_CUDA_FLAGS " -ccbin CC -O2 -arch sm_90 --use_fast_math") +string(APPEND KOKKOS_OPTIONS " -DKokkos_ARCH_HOPPER90=On -DKokkos_ENABLE_CUDA=On -DKokkos_ENABLE_CUDA_LAMBDA=On -DKokkos_ENABLE_SERIAL=ON -DKokkos_ENABLE_OPENMP=Off -DKokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC=Off") +set(CMAKE_CUDA_ARCHITECTURES "90") + +string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") +string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/opt/apps/gcc/14.2.0/lib64 -lstdc++") # workaround for pnetcdf thinking it needs gcc14 abi + +set(MPICC "mpicc") +set(MPICXX "mpicxx") # Needs MPICH_CXX to use nvcc $SHELL{which nvcc} +set(MPIFC "mpif90") +set(SCC "gcc") +set(SCXX "nvcc") +set(SFC "gfortran") diff --git a/cime_config/machines/config_batch.xml b/cime_config/machines/config_batch.xml index 75c60a438b06..f884ca88450a 100644 --- a/cime_config/machines/config_batch.xml +++ b/cime_config/machines/config_batch.xml @@ -618,6 +618,27 @@ + + + -n {{ total_tasks }} + --partition=gh + + + gh-dev + gh + + + + + + -n {{ total_tasks }} + --partition=gg + + + gg + + + qsub diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index b66ca9214cc8..6713fd5cd454 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -515,9 +515,7 @@ MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} - - - $ENV{CRAY_LD_LIBRARY_PATH}:$ENV{LD_LIBRARY_PATH} + $ENV{CRAY_LD_LIBRARY_PATH}:$ENV{LD_LIBRARY_PATH} 128M @@ -572,6 +570,229 @@ + + vista at TACC https://docs.tacc.utexas.edu/hpc/vista gh (grace-hopper) GH200 Superchip -- Grace CPU (72 cores) and 1 NVIDIA H100 GPU + .*vista.* + Linux + gnugpu,gnu,nvidiagpu,nvidia + openmpi + CDA24017 + $ENV{WORK} + e3sm + $ENV{WORK}/e3sm_scratch/vista-gh + $ENV{WORK}/www/$ENV{USER} + $ENV{WORK}/inputdata + $ENV{WORK}/inputdata/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + $ENV{WORK}/baselines/$COMPILER + $ENV{WORK}/tools/cprnc/cprnc + 4 + e3sm_developer + 2 + slurm + e3sm + 72 + 144 + 144 + 1 + 144 + 144 + FALSE + + ibrun + + --label + -n {{ total_tasks }} + + + + /opt/apps/lmod/lmod/init/perl + /opt/apps/lmod/lmod/init/python + /opt/apps/lmod/lmod/init/sh + /opt/apps/lmod/lmod/init/csh + /opt/apps/lmod/lmod/libexec/lmod perl + /opt/apps/lmod/lmod/libexec/lmod python + module -q + module -q + + + netcdf + pnetcdf + hdf5 + gcc + nvidia + cuda + perftools-base + perftools + darshan + + + + gcc/13.2.0 + openmpi/5.0.5 + + + + nvidia/24.7 + openmpi/5.0.8 + + + + gcc/13.2.0 + cuda/12.6 + openmpi/5.0.5 + + + + cuda/12.6 + openmpi/5.0.8 + + + + ucx/1.19.1 + netcdf/4.9.2 + pnetcdf/1.13.0 + cmake + + + + $CIME_OUTPUT_ROOT/$CASE/run + $CIME_OUTPUT_ROOT/$CASE/bld + 0.1 + 0.20 + + + 128M + spread + threads + FALSE + kdreg2 + $ENV{TACC_NETCDF_DIR} + $ENV{TACC_PNETCDF_DIR} + /opt/apps/gcc/14.2.0/lib64:$ENV{LD_LIBRARY_PATH} + + + + 144 + + + 144 + + + 1 + 1 + $SHELL{which nvcc} + + + 1 + 1 + $SHELL{which nvcc} + + + -1 + + + + + vista at TACC https://docs.tacc.utexas.edu/hpc/vista (gg -- grace-grace) CPU-only: NVIDIA Grace CPU Superchip (72x2 or 144 cores) + .*vista.* + Linux + gnu,nvidia + openmpi + CDA24017 + $ENV{WORK} + e3sm + $ENV{WORK}/e3sm_scratch/vista-gg + $ENV{WORK}/www/$ENV{USER} + $ENV{WORK}/inputdata + $ENV{WORK}/inputdata/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + $ENV{WORK}/baselines/$COMPILER + $ENV{WORK}/tools/cprnc/cprnc + 4 + e3sm_developer + 2 + slurm + e3sm + 288 + 144 + FALSE + + ibrun + + --label + -n {{ total_tasks }} + + + + /opt/apps/lmod/lmod/init/perl + /opt/apps/lmod/lmod/init/python + /opt/apps/lmod/lmod/init/sh + /opt/apps/lmod/lmod/init/csh + /opt/apps/lmod/lmod/libexec/lmod perl + /opt/apps/lmod/lmod/libexec/lmod python + module -q + module -q + + + netcdf + pnetcdf + hdf5 + gcc + nvidia + cuda + perftools-base + perftools + darshan + + + + gcc + gcc/15.1.0 + openmpi/5.0.5 + + + + nvidia/25.5 + openmpi/5.0.8 + + + + ucx/1.19.1 + netcdf/4.9.2 + pnetcdf/1.13.0 + cmake + + + + $CIME_OUTPUT_ROOT/$CASE/run + $CIME_OUTPUT_ROOT/$CASE/bld + 0.1 + 0.20 + + + 128M + spread + threads + FALSE + kdreg2 + $ENV{TACC_NETCDF_DIR} + $ENV{TACC_PNETCDF_DIR} + 1 + 1 + + + + 144 + + + 144 + + + -1 + + + Muller CPU-only nodes on internal NERSC machine, similar to pm-cpu (very small) $ENV{NERSC_HOST}:muller @@ -2416,7 +2637,9 @@ $ENV{SEMS_NETCDF_ROOT}/include $ENV{SEMS_NETCDF_ROOT}/lib 64M - + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /projects/ccsm/moab; else echo "$MOAB_ROOT"; fi} + $SHELL{dirname $(dirname $(which h5dump))} + $ENV{SEMS_NETCDF_ROOT} diff --git a/cime_config/tests.py b/cime_config/tests.py index 38ebba26568d..d578409eaae6 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -340,7 +340,7 @@ }, "e3sm_developer" : { - "inherit" : ("e3sm_land_developer", "e3sm_atm_developer", "e3sm_ice_developer", "e3sm_cryo_developer"), + "inherit" : ("e3sm_land_developer", "e3sm_atm_developer", "e3sm_ice_developer", "e3sm_cryo_developer", "e3sm_gcam_developer"), "time" : "0:45:00", "tests" : ( "ERS.ne4pg2_oQU480_rx1.A", @@ -396,7 +396,7 @@ "e3sm_extra_coverage" : { "inherit" : ("e3sm_atm_extra_coverage", "e3sm_ocnice_extra_coverage"), "tests" : ( - "SMS_Vmct_D_Ln3.TL319_EC30to60E2r2_wQU225EC30to60E2r2.GMPAS-JRA1p5-WW3.ww3-jra_1958", + "SMS_Vmct_D_Ln3.TL319_IcoswISC30E3r5_wQU225Icos30E3r5.GMPAS-JRA1p5-WW3.ww3-jra_1958", ) }, @@ -406,6 +406,7 @@ "tests" : ( "SMS_Ld3.ne120pg2_r025_RRSwISC6to18E3r5.WCYCL1850NS.eam-cosplite", "SMS.T62_SOwISC12to30E3r3.GMPAS-IAF", + "SMS_Ld3.ne30pg2_r05_SOwISC12to30E3r3.CRYO1850-CMIP7", ) }, @@ -435,6 +436,7 @@ "SMS_Ld1.ne30pg2_r05_IcoswISC30E3r5.WCYCLSSP370.allactive-wcprodssp", "SMS_Ld1.ne30pg2_r05_IcoswISC30E3r5.WCYCLSSP585.allactive-wcprodssp", "SMS_Ld1_P512.northamericax4v1pg2_r025_IcoswISC30E3r5.WCYCL1850.allactive-wcprodrrm_1850", + "SMS_Vmct_D_Ld1.TL319_IcoswISC30E3r5_wQU225Icos30E3r5.GMPAS-JRA1p5-WW3.ww3-jra_1958", "SMS_D_Ld1.ne30pg2_r05_IcoswISC30E3r5.CRYO1850", "SMS_D_Ld1.ne30pg2_r05_IcoswISC30E3r5.CRYO1850-CMIP7", ) @@ -1140,8 +1142,17 @@ "e3sm_gcam_developer" : { "time" : "1:00:00", "tests" : ( - "SMS.ne30pg2_f09_oEC60to30v3.SSP245_ZATM_BGC", - "ERS.ne30pg2_f09_oEC60to30v3.SSP245_ZATM_BGC", + "SMS_Vmct.ne30pg2_f09_oEC60to30v3.SSP245_ZATM_BGC", + "ERS_Vmct.ne30pg2_f09_oEC60to30v3.SSP245_ZATM_BGC", ) }, + "e3sm_test_bless" : { + "time" : "10:00", + "tests" : ( + "TESTRUNDIFF_P1.f19_g16.A", + "TESTRUNDIFF_P2.f19_g16.A", + "TESTRUNDIFF_P4.f19_g16.A", + "TESTRUNDIFF_P8.f19_g16.A", + ) + }, } diff --git a/components/cmake/build_model.cmake b/components/cmake/build_model.cmake index b3a076a8216a..18d5ab728bdd 100644 --- a/components/cmake/build_model.cmake +++ b/components/cmake/build_model.cmake @@ -263,6 +263,13 @@ macro(build_model COMP_CLASS COMP_NAME) add_executable(${TARGET_NAME}) target_sources(${TARGET_NAME} PRIVATE ${REAL_SOURCES}) + # driver-mct/main sources (e.g. cime_comp_mod.F90) use netcdf directly, but + # the component libraries only link netcdf PRIVATEly (via csm_share), so + # its usage requirements (e.g. include dirs for netcdf.mod) do not + # propagate up to this exe target. Find/link it explicitly here too. + find_package(NETCDF REQUIRED) + target_link_libraries(${TARGET_NAME} netcdf) + foreach(ITEM IN LISTS COMP_CLASSES) if (NOT ITEM STREQUAL "cpl") target_link_libraries(${TARGET_NAME} ${ITEM}) diff --git a/components/eam/bld/namelist_files/namelist_definition.xml b/components/eam/bld/namelist_files/namelist_definition.xml index 258c3ef00e27..c5da065afb33 100644 --- a/components/eam/bld/namelist_files/namelist_definition.xml +++ b/components/eam/bld/namelist_files/namelist_definition.xml @@ -6794,6 +6794,11 @@ Bottom of sponge layer in hPa. Default: 0 (use default value based on reference pressure at model top). + +Default: Set by build-namelist. + + Default: Set by build-namelist. diff --git a/components/eam/bld/namelist_files/use_cases/SSP245_eam_CMIP6_chemUCI-Linoz-mam5-vbs_EHC.xml b/components/eam/bld/namelist_files/use_cases/SSP245_eam_CMIP6_chemUCI-Linoz-mam5-vbs_EHC.xml index db1c6f3c9dcd..50d42f36bd78 100644 --- a/components/eam/bld/namelist_files/use_cases/SSP245_eam_CMIP6_chemUCI-Linoz-mam5-vbs_EHC.xml +++ b/components/eam/bld/namelist_files/use_cases/SSP245_eam_CMIP6_chemUCI-Linoz-mam5-vbs_EHC.xml @@ -16,9 +16,6 @@ 1.e-5 - -1 - INTERP_MISSING_MONTHS atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP245_ne30/emissions-cmip6_ssp245_e3sm_NO2_aircraft_vertical_2015-2100_1.9x2.5_c20240219.nc diff --git a/components/eam/bld/namelist_files/use_cases/SSP370_eam_CMIP6_chemUCI-Linoz-mam5-vbs_EHC.xml b/components/eam/bld/namelist_files/use_cases/SSP370_eam_CMIP6_chemUCI-Linoz-mam5-vbs_EHC.xml index 34b63ceade1f..78835dce3237 100755 --- a/components/eam/bld/namelist_files/use_cases/SSP370_eam_CMIP6_chemUCI-Linoz-mam5-vbs_EHC.xml +++ b/components/eam/bld/namelist_files/use_cases/SSP370_eam_CMIP6_chemUCI-Linoz-mam5-vbs_EHC.xml @@ -20,9 +20,6 @@ 1.e-5 - -1 - INTERP_MISSING_MONTHS atm/cam/chem/trop_mozart_aero/emis/CMIP6_SSP370_ne30/emissions-cmip6_ssp370_e3sm_NO2_aircraft_vertical_2015-2100_1.9x2.5_c20240208.nc diff --git a/components/eam/src/dynamics/se/gravity_waves_sources.F90 b/components/eam/src/dynamics/se/gravity_waves_sources.F90 index c6fcc76dc1fe..f4ae2cc9c157 100644 --- a/components/eam/src/dynamics/se/gravity_waves_sources.F90 +++ b/components/eam/src/dynamics/se/gravity_waves_sources.F90 @@ -242,7 +242,7 @@ subroutine compute_frontogenesis( frontgf, frontga, tl, & do k = 1,nlev ! latlon -> cartesian - Summing along the third dimension is a sum over components for each point do component=1,3 - dum_cart(:,:,component,k)=sum( elem(ie)%vec_sphere2cart(:,:,component,:) * elem(ie)%state%v(:,:,:,k,tl) ,3) + dum_cart(:,:,component,k)=sum( elem(ie)%vec_sphere2cart(:,:,component,1:2) * elem(ie)%state%v(:,:,:,k,tl) ,3) end do end do diff --git a/components/eam/src/physics/cam/cflx.F90 b/components/eam/src/physics/cam/cflx.F90 index 86f6e398679f..f4484cc3a5e8 100644 --- a/components/eam/src/physics/cam/cflx.F90 +++ b/components/eam/src/physics/cam/cflx.F90 @@ -12,7 +12,7 @@ module cflx contains - subroutine cflx_tend (state, cam_in, ztodt, ptend) + subroutine cflx_tend (state, cam_in, ztodt, ptend, skip_co2, co2_only) use physics_types, only: physics_state, physics_ptend, & physics_ptend_init, & @@ -20,7 +20,7 @@ subroutine cflx_tend (state, cam_in, ztodt, ptend) use physconst, only: gravit use ppgrid, only: pver, pcols use constituents, only: pcnst, cnst_get_ind, cnst_type - use co2_cycle, only: co2_cycle_set_cnst_type + use co2_cycle, only: co2_cycle_set_cnst_type, co2_transport, c_i use camsrfexch, only: cam_in_t implicit none @@ -30,6 +30,8 @@ subroutine cflx_tend (state, cam_in, ztodt, ptend) type(physics_state), intent(inout) :: state ! Physics state variables type(cam_in_t), intent(in) :: cam_in ! contains surface fluxes of constituents real(r8), intent(in) :: ztodt ! 2 delta-t [ s ] + logical, intent(in), optional :: skip_co2 ! if .true., skip CO2 tracers (apply all others) + logical, intent(in), optional :: co2_only ! if .true., apply CO2 tracers only ! Output Auguments @@ -42,15 +44,31 @@ subroutine cflx_tend (state, cam_in, ztodt, ptend) real(r8) :: tmp1(pcols) real(r8) :: rztodt ! 1./ztodt - integer :: m + integer :: m, k logical :: lq(pcnst) + logical :: l_skip_co2, l_co2_only + logical :: co2_mask(pcnst) ! .true. for CO2 tracer indices character(len=3), dimension(pcnst) :: cnst_type_loc ! local override option for constituents cnst_type ncol = state%ncol + ! Process optional arguments + l_skip_co2 = .false. + if (present(skip_co2)) l_skip_co2 = skip_co2 + l_co2_only = .false. + if (present(co2_only)) l_co2_only = co2_only + + ! Build a mask identifying CO2 tracer indices + co2_mask(:) = .false. + if (co2_transport()) then + do k = 1, size(c_i) + if (c_i(k) >= 1 .and. c_i(k) <= pcnst) co2_mask(c_i(k)) = .true. + end do + end if + !------------------------------------------------------- ! Assume 'wet' mixing ratios in surface diffusion code. ! don't convert co2 tracers to wet mixing ratios @@ -60,11 +78,27 @@ subroutine cflx_tend (state, cam_in, ztodt, ptend) call set_dry_to_wet(state, cnst_type_loc) !------------------------------------------------------- - ! Initialize ptend - - lq(:) = .TRUE. + ! Initialize ptend with appropriate tracer mask + ! skip_co2=.true.: apply all tracers except CO2 (used by tphysbc when cflx_cpl_opt==2) + ! co2_only=.true.: apply CO2 tracers only (used by tphysac when cflx_cpl_opt==2) + ! default (both .false.): apply all tracers + + if (l_co2_only) then + lq(:) = co2_mask(:) + else + lq(:) = .TRUE. + if (l_skip_co2) lq(:) = lq(:) .and. (.not. co2_mask(:)) + end if call physics_ptend_init(ptend, state%psetcols, 'clubb_srf', lq=lq) + ! If no tracers are selected, ptend%q is not allocated; exit cleanly. + if (.not. any(lq)) then + cnst_type_loc(:) = cnst_type(:) + call co2_cycle_set_cnst_type(cnst_type_loc, 'wet') + call set_wet_to_dry(state, cnst_type_loc) + return + end if + !------------------------------------------------------- ! Calculate tracer mixing ratio tendencies from cflx @@ -73,6 +107,7 @@ subroutine cflx_tend (state, cam_in, ztodt, ptend) tmp1(:ncol) = ztodt * gravit * state%rpdel(:ncol,pver) do m = 2, pcnst + if (.not. lq(m)) cycle ptend%q(:ncol,pver,m) = ptend%q(:ncol,pver,m) + tmp1(:ncol) * cam_in%cflx(:ncol,m) enddo @@ -80,6 +115,7 @@ subroutine cflx_tend (state, cam_in, ztodt, ptend) ! Convert tendencies of dry constituents to dry basis. do m = 1,pcnst + if (.not. lq(m)) cycle if (cnst_type(m).eq.'dry') then ptend%q(:ncol,:pver,m) = ptend%q(:ncol,:pver,m)*state%pdel(:ncol,:pver)/state%pdeldry(:ncol,:pver) endif diff --git a/components/eam/src/physics/cam/co2_diagnostics.F90 b/components/eam/src/physics/cam/co2_diagnostics.F90 index 81381e9c8d15..2288fe61805f 100644 --- a/components/eam/src/physics/cam/co2_diagnostics.F90 +++ b/components/eam/src/physics/cam/co2_diagnostics.F90 @@ -24,8 +24,8 @@ module co2_diagnostics use cam_logfile , only: iulog use spmd_utils , only: masterproc use cam_abortutils , only: endrun -use time_manager , only: is_first_step, is_last_step, get_prev_date, & - get_curr_date, is_end_curr_month +use time_manager , only: is_first_step, is_first_restart_step, is_last_step, & + get_prev_date, get_curr_date, is_end_curr_month implicit none private @@ -280,7 +280,7 @@ subroutine get_carbon_sfc_fluxes(state, cam_in, dtime) end do end if - if ( .not. is_first_step() ) then + if ( .not. is_first_step() .or. is_first_restart_step() ) then do i = 1, ncol state%c_iflx_sfc(i) = state%c_iflx_sfc(i) + (sfc_flux(i) * dtime) state%c_iflx_ocn(i) = state%c_iflx_ocn(i) + (sfc_flux_ocn(i) * dtime) @@ -365,7 +365,7 @@ subroutine get_carbon_air_fluxes(state, pbuf, dtime, pbuf_name) end do end if - if ( .not. is_first_step() ) then + if ( .not. is_first_step() .or. is_first_restart_step() ) then do i = 1, ncol state%c_iflx_air(i) = state%c_iflx_air(i) + (fossil_flux(i) * dtime) state%c_mflx_air(i) = state%c_mflx_air(i) + (fossil_flux(i) * dtime) diff --git a/components/eam/src/physics/cam/physpkg.F90 b/components/eam/src/physics/cam/physpkg.F90 index d31659bb0fa5..57ac1f31c891 100644 --- a/components/eam/src/physics/cam/physpkg.F90 +++ b/components/eam/src/physics/cam/physpkg.F90 @@ -1494,18 +1494,6 @@ subroutine phys_run2(phys_state, ztodt, phys_tend, pbuf2d, cam_out, & if ( is_end_curr_month() ) then phys_state(c)%tc_mnst(:ncol) = phys_state(c)%tc_curr(:ncol) end if - ! upon restart with cflx_cpl_opt=2, these need to be re-zeroed - ! because get_carbon_sfc_fluxes has not been called yet, - ! and co2_diags_read_fields has been called to fill them with old values - ! there may still be an issue with the timestep-level values, but don't - ! zero them yet so that they can be diagnosed - call phys_getopts( cflx_cpl_opt_out = cflx_cpl_opt) - if ( is_first_restart_step() .and. is_start_curr_month() .and. cflx_cpl_opt == 2) then - phys_state(c)%c_mflx_sfc(:ncol) = 0._r8 - phys_state(c)%c_mflx_ocn(:ncol) = 0._r8 - phys_state(c)%c_mflx_sff(:ncol) = 0._r8 - phys_state(c)%c_mflx_lnd(:ncol) = 0._r8 - end if end do call co2_diags_store_fields(phys_state, pbuf2d) end if @@ -1915,9 +1903,14 @@ subroutine tphysac (ztodt, cam_in, & if (cflx_cpl_opt==1) then call cflx_tend( state, cam_in, ztodt, ptend) - call physics_update(state, ptend, ztodt, tend) - !!!! todo: delete !!! - if (masterproc) write(iulog,*) 'cflx-log: surface flux tendencies applied in tphysac after clubb_surface' + call physics_update(state, ptend, ztodt, tend) + if (masterproc) write(iulog,*) 'cflx-log: surface flux tendencies applied in tphysac after clubb_surface' + elseif (cflx_cpl_opt==2) then + ! Apply CO2 tracer fluxes here in tphysac; non-CO2 fluxes were already applied + ! in tphysbc (see GH #8201). This avoids duplicate CO2 additions during init. + call cflx_tend( state, cam_in, ztodt, ptend, co2_only=.true.) + call physics_update(state, ptend, ztodt, tend) + if (masterproc) write(iulog,*) 'cflx-log: CO2 surface flux tendencies applied in tphysac after clubb_surface' end if call cnd_diag_checkpoint( diag, 'CFLXAPP', state, pbuf, cam_in, cam_out ) @@ -1948,12 +1941,10 @@ subroutine tphysac (ztodt, cam_in, & end if ! l_vdiff endif - ! collect surface carbon fluxes, but only if they have been updated by cflx_tend above, (cflx_cpl_opt==1) - ! or by vertical_diffusion_tend above (l_vdiff==.true.) or if they are not being updated in tphysbc (cflx_cpl_opt /= 2) - ! otherwise, this function is called by tphysbc after cflx_tend() and update_physics() - if (cflx_cpl_opt /= 2) then - call get_carbon_sfc_fluxes(state, cam_in, ztodt) - endif + ! collect surface carbon fluxes after cflx_tend (or vertical_diffusion_tend) has been applied. + ! For cflx_cpl_opt==2, CO2 fluxes are now applied above in tphysac (not tphysbc), so + ! get_carbon_sfc_fluxes is always called here (see GH #8201). + call get_carbon_sfc_fluxes(state, cam_in, ztodt) if (l_rayleigh) then !=================================================== @@ -2317,7 +2308,6 @@ subroutine tphysbc (ztodt, & use debug_info, only: get_debug_chunk, get_debug_macmiciter use lnd_infodata, only: precip_downscaling_method use cflx, only: cflx_tend - use co2_diagnostics, only: get_carbon_sfc_fluxes implicit none @@ -2867,12 +2857,13 @@ subroutine tphysbc (ztodt, & ! on tracers for which cam_in%cflx(:,m) is zero at this point. !if ( do_clubb_sgs .and. (cflx_cpl_opt==2) ) then - if ( cflx_cpl_opt==2 ) then - call cflx_tend( state, cam_in, ztodt, ptend) + if ( (do_clubb_sgs .or. do_shoc_sgs) .and. (cflx_cpl_opt==2) ) then + ! Apply surface fluxes for all tracers EXCEPT CO2; CO2 is applied in tphysac + ! to avoid redundant additions during the multi-call init sequence (see GH #8201) + call cflx_tend( state, cam_in, ztodt, ptend, skip_co2=.true.) call physics_update(state, ptend, ztodt, tend) - call get_carbon_sfc_fluxes(state, cam_in, ztodt) ! for examining surface cflx update timing - aldivi - if (masterproc) write(iulog,*) 'cflx-log: surface flux tendencies applied in tphysbc.' + if (masterproc) write(iulog,*) 'cflx-log: surface flux tendencies (non-CO2) applied in tphysbc.' end if !======================================================================================== diff --git a/components/eam/src/physics/cam/tropopause.F90 b/components/eam/src/physics/cam/tropopause.F90 index 96df8e3f43ec..46c3f43ef854 100644 --- a/components/eam/src/physics/cam/tropopause.F90 +++ b/components/eam/src/physics/cam/tropopause.F90 @@ -94,6 +94,12 @@ module tropopause real(r8) :: cnst_faktor ! = -gravit/rair real(r8) :: cnst_ka1 ! = cnst_kap - 1._r8 + ! If pressure values in all model layers are higher than the threshold specified below, + ! do not attempt to locate the tropopause. The value used here is somewhat + ! arbitrary and is taken from subroutine tropopause_twmo. + + real(r8),parameter :: ptop_thresh = 45000._r8 ! unit: Pa + !================================================================================================ contains !================================================================================================ @@ -1557,6 +1563,11 @@ subroutine tropopause_output(pstate) lchnk = pstate%lchnk ncol = pstate%ncol + ! Skip the rest of the subroutine if pressure values in all model layers are + ! higher than ptop_thresh. This is unlikely in typical global simulations but + ! can happen in idealized tests. + if (minval(pstate%pmid(:ncol,:)).gt.ptop_thresh) return + ! Find the tropopause using the default algorithm backed by the climatology. call tropopause_find(pstate, tropLev, tropP=tropP, tropT=tropT, tropZ=tropZ) @@ -1665,6 +1676,11 @@ subroutine tropopause_e90_3d_output(pstate) if (e90_ndx < 0) return + ! Skip the rest of the subroutine if pressure values in all model layers are + ! higher than ptop_thresh. This is unlikely in typical global simulations but + ! can happen in idealized tests. + if (minval(pstate%pmid(:ncol,:)).gt.ptop_thresh) return + ! Find the tropopause call tropopause_e90_3d(pstate, tropLevB, tropLevU, tropFlag, tropFlagInt, tropP=tropP, tropT=tropT, tropZ=tropZ) diff --git a/components/eam/src/physics/rrtmg/radiation.F90 b/components/eam/src/physics/rrtmg/radiation.F90 index 8c16b694d777..d3a3120fc6b7 100644 --- a/components/eam/src/physics/rrtmg/radiation.F90 +++ b/components/eam/src/physics/rrtmg/radiation.F90 @@ -19,7 +19,7 @@ module radiation use ppgrid, only: pcols, pver, pverp, begchunk, endchunk use physics_types, only: physics_state, physics_ptend use physconst, only: cappa -use time_manager, only: get_nstep, is_first_restart_step +use time_manager, only: get_nstep, is_first_restart_step, is_first_step use cam_abortutils, only: endrun use error_messages, only: handle_err use cam_control_mod, only: lambm0, obliqr, mvelpp, eccen @@ -1145,6 +1145,28 @@ subroutine radiation_tend(state,ptend, pbuf, & dosw = radiation_do('sw') ! do shortwave heating calc this timestep? dolw = radiation_do('lw') ! do longwave heating calc this timestep? + ! In sensitivity experiments with iradsw = 0, dosw is always .false.; + ! consequently, the "if (dosw) then" blocks later in this subroutine are skipped. + ! With a debug build, arrays like qrs, fsnt, and fsns may be left + ! in an uninitialized state and subsequently cause floating-point exception. + ! Here, we initialize these arrays with zero at the first timestep to avoid trouble. + + if ( (.not.dosw) .and. is_first_step() ) then + qrs(1:ncol,1:pver) = 0._r8 + fsnt(1:ncol) = 0._r8 + fsns(1:ncol) = 0._r8 + end if + + ! Similarly, initialize qrl, flnt, and flns with 0 for experiments that have iradlw = 0. + + if ( (.not.dolw) .and. is_first_step() ) then + qrl(1:ncol,1:pver) = 0._r8 + flnt(1:ncol) = 0._r8 + flns(1:ncol) = 0._r8 + end if + + !----------- + if (dosw .or. dolw) then ! construct an RRTMG state object diff --git a/components/eam/src/utils/hycoef.F90 b/components/eam/src/utils/hycoef.F90 index 7fb4317fa741..821c543a3f3a 100644 --- a/components/eam/src/utils/hycoef.F90 +++ b/components/eam/src/utils/hycoef.F90 @@ -301,6 +301,9 @@ subroutine hycoef_read(File) ierr = pio_get_var(File, hyam_desc, hyam) ierr = pio_get_var(File, hybm_desc, hybm) + ! Make sure hybi(1) is zero, in order to be consistent with pressure calculations in the SE dycore. + if (hybi(1) .ne. 0._r8) call endrun(routine//':ERROR: hybi(1) is non-zero.') + #if ( defined OFFLINE_DYN ) ! make sure top interface is non zero for fv dycore if (hyai(1) .eq. 0._r8) then diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index 0ca08c437807..5a39e19dafdd 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -51,11 +51,6 @@ if (SCREAM_CIME_BUILD) ${CMAKE_CURRENT_SOURCE_DIR}/cmake/cime) endif () -if (NOT CMAKE_CXX_STANDARD) - # Default to C++17 in EAMxx - set(CMAKE_CXX_STANDARD 17) -endif() - if (NOT SCREAM_CIME_BUILD) project(SCREAM CXX C Fortran) @@ -462,6 +457,12 @@ endif() # Configure all tpls and subfolders # #################################################################### +# Set compiler-specific flags. Do it BEFORE ekat, so ekat gets the flags too +include(EkatSetCompilerFlags) +ResetFlags() +SetCommonFlags() +SetProfilingFlags(PROFILER ${EKAT_PROFILING_TOOL} COVERAGE ${EKAT_ENABLE_COVERAGE}) + # We will use the sharedlib one in CIME builds if (NOT SCREAM_CIME_BUILD) set (EKAT_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../externals/ekat) @@ -500,12 +501,6 @@ if (NOT SCREAM_CIME_BUILD) add_subdirectory(${EKAT_SOURCE_DIR} ${CMAKE_BINARY_DIR}/externals/ekat) endif() -# Set compiler-specific flags -include(EkatSetCompilerFlags) -ResetFlags() -SetCommonFlags() -SetProfilingFlags(PROFILER ${EKAT_PROFILING_TOOL} COVERAGE ${EKAT_ENABLE_COVERAGE}) - include(EkatMpiUtils) # We should avoid cxx bindings in mpi; they are already deprecated, # and can cause headaches at link time, cause they require -lmpi_cxx diff --git a/components/eamxx/cime_config/namelist_defaults_eamxx.xml b/components/eamxx/cime_config/namelist_defaults_eamxx.xml index 29c61a6260fe..3e206a3b8b67 100644 --- a/components/eamxx/cime_config/namelist_defaults_eamxx.xml +++ b/components/eamxx/cime_config/namelist_defaults_eamxx.xml @@ -296,6 +296,16 @@ be lost if SCREAM_HACK_XML is not enabled. 0.0 + + + 0 + + + + + 0 + + false @@ -435,26 +445,26 @@ be lost if SCREAM_HACK_XML is not enabled. - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/DMSflux.2010.ne30pg2_conserv.POPmonthlyClimFromACES4BGC_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so2_surf_ne30pg2_2010_clim_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_bc_a4_surf_ne30pg2_2010_clim_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a1_surf_ne30pg2_2010_clim_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a2_surf_ne30pg2_2010_clim_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a4_surf_ne30pg2_2010_clim_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_pom_a4_surf_ne30pg2_2010_clim_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so4_a1_surf_ne30pg2_2010_clim_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so4_a2_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/DMSflux.2010.ne30pg2_conserv.POPmonthlyClimFromACES4BGC_c20260730.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so2_surf_ne30pg2_2010_clim_c20260730.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_bc_a4_surf_ne30pg2_2010_clim_c20260730.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a1_surf_ne30pg2_2010_clim_c20260730.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a2_surf_ne30pg2_2010_clim_c20260730.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a4_surf_ne30pg2_2010_clim_c20260730.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_pom_a4_surf_ne30pg2_2010_clim_c20260730.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so4_a1_surf_ne30pg2_2010_clim_c20260730.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so4_a2_surf_ne30pg2_2010_clim_c20260730.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/DMSflux.2010.ne4pg2_conserv.POPmonthlyClimFromACES4BGC_c20240814.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so2_surf_ne4pg2_2010_clim_c20240815.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_bc_a4_surf_ne4pg2_2010_clim_c20240815.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a1_surf_ne4pg2_2010_clim_c20240815.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a2_surf_ne4pg2_2010_clim_c20240815.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a4_surf_ne4pg2_2010_clim_c20240815.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_pom_a4_surf_ne4pg2_2010_clim_c20240815.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a1_surf_ne4pg2_2010_clim_c20240815.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a2_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/DMSflux.2010.ne4pg2_conserv.POPmonthlyClimFromACES4BGC_c20260730.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so2_surf_ne4pg2_2010_clim_c20260730.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_bc_a4_surf_ne4pg2_2010_clim_c20260730.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a1_surf_ne4pg2_2010_clim_c20260730.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a2_surf_ne4pg2_2010_clim_c20260730.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a4_surf_ne4pg2_2010_clim_c20260730.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_pom_a4_surf_ne4pg2_2010_clim_c20260730.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a1_surf_ne4pg2_2010_clim_c20260730.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a2_surf_ne4pg2_2010_clim_c20260730.nc 1.0 @@ -559,6 +569,8 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/scream/init/spc_from_v3LRamip_2010_clim_ne30pg2_c20260206.nc ${DIN_LOC_ROOT}/atm/scream/init/spc_from_v3LRamip_2010_clim_ne4pg2_c20260211.nc + o3 + yearly_periodic @@ -915,6 +927,7 @@ be lost if SCREAM_HACK_XML is not enabled. 6 2 3.0 + 0.0 1 1 1 diff --git a/components/eamxx/cmake/machine-files/vista.cmake b/components/eamxx/cmake/machine-files/vista.cmake new file mode 100644 index 000000000000..e37504d5de04 --- /dev/null +++ b/components/eamxx/cmake/machine-files/vista.cmake @@ -0,0 +1,37 @@ +include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) +common_setup() + +#message(STATUS "vista PROJECT_NAME=${PROJECT_NAME} USE_CUDA=${USE_CUDA} KOKKOS_ENABLE_CUDA=${KOKKOS_ENABLE_CUDA}") +if ("${PROJECT_NAME}" STREQUAL "E3SM") + if (USE_CUDA) + include (${EKAT_MACH_FILES_PATH}/kokkos/nvidia-h100.cmake) # H100=Hopper=Ampere90=Hopper90 Kokkos_ARCH_HOPPER90 + include (${EKAT_MACH_FILES_PATH}/kokkos/cuda.cmake) + else() + include (${EKAT_MACH_FILES_PATH}/kokkos/nvidia-grace.cmake) # KOKKOS_ARCH_ARMV9_GRACE + include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) + #include (${EKAT_MACH_FILES_PATH}/kokkos/serial.cmake) + endif() +else() + include (${EKAT_MACH_FILES_PATH}/kokkos/nvidia-h100.cmake) + include (${EKAT_MACH_FILES_PATH}/kokkos/cuda.cmake) +endif() + +include (${EKAT_MACH_FILES_PATH}/mpi/srun.cmake) # should be changed to use ibrun + +set(EKAT_MPI_EXTRA_ARGS "${EKAT_MPI_EXTRA_ARGS} --gpus-per-task=1" CACHE STRING "" FORCE) + +#option(Kokkos_ARCH_AMPERE90 "" ON) +set(CMAKE_CXX_FLAGS "-DTHRUST_IGNORE_CUB_VERSION_CHECK" CACHE STRING "" FORCE) + +#set(CMAKE_CUDA_FLAGS "-allow-unsupported-compiler" CACHE STRING "" FORCE) #ndk try for gcc14 + +#message(STATUS "vista CMAKE_CXX_COMPILER_ID=${CMAKE_CXX_COMPILER_ID} CMAKE_Fortran_COMPILER_VERSION=${CMAKE_Fortran_COMPILER_VERSION}") +if ("${PROJECT_NAME}" STREQUAL "E3SM") + if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + if (CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER_EQUAL 10) + set(CMAKE_Fortran_FLAGS "-fallow-argument-mismatch" CACHE STRING "" FORCE) # only works with gnu v10 and above + endif() + endif() +else() + set(CMAKE_Fortran_FLAGS "-fallow-argument-mismatch" CACHE STRING "" FORCE) # only works with gnu v10 and above +endif() diff --git a/components/eamxx/src/control/atmosphere_driver.cpp b/components/eamxx/src/control/atmosphere_driver.cpp index 516b8d9b03f4..79cb73268974 100644 --- a/components/eamxx/src/control/atmosphere_driver.cpp +++ b/components/eamxx/src/control/atmosphere_driver.cpp @@ -312,6 +312,10 @@ void AtmosphereDriver::create_grids() setup_shoc_tms_links(); } + if (m_atm_process_group->has_process("shoc")) { + setup_shoc_3d_turbulence_link(); + } + // IOP object needs the grids_manager to have been created, but is then needed in set_grids() // implementation of some processes, so setup here. const bool enable_iop = @@ -518,6 +522,21 @@ void AtmosphereDriver::setup_shoc_tms_links () shoc_process->get_params().set("apply_tms", true); } +void AtmosphereDriver::setup_shoc_3d_turbulence_link () +{ + EKAT_REQUIRE_MSG(m_atm_process_group->has_process("shoc"), + "Error! Attempting to setup 3D turbulence link for " + "SHOC, but SHOC is not defined.\n"); + + if (m_atm_process_group->has_process("homme")) { + auto homme_process = m_atm_process_group->get_process_nonconst("homme"); + const bool do_3d_turbulence = homme_process->get_params().get("do_3d_turbulence", false); + + auto shoc_process = m_atm_process_group->get_process_nonconst("shoc"); + shoc_process->get_params().set("do_3d_turbulence_shoc", do_3d_turbulence); + } +} + void AtmosphereDriver::add_additional_column_data_to_property_checks () { // Get list of additional data fields from driver_options parameters. // If no fields given, return. @@ -609,33 +628,37 @@ void AtmosphereDriver::create_fields() } // Now go through the input fields/groups to the atm proc group, - // and mark them as part of the RESTART group. + // and mark them as part of the RESTART/STARTUP/TOPOGRAPHY groups. // Skip fields in the ACCUMULATED group, since those are reset to 0 // at the beginning of each atm step, so there is no need to read // them from the IC or restart file. - for (const auto& f : m_atm_process_group->get_fields_in()) { + auto is_topography_field = [] (const std::string& name) { + return name=="phis" or name=="sgh" or name=="sgh30"; + }; + + auto set_groups = [&](const Field& f) { const auto& fid = f.get_header().get_identifier(); const auto& fgroups = f.get_header().get_tracking().get_groups_names(); if (not ekat::contains(fgroups, "ACCUMULATED")) { m_field_mgr->add_to_group(fid, "RESTART"); - } - } - for (const auto& g : m_atm_process_group->get_groups_in()) { - if (g.m_monolithic_field) { - const auto& mf = *g.m_monolithic_field; - const auto& mfgroups = mf.get_header().get_tracking().get_groups_names(); - if (not ekat::contains(mfgroups, "ACCUMULATED")) { - m_field_mgr->add_to_group(mf.get_header().get_identifier(), "RESTART"); - } - } else { - for (const auto& fn : g.m_info->m_fields_names) { - auto field = m_field_mgr->get_field(fn, g.grid_name()); - const auto& fgroups = field.get_header().get_tracking().get_groups_names(); - if (not ekat::contains(fgroups, "ACCUMULATED")) { - m_field_mgr->add_to_group(fn, g.grid_name(), "RESTART"); - } + m_field_mgr->add_to_group(fid, "STARTUP"); + if (is_topography_field(fid.name())) { + m_field_mgr->add_to_group(fid, "TOPOGRAPHY"); } } + }; + + // Process input fields + for (const auto& f : m_atm_process_group->get_fields_in()) + set_groups(f); + + // Process input groups + for (const auto& g : m_atm_process_group->get_groups_in()) { + if (g.m_monolithic_field) + set_groups(*g.m_monolithic_field); + else + for (const auto& it : g.m_individual_fields) + set_groups(*it.second); } auto& driver_options_pl = m_atm_params.sublist("driver_options"); diff --git a/components/eamxx/src/control/atmosphere_driver.hpp b/components/eamxx/src/control/atmosphere_driver.hpp index c3d6735d7abb..fcc83ab43645 100644 --- a/components/eamxx/src/control/atmosphere_driver.hpp +++ b/components/eamxx/src/control/atmosphere_driver.hpp @@ -105,6 +105,9 @@ class AtmosphereDriver // tms' surface drag coefficient. void setup_shoc_tms_links(); + // Propagate HOMME's parsed 3D turbulence flag to SHOC's internal runtime option. + void setup_shoc_3d_turbulence_link(); + // Add column data to all pre/postcondition property checks // for use in output. void add_additional_column_data_to_property_checks (); diff --git a/components/eamxx/src/dynamics/homme/CMakeLists.txt b/components/eamxx/src/dynamics/homme/CMakeLists.txt index 59750c48e278..b7c68de22783 100644 --- a/components/eamxx/src/dynamics/homme/CMakeLists.txt +++ b/components/eamxx/src/dynamics/homme/CMakeLists.txt @@ -150,6 +150,7 @@ macro (CreateDynamicsLib HOMME_TARGET NP PLEV QSIZE) ${SCREAM_DYNAMICS_SRC_DIR}/eamxx_homme_rayleigh_friction.cpp ${SCREAM_DYNAMICS_SRC_DIR}/physics_dynamics_remapper.cpp ${SCREAM_DYNAMICS_SRC_DIR}/homme_grids_manager.cpp + ${SCREAM_DYNAMICS_SRC_DIR}/eamxx_homme_3d_turbulence_strain.cpp ${SCREAM_DYNAMICS_SRC_DIR}/interface/homme_context_mod.F90 ${SCREAM_DYNAMICS_SRC_DIR}/interface/homme_driver_mod.F90 ${SCREAM_DYNAMICS_SRC_DIR}/interface/homme_grid_mod.F90 diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_3d_turbulence_strain.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_3d_turbulence_strain.cpp new file mode 100644 index 000000000000..97d8f9144e2e --- /dev/null +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_3d_turbulence_strain.cpp @@ -0,0 +1,249 @@ +#include "eamxx_homme_process_interface.hpp" + +// HOMMEXX includes +#include "Context.hpp" +#include "ElementsGeometry.hpp" +#include "ElementsState.hpp" +#include "ReferenceElement.hpp" +#include "TimeLevel.hpp" +#include "Types.hpp" +#include "utilities/ViewUtils.hpp" + +// Scream includes +#include "dynamics/homme/homme_dimensions.hpp" +#include "share/util/eamxx_column_ops.hpp" + +// EKAT includes +#include + +namespace scream +{ + +namespace { + +// Project a local velocity vector (u,v,w) into one Cartesian component. +template +KOKKOS_INLINE_FUNCTION +Real local_to_cart_component( + const BasisViewType& basis_sph2cart, + const Real u, + const Real v, + const Real w) +{ + return basis_sph2cart(0) * u + + basis_sph2cart(1) * v + + basis_sph2cart(2) * w; +} + +} // anonymous namespace + +void HommeDynamics::compute_horizontal_derivs_of_car_velocity () +{ + using namespace Homme; + + constexpr int NGP = HOMMEXX_NP; + + const auto& c = Context::singleton(); + const auto& state = c.get(); + const auto& geom = c.get(); + const auto& ref_fe = c.get(); + const auto& tl = c.get(); + + const int nelem = m_dyn_grid->get_num_local_dofs() / (NGP*NGP); + const int n0 = tl.n0; + const auto& grad_Ux_field = m_helper_fields.at("grad_Ux_dyn"); + const auto& grad_Ux_layout = grad_Ux_field.get_header().get_identifier().get_layout(); + const int nlev_scalar = grad_Ux_layout.dims().back(); + + const auto w_int_dyn = state.m_w_i; + + auto grad_Ux_dyn = m_helper_fields.at("grad_Ux_dyn").template get_view(); + auto grad_Uy_dyn = m_helper_fields.at("grad_Uy_dyn").template get_view(); + auto grad_Uz_dyn = m_helper_fields.at("grad_Uz_dyn").template get_view(); + + const auto dvv = ref_fe.get_deriv(); + const auto dinv = geom.m_dinv; + const auto vec_sph2cart = geom.m_vec_sph2cart; + const Real scale_factor_inv = 1.0 / geom.m_scale_factor; + + using TeamPolicy = Kokkos::TeamPolicy; + using MemberType = typename TeamPolicy::member_type; + const int ncols = nelem*NGP*NGP; + const TeamPolicy policy(ncols, Kokkos::AUTO()); + const auto dsdx_Ux_all = m_dsdx_Ux_all; + const auto dsdy_Ux_all = m_dsdy_Ux_all; + const auto dsdx_Uy_all = m_dsdx_Uy_all; + const auto dsdy_Uy_all = m_dsdy_Uy_all; + const auto dsdx_Uz_all = m_dsdx_Uz_all; + const auto dsdy_Uz_all = m_dsdy_Uz_all; + + Kokkos::parallel_for( + "compute_horizontal_derivs_of_car_velocity", + policy, + KOKKOS_LAMBDA (const MemberType& team) { + + const int ie = team.league_rank() / (NGP*NGP); + const int igp = (team.league_rank() / NGP) % NGP; + const int jgp = team.league_rank() % NGP; + const int icol = team.league_rank(); + + // Grab the scratch storage associated with this (ie,igp,jgp) column. + const auto dsdx_Ux = Kokkos::subview(dsdx_Ux_all, icol, Kokkos::ALL()); + const auto dsdy_Ux = Kokkos::subview(dsdy_Ux_all, icol, Kokkos::ALL()); + const auto dsdx_Uy = Kokkos::subview(dsdx_Uy_all, icol, Kokkos::ALL()); + const auto dsdy_Uy = Kokkos::subview(dsdy_Uy_all, icol, Kokkos::ALL()); + const auto dsdx_Uz = Kokkos::subview(dsdx_Uz_all, icol, Kokkos::ALL()); + const auto dsdy_Uz = Kokkos::subview(dsdy_Uz_all, icol, Kokkos::ALL()); + + // Accumulate reference-element derivatives in the two local horizontal directions. + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_scalar), [&] (const int ilev) { + dsdx_Ux(ilev) = 0; + dsdy_Ux(ilev) = 0; + dsdx_Uy(ilev) = 0; + dsdy_Uy(ilev) = 0; + dsdx_Uz(ilev) = 0; + dsdy_Uz(ilev) = 0; + }); + team.team_barrier(); + + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_scalar), [&] (const int ilev) { + Real dsdx_ux = 0; + Real dsdy_ux = 0; + Real dsdx_uy = 0; + Real dsdy_uy = 0; + Real dsdx_uz = 0; + Real dsdy_uz = 0; + + for (int kgp = 0; kgp < NGP; ++kgp) { + // The horizontal stencil uses interface w, so average the two + // adjacent interface values onto midpoint levels on the fly. + const auto w_row_i = Kokkos::subview(w_int_dyn, ie, n0, igp, kgp, Kokkos::ALL()); + const auto w_col_i = Kokkos::subview(w_int_dyn, ie, n0, kgp, jgp, Kokkos::ALL()); + const auto w_row_i_real = Homme::viewAsReal(w_row_i); + const auto w_col_i_real = Homme::viewAsReal(w_col_i); + + const auto row_x = Kokkos::subview(vec_sph2cart, ie, Kokkos::ALL(), 0, igp, kgp); + const auto row_y = Kokkos::subview(vec_sph2cart, ie, Kokkos::ALL(), 1, igp, kgp); + const auto row_z = Kokkos::subview(vec_sph2cart, ie, Kokkos::ALL(), 2, igp, kgp); + const auto col_x = Kokkos::subview(vec_sph2cart, ie, Kokkos::ALL(), 0, kgp, jgp); + const auto col_y = Kokkos::subview(vec_sph2cart, ie, Kokkos::ALL(), 1, kgp, jgp); + const auto col_z = Kokkos::subview(vec_sph2cart, ie, Kokkos::ALL(), 2, kgp, jgp); + const auto u_row_view = + Homme::viewAsReal(Kokkos::subview(state.m_v, ie, n0, 0, igp, kgp, Kokkos::ALL())); + const auto v_row_view = + Homme::viewAsReal(Kokkos::subview(state.m_v, ie, n0, 1, igp, kgp, Kokkos::ALL())); + const auto u_col_view = + Homme::viewAsReal(Kokkos::subview(state.m_v, ie, n0, 0, kgp, jgp, Kokkos::ALL())); + const auto v_col_view = + Homme::viewAsReal(Kokkos::subview(state.m_v, ie, n0, 1, kgp, jgp, Kokkos::ALL())); + + const Real u_row = u_row_view(ilev); + const Real v_row = v_row_view(ilev); + const Real w_row = 0.5 * (w_row_i_real(ilev) + w_row_i_real(ilev + 1)); + + const Real u_col = u_col_view(ilev); + const Real v_col = v_col_view(ilev); + const Real w_col = 0.5 * (w_col_i_real(ilev) + w_col_i_real(ilev + 1)); + + dsdx_ux += dvv(jgp,kgp) * local_to_cart_component(row_x, u_row, v_row, w_row); + dsdy_ux += dvv(igp,kgp) * local_to_cart_component(col_x, u_col, v_col, w_col); + + dsdx_uy += dvv(jgp,kgp) * local_to_cart_component(row_y, u_row, v_row, w_row); + dsdy_uy += dvv(igp,kgp) * local_to_cart_component(col_y, u_col, v_col, w_col); + + dsdx_uz += dvv(jgp,kgp) * local_to_cart_component(row_z, u_row, v_row, w_row); + dsdy_uz += dvv(igp,kgp) * local_to_cart_component(col_z, u_col, v_col, w_col); + } + + dsdx_Ux(ilev) = dsdx_ux; + dsdy_Ux(ilev) = dsdy_ux; + dsdx_Uy(ilev) = dsdx_uy; + dsdy_Uy(ilev) = dsdy_uy; + dsdx_Uz(ilev) = dsdx_uz; + dsdy_Uz(ilev) = dsdy_uz; + }); + team.team_barrier(); + + // Convert the reference-element derivatives into physical horizontal + // gradients using the inverse metric tensor on this curved element. + const auto dinv_ij = Kokkos::subview(dinv, ie, Kokkos::ALL(), Kokkos::ALL(), igp, jgp); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_scalar), [&] (const int ilev) { + grad_Ux_dyn(ie,0,igp,jgp,ilev) = (dinv_ij(0,0) * dsdx_Ux(ilev) + dinv_ij(0,1) * dsdy_Ux(ilev)) * scale_factor_inv; + grad_Uy_dyn(ie,0,igp,jgp,ilev) = (dinv_ij(0,0) * dsdx_Uy(ilev) + dinv_ij(0,1) * dsdy_Uy(ilev)) * scale_factor_inv; + grad_Uz_dyn(ie,0,igp,jgp,ilev) = (dinv_ij(0,0) * dsdx_Uz(ilev) + dinv_ij(0,1) * dsdy_Uz(ilev)) * scale_factor_inv; + + grad_Ux_dyn(ie,1,igp,jgp,ilev) = (dinv_ij(1,0) * dsdx_Ux(ilev) + dinv_ij(1,1) * dsdy_Ux(ilev)) * scale_factor_inv; + grad_Uy_dyn(ie,1,igp,jgp,ilev) = (dinv_ij(1,0) * dsdx_Uy(ilev) + dinv_ij(1,1) * dsdy_Uy(ilev)) * scale_factor_inv; + grad_Uz_dyn(ie,1,igp,jgp,ilev) = (dinv_ij(1,0) * dsdx_Uz(ilev) + dinv_ij(1,1) * dsdy_Uz(ilev)) * scale_factor_inv; + }); + }); + + Kokkos::fence(); +} + +void HommeDynamics::compute_local_strain_components3d () +{ + using namespace Homme; + + constexpr int NGP = HOMMEXX_NP; + + const auto& c = Context::singleton(); + const auto& geom = c.get(); + + const int nelem = m_dyn_grid->get_num_local_dofs() / (NGP*NGP); + + auto grad_Ux_dyn = m_helper_fields.at("grad_Ux_dyn").template get_view(); + auto grad_Uy_dyn = m_helper_fields.at("grad_Uy_dyn").template get_view(); + auto grad_Uz_dyn = m_helper_fields.at("grad_Uz_dyn").template get_view(); + + auto& shear_components_field = m_helper_fields.at("shear_strain3d_components_dyn"); + const auto& shear_components_layout = shear_components_field.get_header().get_identifier().get_layout(); + auto shear_components_dyn = shear_components_field.template get_view(); + + const auto vec_sph2cart = geom.m_vec_sph2cart; + + const int nlev_scalar = shear_components_layout.dims().back(); + + using Policy = Kokkos::MDRangePolicy>; + const Policy policy({0, 0, 0, 0}, {nelem, NGP, NGP, nlev_scalar}); + + Kokkos::parallel_for( + "compute_local_strain_components3d", + policy, + KOKKOS_LAMBDA (const int ie, const int igp, const int jgp, const int ilev) { + + // The stored gradients are Cartesian components differentiated along the + // two local horizontal directions. Project them back into the local basis + // so SHOC receives the six local shear-tensor components it expects. + const Real gx0 = grad_Ux_dyn(ie,0,igp,jgp,ilev); + const Real gy0 = grad_Uy_dyn(ie,0,igp,jgp,ilev); + const Real gz0 = grad_Uz_dyn(ie,0,igp,jgp,ilev); + + const Real gx1 = grad_Ux_dyn(ie,1,igp,jgp,ilev); + const Real gy1 = grad_Uy_dyn(ie,1,igp,jgp,ilev); + const Real gz1 = grad_Uz_dyn(ie,1,igp,jgp,ilev); + + const Real b0_0 = vec_sph2cart(ie, 0, 0, igp, jgp); + const Real b0_1 = vec_sph2cart(ie, 0, 1, igp, jgp); + const Real b0_2 = vec_sph2cart(ie, 0, 2, igp, jgp); + + const Real b1_0 = vec_sph2cart(ie, 1, 0, igp, jgp); + const Real b1_1 = vec_sph2cart(ie, 1, 1, igp, jgp); + const Real b1_2 = vec_sph2cart(ie, 1, 2, igp, jgp); + + const Real b2_0 = vec_sph2cart(ie, 2, 0, igp, jgp); + const Real b2_1 = vec_sph2cart(ie, 2, 1, igp, jgp); + const Real b2_2 = vec_sph2cart(ie, 2, 2, igp, jgp); + + shear_components_dyn(ie,0,igp,jgp,ilev) = b0_0 * gx0 + b0_1 * gy0 + b0_2 * gz0; + shear_components_dyn(ie,1,igp,jgp,ilev) = b0_0 * gx1 + b0_1 * gy1 + b0_2 * gz1; + shear_components_dyn(ie,2,igp,jgp,ilev) = b1_0 * gx0 + b1_1 * gy0 + b1_2 * gz0; + shear_components_dyn(ie,3,igp,jgp,ilev) = b1_0 * gx1 + b1_1 * gy1 + b1_2 * gz1; + shear_components_dyn(ie,4,igp,jgp,ilev) = b2_0 * gx0 + b2_1 * gy0 + b2_2 * gz0; + shear_components_dyn(ie,5,igp,jgp,ilev) = b2_0 * gx1 + b2_1 * gy1 + b2_2 * gz1; + }); + + Kokkos::fence(); +} + +} // namespace scream diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_fv_phys.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_fv_phys.cpp index a27f966889a8..8f21a676dea2 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_fv_phys.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_fv_phys.cpp @@ -139,6 +139,11 @@ void HommeDynamics::fv_phys_dyn_to_fv_phys (const util::TimeStamp& ts, const boo auto f = get_field_out(n,pgn); f.get_header().get_tracking().update_time_stamp(ts); } + const auto& params = Homme::Context::singleton().get(); + if (params.do_3d_turbulence) { + auto f = get_field_out("tke_shear_strain3d_components",pgn); + f.get_header().get_tracking().update_time_stamp(ts); + } auto Q = get_group_out("tracers",pgn).m_monolithic_field; Q->get_header().get_tracking().update_time_stamp(ts); } @@ -196,7 +201,21 @@ void HommeDynamics::remap_dyn_to_fv_phys (GllFvRemapTmp* t) const { get_field_out("pseudo_density", gn).get_view().data(), nelem, npg, nlev); - gfr.run_dyn_to_fv_phys(time_idx, ps, phis, T, omega, uv, q, &dp); + const auto& params = c.get(); + if (params.do_3d_turbulence) { + const auto strain3d_components_gll = Homme::GllFvRemap::CPhys3T( + m_helper_fields.at("shear_strain3d_components_dyn").get_view().data(), + nelem, NGP*NGP, 6, nlev); + const auto strain3d_components_fv = Homme::GllFvRemap::Phys3T( + get_field_out("tke_shear_strain3d_components", gn).get_view().data(), + nelem, npg, 6, nlev); + gfr.run_dyn_to_fv_phys(time_idx, ps, phis, T, omega, + &strain3d_components_gll, &strain3d_components_fv, + uv, q, &dp); + } else { + gfr.run_dyn_to_fv_phys(time_idx, ps, phis, T, omega, + nullptr, nullptr, uv, q, &dp); + } Kokkos::fence(); } @@ -216,10 +235,6 @@ void HommeDynamics::remap_fv_phys_to_dyn () const { const auto uv_ndim = m_helper_fields.at("FM_phys").get_view().extent_int(1); assert(uv_ndim == 2); - // SGS Eddy diffusivities on FV phys grid - const auto Km_phys = get_field_in("eddy_diff_mom",gn).get_view(); - const auto Kh_phys = get_field_in("eddy_diff_heat",gn).get_view(); - const auto T = Homme::GllFvRemap::CPhys2T( m_helper_fields.at("FT_phys").get_view().data(), nelem, npg, nlev); @@ -230,10 +245,17 @@ void HommeDynamics::remap_fv_phys_to_dyn () const { get_group_in("tracers", gn).m_monolithic_field->get_view().data(), nelem, npg, nq, nlev); - const auto Km = Homme::GllFvRemap::CPhys2T(Km_phys.data(), nelem, npg, nlev); - const auto Kh = Homme::GllFvRemap::CPhys2T(Kh_phys.data(), nelem, npg, nlev); + const auto& params = c.get(); + if (params.do_3d_turbulence) { + const auto Km_phys = get_field_in("eddy_diff_mom",gn).get_view(); + const auto Kh_phys = get_field_in("eddy_diff_heat",gn).get_view(); + const auto Km = Homme::GllFvRemap::CPhys2T(Km_phys.data(), nelem, npg, nlev); + const auto Kh = Homme::GllFvRemap::CPhys2T(Kh_phys.data(), nelem, npg, nlev); + gfr.run_fv_phys_to_dyn(time_idx, T, uv, q, &Km, &Kh); + } else { + gfr.run_fv_phys_to_dyn(time_idx, T, uv, q); + } - gfr.run_fv_phys_to_dyn(time_idx, T, uv, q, Km, Kh); Kokkos::fence(); gfr.run_fv_phys_to_dyn_dss(); Kokkos::fence(); diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp index 4884fc5154c4..cca2e0b58f9f 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp @@ -176,8 +176,12 @@ void HommeDynamics::create_requests () add_field("p_dry_int", pg_scalar3d_int, Pa, pgn,N); add_field("p_dry_mid", pg_scalar3d_mid, Pa, pgn,N); add_field("omega", pg_scalar3d_mid, Pa/s, pgn,N); - add_field("eddy_diff_heat", pg_scalar3d_mid, m2/s, pgn,N); - add_field("eddy_diff_mom", pg_scalar3d_mid, m2/s, pgn,N); + if (params.do_3d_turbulence) { + add_field("eddy_diff_heat", pg_scalar3d_mid, m2/s, pgn,N); + add_field("eddy_diff_mom", pg_scalar3d_mid, m2/s, pgn,N); + auto pg_shear_components_mid = m_phys_grid->get_3d_vector_layout(LEV,6); + add_field("tke_shear_strain3d_components", pg_shear_components_mid, 1/s, pgn,N); + } add_tracer("qv", m_phys_grid, kg/kg, N); add_group("tracers",pgn,N, MonolithicAlloc::Required); @@ -212,8 +216,14 @@ void HommeDynamics::create_requests () create_helper_field("phis_dyn", {EL, GP,GP}, {nelem, NP,NP }, dgn); create_helper_field("omega_dyn", {EL, GP,GP,LEV}, {nelem, NP,NP,nlev_mid}, dgn); create_helper_field("Qdp_dyn", {EL,TL,CMP,GP,GP,LEV}, {nelem,QTL,HOMMEXX_QSIZE_D,NP,NP,nlev_mid},dgn); - create_helper_field("Km_dyn", {EL, GP,GP,LEV}, {nelem, NP,NP,nlev_mid}, dgn); - create_helper_field("Kh_dyn", {EL, GP,GP,LEV}, {nelem, NP,NP,nlev_mid}, dgn); + if (params.do_3d_turbulence) { + create_helper_field("Km_dyn", {EL, GP,GP,LEV}, {nelem, NP,NP,nlev_mid}, dgn); + create_helper_field("Kh_dyn", {EL, GP,GP,LEV}, {nelem, NP,NP,nlev_mid}, dgn); + create_helper_field("grad_Ux_dyn", {EL,CMP, GP,GP,LEV}, {nelem,2, NP,NP,nlev_mid}, dgn); + create_helper_field("grad_Uy_dyn", {EL,CMP, GP,GP,LEV}, {nelem,2, NP,NP,nlev_mid}, dgn); + create_helper_field("grad_Uz_dyn", {EL,CMP, GP,GP,LEV}, {nelem,2, NP,NP,nlev_mid}, dgn); + create_helper_field("shear_strain3d_components_dyn", {EL,CMP,GP,GP,LEV}, {nelem,6,NP,NP,nlev_mid}, dgn); + } // For BFB restart, we need to read in the state on the dyn grid. The state above has NTL time slices, // but only one is really needed for restart. Therefore, we create "dynamic" subfields for @@ -272,6 +282,8 @@ void HommeDynamics::create_requests () size_t HommeDynamics::requested_buffer_size_in_bytes() const { using namespace Homme; + constexpr int num_turb3d_scratch_buffers = 8; + constexpr int np2 = HOMMEXX_NP*HOMMEXX_NP; auto& c = Context::singleton(); auto& params = c.get(); @@ -314,7 +326,13 @@ size_t HommeDynamics::requested_buffer_size_in_bytes() const } fv_phys_requested_buffer_size_in_bytes(); - return fbm.allocated_size()*sizeof(Real); + size_t requested_bytes = fbm.allocated_size()*sizeof(Real); + if (params.do_3d_turbulence) { + const size_t ncols = num_elems*np2; + requested_bytes += num_turb3d_scratch_buffers*sizeof(Real)*ncols*NUM_PHYSICAL_LEV; + } + + return requested_bytes; } void HommeDynamics::init_buffers(const ATMBufferManager &buffer_manager) @@ -325,14 +343,36 @@ void HommeDynamics::init_buffers(const ATMBufferManager &buffer_manager) using namespace Homme; auto& c = Context::singleton(); auto& fbm = c.get(); + const auto& params = c.get(); // Reset Homme buffer to use AD buffer memory. // Internally, homme will actually initialize its own buffers. EKAT_REQUIRE(buffer_manager.allocated_bytes()%sizeof(Real)==0); // Sanity check + const int fbm_size = fbm.allocated_size(); Real* mem = reinterpret_cast(buffer_manager.get_memory()); - fbm.allocate(mem, buffer_manager.allocated_bytes()/sizeof(Real)); - mem += fbm.allocated_size(); + fbm.allocate(mem, fbm_size); + mem += fbm_size; + + if (params.do_3d_turbulence) { + constexpr int np2 = HOMMEXX_NP*HOMMEXX_NP; + const int ncols = c.get().num_elems()*np2; + const int scratch_col_size = ncols*NUM_PHYSICAL_LEV; + + auto assign_scratch = [&](HommeDynamics::fixed_view_2d_phys& view) { + view = HommeDynamics::fixed_view_2d_phys(mem, ncols); + mem += scratch_col_size; + }; + + assign_scratch(m_w_mid_row_all); + assign_scratch(m_w_mid_col_all); + assign_scratch(m_dsdx_Ux_all); + assign_scratch(m_dsdy_Ux_all); + assign_scratch(m_dsdx_Uy_all); + assign_scratch(m_dsdy_Uy_all); + assign_scratch(m_dsdx_Uz_all); + assign_scratch(m_dsdy_Uz_all); + } size_t used_mem = (mem - buffer_manager.get_memory())*sizeof(Real); EKAT_REQUIRE_MSG(used_mem==requested_buffer_size_in_bytes(), @@ -352,6 +392,15 @@ void HommeDynamics::initialize_impl (const RunType run_type) const auto& c = Homme::Context::singleton(); const auto& params = c.get(); + // The first fv_phys D->P remap during initialization happens before the + // dycore has computed these diagnostic components, so start from a benign + // value. Homme overwrites them after each dynamics step when 3D turbulence is + // enabled. + if (params.do_3d_turbulence) { + m_helper_fields.at("shear_strain3d_components_dyn").deep_copy(0); + get_field_out("tke_shear_strain3d_components").deep_copy(0); + } + // Complete Homme prim_init1_xyz sequence prim_complete_init1_phase_f90 (); @@ -426,9 +475,14 @@ void HommeDynamics::initialize_impl (const RunType run_type) m_d2p_remapper->register_field(m_helper_fields.at("Q_dyn"),*get_group_out("Q",pgn).m_monolithic_field); m_d2p_remapper->register_field(m_helper_fields.at("omega_dyn"), get_field_out("omega")); - // Remap SHOC eddy diffusivities from physics grid to dynamics grid - m_p2d_remapper->register_field(get_field_in("eddy_diff_mom",pgn),m_helper_fields.at("Km_dyn")); - m_p2d_remapper->register_field(get_field_in("eddy_diff_heat",pgn),m_helper_fields.at("Kh_dyn")); + if (params.do_3d_turbulence) { + // Remap SHOC eddy diffusivities from physics grid to dynamics grid. + m_p2d_remapper->register_field(get_field_in("eddy_diff_mom",pgn),m_helper_fields.at("Km_dyn")); + m_p2d_remapper->register_field(get_field_in("eddy_diff_heat",pgn),m_helper_fields.at("Kh_dyn")); + + // Remap horizontal/local strain tensor components from dynamics to physics grid. + m_d2p_remapper->register_field(m_helper_fields.at("shear_strain3d_components_dyn"), get_field_out("tke_shear_strain3d_components")); + } m_p2d_remapper->registration_ends(); m_d2p_remapper->registration_ends(); @@ -523,6 +577,14 @@ void HommeDynamics::run_impl (const double dt) prim_run_f90(/* nsplit_iteration = */ subiter+1); } + // This is where we will compute the strain term needed for Shear Production of TKE + if (params.do_3d_turbulence){ + compute_horizontal_derivs_of_car_velocity(); + compute_local_strain_components3d(); + } else if (params.do_3d_turbulence) { + m_helper_fields.at("shear_strain3d_components_dyn").deep_copy(0.0); + } + // Update nstep in the restart extra data, so it can be written to restart if needed. const auto& tl = c.get(); std::any_cast(*m_restart_extra_data["homme_nsteps"]) = tl.nstep; @@ -943,15 +1005,17 @@ void HommeDynamics::init_homme_views () { // by EAMxx, so we set FM(3)=0 right away m_helper_fields.at("FM_dyn").get_component(2).deep_copy(0); - // SGS Eddy diffusivity for momentum - auto Km_in = m_helper_fields.at("Km_dyn").template get_view(); - using turb_type_mom = std::remove_reference::type; - derived.m_turb_diff_mom = turb_type_mom(Km_in.data(), nelem); + if (params.do_3d_turbulence) { + // SGS Eddy diffusivity for momentum + auto Km_in = m_helper_fields.at("Km_dyn").template get_view(); + using turb_type_mom = std::remove_reference::type; + derived.m_turb_diff_mom = turb_type_mom(Km_in.data(), nelem); - // SGS Eddy diffusivity for heat - auto Kh_in = m_helper_fields.at("Kh_dyn").template get_view(); - using turb_type_heat = std::remove_reference::type; - derived.m_turb_diff_heat = turb_type_heat(Kh_in.data(), nelem); + // SGS Eddy diffusivity for heat + auto Kh_in = m_helper_fields.at("Kh_dyn").template get_view(); + using turb_type_heat = std::remove_reference::type; + derived.m_turb_diff_heat = turb_type_heat(Kh_in.data(), nelem); + } } diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp index a1534d6f8ac6..d73799912831 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp @@ -3,6 +3,7 @@ #include "share/atm_process/atmosphere_process.hpp" #include "share/remap/abstract_remapper.hpp" +#include "dynamics/homme/homme_dimensions.hpp" #include #include @@ -36,6 +37,8 @@ class HommeDynamics : public AtmosphereProcess using uview_1d = ekat::Unmanaged>; template using uview_2d = ekat::Unmanaged>; + using fixed_view_2d_phys = Kokkos::View::array_layout, + DefaultDevice, Kokkos::MemoryTraits>; public: @@ -79,6 +82,9 @@ class HommeDynamics : public AtmosphereProcess void initialize_impl (const RunType run_type); + void compute_horizontal_derivs_of_car_velocity (); + void compute_local_strain_components3d (); + // fv_phys refers to the horizontal finite volume (FV) grid for column // parameterizations nested inside the horizontal element grid. The grid names // are "physics_pgn", where N in practice is 2. The name of each routine is @@ -157,6 +163,16 @@ class HommeDynamics : public AtmosphereProcess Real m_raytau0; // Approximate value of decay time at model top (days) // if set to 0, no rayleigh friction is applied + // Scratch reused by the 3D turbulence strain kernels when that feature is active. + fixed_view_2d_phys m_w_mid_row_all; + fixed_view_2d_phys m_w_mid_col_all; + fixed_view_2d_phys m_dsdx_Ux_all; + fixed_view_2d_phys m_dsdy_Ux_all; + fixed_view_2d_phys m_dsdx_Uy_all; + fixed_view_2d_phys m_dsdy_Uy_all; + fixed_view_2d_phys m_dsdx_Uz_all; + fixed_view_2d_phys m_dsdy_Uz_all; + int m_bfb_hash_nstep; }; diff --git a/components/eamxx/src/dynamics/homme/interface/homme_context_mod.F90 b/components/eamxx/src/dynamics/homme/interface/homme_context_mod.F90 index 441172723148..e49822fd5d1e 100644 --- a/components/eamxx/src/dynamics/homme/interface/homme_context_mod.F90 +++ b/components/eamxx/src/dynamics/homme/interface/homme_context_mod.F90 @@ -63,7 +63,7 @@ subroutine set_homme_log_file_name_f90(c_str) bind(c) ! character(len=256), pointer :: full_name character(len=256) :: path, fname - integer :: len, slash, ierr + integer :: len, slash call c_f_pointer(c_str,full_name) len = index(full_name, C_NULL_CHAR) -1 @@ -79,22 +79,20 @@ subroutine set_homme_log_file_name_f90(c_str) bind(c) homme_log_fname = trim(path)//"homme_"//fname - iulog = shr_file_getunit() + ! Only the root rank opens/owns the homme log file (following the same + ! approach used by EAM in atm_comp_mct.F90/atm_comp_esmf.F90). All other + ! ranks simply keep iulog at its module default (stdout), rather than + ! all ranks independently opening the same shared file, which is racy + ! and can lead to corrupted/interleaved writes or spurious I/O errors. if (masterproc) then - ! Create the homme log file on root rank... + iulog = shr_file_getunit() open (unit=iulog,file=trim(homme_log_fname),status='REPLACE', & action='WRITE', access='SEQUENTIAL', position="append") write(iulog,*) " ---- HOMME LOG FILE ----" flush(iulog) - endif - call mpi_barrier(par%comm,ierr) - if (.not. masterproc) then - ! ... and open it on all other ranks - open (unit=iulog,file=trim(homme_log_fname),status='OLD', & - action='WRITE', access='SEQUENTIAL', position="append") - endif - homme_log_set = .true. + homme_log_set = .true. + endif endif end subroutine set_homme_log_file_name_f90 diff --git a/components/eamxx/src/dynamics/homme/interface/homme_driver_mod.F90 b/components/eamxx/src/dynamics/homme/interface/homme_driver_mod.F90 index 809ce948f66f..9cf334442a6b 100644 --- a/components/eamxx/src/dynamics/homme/interface/homme_driver_mod.F90 +++ b/components/eamxx/src/dynamics/homme/interface/homme_driver_mod.F90 @@ -183,7 +183,8 @@ end subroutine prim_copy_cxx_to_f90 subroutine prim_init_model_f90 () bind(c) use prim_driver_mod, only: prim_init_ref_states_views, & prim_init_diags_views, prim_init_kokkos_functors, & - prim_init_state_views + prim_init_state_views, prim_init_tensorvisc, & + prim_init_tensorvisc2 use prim_state_mod, only: prim_printstate use model_init_mod, only: model_init2 use global_norms_mod, only: dss_hvtensor, print_cfl @@ -210,6 +211,15 @@ subroutine prim_init_model_f90 () bind(c) ! Apply dss and bilinear projection to tensor coefficients call dss_hvtensor(elem,hybrid,1,nelemd) + ! Update the C++ tensorVisc view with dss_hvtensor's result (the other, + ! constant, geometry views were already sent to C++ earlier, in + ! prim_complete_init1_phase_f90 -> prim_init_grid_views). + call prim_init_tensorvisc (elem) + + ! Same as above, but for tensorVisc_2 (the sponge-layer tensor + ! coefficient), which dss_hvtensor also updates. + call prim_init_tensorvisc2 (elem) + ! Print advective and viscious CFL estimates call print_cfl(elem,hybrid,1,nelemd) diff --git a/components/eamxx/src/dynamics/homme/interface/homme_params_mod.F90 b/components/eamxx/src/dynamics/homme/interface/homme_params_mod.F90 index 9a3df59b6161..8afa3c8722ae 100644 --- a/components/eamxx/src/dynamics/homme/interface/homme_params_mod.F90 +++ b/components/eamxx/src/dynamics/homme/interface/homme_params_mod.F90 @@ -112,7 +112,7 @@ function get_homme_int_param_f90 (param_name_c) result(param_value) bind(c) end function get_homme_int_param_f90 function get_homme_real_param_f90 (param_name_c) result(param_value) bind(c) - use control_mod, only: nu, nu_div, nu_p, nu_q, nu_s, hypervis_scaling + use control_mod, only: nu, nu_div, nu_p, nu_q, nu_s, hypervis_scaling, laplace_scaling use time_mod, only: tstep ! ! Input(s) @@ -140,6 +140,8 @@ function get_homme_real_param_f90 (param_name_c) result(param_value) bind(c) param_value = nu_s case("hypervis_scaling") param_value = hypervis_scaling + case("laplace_scaling") + param_value = laplace_scaling case("dt") param_value = tstep case default @@ -150,7 +152,7 @@ function get_homme_real_param_f90 (param_name_c) result(param_value) bind(c) end function get_homme_real_param_f90 function get_homme_bool_param_f90 (param_name_c) result(param_value) bind(c) - use control_mod, only: moisture + use control_mod, only: moisture, do_3d_turbulence ! ! Input(s) ! @@ -171,6 +173,8 @@ function get_homme_bool_param_f90 (param_name_c) result(param_value) bind(c) else param_value = .true. endif + case("do_3d_turbulence") + param_value = do_3d_turbulence case default call abortmp ("[get_homme_bool_param_f90] Error! Unrecognized parameter name.") param_value = .false. diff --git a/components/eamxx/src/physics/CMakeLists.txt b/components/eamxx/src/physics/CMakeLists.txt index 296b0b8b50bc..e8d282f1bd6d 100644 --- a/components/eamxx/src/physics/CMakeLists.txt +++ b/components/eamxx/src/physics/CMakeLists.txt @@ -20,3 +20,4 @@ if (SCREAM_ENABLE_MAM) add_subdirectory(mam) endif() add_subdirectory(gw) +add_subdirectory(specialized_tracers) diff --git a/components/eamxx/src/physics/mam/CMakeLists.txt b/components/eamxx/src/physics/mam/CMakeLists.txt index 0ea5c1b7d05a..d23a7f5d0a4b 100644 --- a/components/eamxx/src/physics/mam/CMakeLists.txt +++ b/components/eamxx/src/physics/mam/CMakeLists.txt @@ -8,9 +8,9 @@ endif() # configure and build mam4xx (C++ port of MAM4) if (SCREAM_DOUBLE_PRECISION) - set(MAM4XX_PRECISION "double" CACHE BOOL "Enable double precision for mam4xx") + set(MAM4XX_PRECISION "double" CACHE STRING "Enable double precision for mam4xx") else() - set(MAM4XX_PRECISION "single" CACHE BOOL "Enable single precision for mam4xx") + set(MAM4XX_PRECISION "single" CACHE STRING "Enable single precision for mam4xx") endif() set(MAM4XX_ENABLE_GPU ${EAMXX_ENABLE_GPU} CACHE BOOL "Enable mam4xx GPU configuration" FORCE) set(MAM4XX_ENABLE_TESTS OFF CACHE BOOL "Disable mam4xx tests within E3SM" FORCE) @@ -37,11 +37,11 @@ add_library(mam target_compile_definitions(mam PUBLIC EAMXX_HAS_MAM) target_link_libraries(mam PUBLIC eamxx_physics_share csm_share scream_share mam4xx) -#if (NOT SCREAM_LIB_ONLY) -# add_subdirectory(tests) -#endif() - if (TARGET eamxx_physics) # Add this library to eamxx_physics target_link_libraries(eamxx_physics INTERFACE mam) endif() + +if (NOT SCREAM_LIB_ONLY) + add_subdirectory(tests) +endif() diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp index ca89de44144b..0d8cff061336 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp @@ -271,29 +271,14 @@ void call_function_dropmixnuc( //--------------------------------------------------------------------------- // ## Initialize the ndrop class. //--------------------------------------------------------------------------- - const int ntot_amode = mam_coupling::num_aero_modes(); - const int maxd_aspectype = mam4::ndrop::maxd_aspectype; - const int nspec_max = mam4::ndrop::nspec_max; - int nspec_amode[ntot_amode] = {}; - int lspectype_amode[maxd_aspectype][ntot_amode] = {}; - int lmassptr_amode[maxd_aspectype][ntot_amode] = {}; - int numptr_amode[ntot_amode] = {}; - int mam_idx[ntot_amode][nspec_max] = {}; - int mam_cnst_idx[ntot_amode][nspec_max] = {}; - - Real specdens_amode[maxd_aspectype] = {}; - Real spechygro[maxd_aspectype] = {}; + const int ntot_amode = mam_coupling::num_aero_modes(); Real exp45logsig[ntot_amode] = {}, alogsig[ntot_amode] = {}, num2vol_ratio_min_nmodes[ntot_amode] = {}, num2vol_ratio_max_nmodes[ntot_amode] = {}; - Real aten = 0; - mam4::ndrop::get_e3sm_parameters(nspec_amode, lspectype_amode, lmassptr_amode, - numptr_amode, specdens_amode, spechygro, - mam_idx, mam_cnst_idx); + Real aten = 0; mam4::ndrop::ndrop_init(exp45logsig, alogsig, aten, num2vol_ratio_min_nmodes, num2vol_ratio_max_nmodes); //--------------------------------------------------------------------------- - //--------------------------------------------------------------------------- const bool local_enable_aero_vertical_mix = enable_aero_vertical_mix; Kokkos::parallel_for( "MAMAci::run_impl::call_function_dropmixnuc", team_policy, @@ -372,10 +357,8 @@ void call_function_dropmixnuc( // in zm[kk] - zm[kk+1], for pver zm[kk-1] - zm[kk] ekat::subview(zm, icol), ekat::subview(state_q_work_loc, icol), ekat::subview(nc, icol), ekat::subview(kvh_int, icol), // kvh[kk+1] - ekat::subview(cloud_frac, icol), lspectype_amode, specdens_amode, - spechygro, lmassptr_amode, num2vol_ratio_min_nmodes, - num2vol_ratio_max_nmodes, numptr_amode, nspec_amode, exp45logsig, - alogsig, aten, mam_idx, mam_cnst_idx, + ekat::subview(cloud_frac, icol), num2vol_ratio_min_nmodes, + num2vol_ratio_max_nmodes, exp45logsig, alogsig, aten, local_enable_aero_vertical_mix, ekat::subview(qcld, icol), // out ekat::subview(wsub, icol), // in ekat::subview(cloud_frac_prev, icol), // in diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 2ae04f5586f7..90b8a4fc387c 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -210,15 +210,9 @@ void MAMSrfOnlineEmiss::create_requests() { srf_emiss_species_.push_back(so4_a2); //-------------------------------------------------------------------- - // Init data structures to read and interpolate + // Register sector fields in FM for surface emissions. + // DataInterpolation is set up in initialize_impl. //-------------------------------------------------------------------- - for(srf_emiss_ &ispec_srf : srf_emiss_species_) { - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, ispec_srf.data_file, ispec_srf.sectors, srf_map_file, - // output - ispec_srf.horizInterp_, ispec_srf.data_start_, ispec_srf.data_end_, - ispec_srf.data_out_, ispec_srf.dataReader_); - } // srf emissions file read init // ------------------------------------------------------------- // Setup to enable reading soil erodibility file @@ -316,33 +310,45 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // Output fields // --------------------------------------------------------------- // Constituent fluxes of species in [kg/m2/s] - constituent_fluxes_ = get_field_out("constituent_fluxes").get_view(); - - // --------------------------------------------------------------- - // Allocate memory for local and work arrays - // --------------------------------------------------------------- - - // Work array to store fluxes after unit conversions to kg/m2/s - fluxes_in_mks_units_ = view_1d("fluxes_in_mks_units", ncol_); - - // Current month ( 0-based) - const int curr_month = start_of_step_ts().get_month() - 1; - - // Load the first month into data_end. - - // Note: At the first time step, the data will be moved into data_beg, - // and data_end will be reloaded from file with the new month. + constituent_fluxes_ = get_field_out("constituent_fluxes"); //-------------------------------------------------------------------- - // Update surface emissions from file + // Setup data interpolation for surface emissions. //-------------------------------------------------------------------- - for(srf_emiss_ &ispec_srf : srf_emiss_species_) { - srfEmissFunc::update_srfEmiss_data_from_file( - ispec_srf.dataReader_, start_of_step_ts(), curr_month, - ispec_srf.scale_factor, *ispec_srf.horizInterp_, - ispec_srf.data_end_); // output + { + using namespace ekat::units; + using namespace ShortFieldTagsNames; + const FieldLayout scalar2d = grid_->get_2d_scalar_layout(); + const auto srf_map_file = m_params.get("srf_remap_file", ""); + const auto srf_time_interp = DataInterpolation::Linear; + for(srf_emiss_ &ispec_srf : srf_emiss_species_) { + std::vector srf_fields; + srf_fields.reserve(ispec_srf.sectors.size()); + for(const auto §or_name : ispec_srf.sectors) { + Field field(FieldIdentifier(sector_name, scalar2d, none, grid_->name())); + field.allocate_view(); + srf_fields.push_back(field); + } + ispec_srf.emiss_sector_fields_ = srf_fields; + + ispec_srf.data_interp_ = std::make_shared(grid_, srf_fields); + ispec_srf.data_interp_->set_logger(m_atm_logger); + ispec_srf.data_interp_->setup_periodic_time_database( + {ispec_srf.data_file}); + ispec_srf.data_interp_->create_horiz_remappers( + srf_map_file == "none" ? "" : srf_map_file); + + DataInterpolation::VertRemapData remap_data; + remap_data.vr_type = DataInterpolation::None; + ispec_srf.data_interp_->create_vert_remapper(remap_data); + + ispec_srf.data_interp_->init_time_interpolation(start_of_step_ts(), srf_time_interp); + } } + // Current month ( 0-based) + const int curr_month = start_of_step_ts().get_month() - 1; + //----------------------------------------------------------------- // Read Soil erodibility data //----------------------------------------------------------------- @@ -386,7 +392,7 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { Kokkos::fence(); // Constituent fluxes [kg/m^2/s] - auto constituent_fluxes = this->constituent_fluxes_; + auto constituent_fluxes = constituent_fluxes_.get_view(); // Zero out constituent fluxes only for gasses and aerosols init_fluxes(ncol_, // in @@ -456,18 +462,7 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { //-------------------------------------------------------------------- for(srf_emiss_ &ispec_srf : srf_emiss_species_) { - // Update TimeState, note the addition of dt - ispec_srf.timeState_.t_now = ts.frac_of_year_in_days(); - - // Update time state and if the month has changed, update the data. - srfEmissFunc::update_srfEmiss_timestate( - ispec_srf.dataReader_, ts, *ispec_srf.horizInterp_, ispec_srf.scale_factor, - // output - ispec_srf.timeState_, ispec_srf.data_start_, ispec_srf.data_end_); - - // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(ispec_srf.timeState_, ispec_srf.data_start_, - ispec_srf.data_end_, ispec_srf.data_out_); + ispec_srf.data_interp_->run(ts); //-------------------------------------------------------------------- // Modify units to MKS units (from molecules/cm2/s to kg/m2/s) @@ -476,18 +471,17 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // constituent_fluxes_) const int species_index = spcIndex_in_pcnst_.at(ispec_srf.species_name); + auto constituent_fluxes_ispe_srf = constituent_fluxes_.get_component(species_index); // modify units from molecules/cm2/s to kg/m2/s - auto fluxes_in_mks_units = this->fluxes_in_mks_units_; - const Real mfactor = - amufac * mam4::gas_chemistry::adv_mass[species_index - offset_]; - const view_1d ispec_outdata0 = - ekat::subview(ispec_srf.data_out_.emiss_sectors, 0); - // Parallel loop over all the columns to update units - Kokkos::parallel_for( - "srf_emis_fluxes", ncol_, KOKKOS_LAMBDA(int icol) { - fluxes_in_mks_units(icol) = ispec_outdata0(icol) * mfactor; - constituent_fluxes(icol, species_index) = fluxes_in_mks_units(icol); - }); + constituent_fluxes_ispe_srf.deep_copy(0.0); + + for(const auto §or_field : ispec_srf.emiss_sector_fields_) { + constituent_fluxes_ispe_srf.update(sector_field, 1, 1); + } + + const Real mfactor = amufac * ispec_srf.scale_factor * + mam4::gas_chemistry::adv_mass[species_index - offset_]; + constituent_fluxes_ispe_srf.scale(mfactor); } // for loop for species Kokkos::fence(); } // run_impl ends diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 1f2115bbfaf7..3bce8d01b59e 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -2,10 +2,10 @@ #define EAMXX_MAM_SRF_ONLINE_EMISS_HPP #include "share/remap/abstract_remapper.hpp" +#include "share/algorithm/eamxx_data_interpolation.hpp" // For MAM4 aerosol configuration #include -#include // For reading marine organics file #include @@ -42,15 +42,12 @@ class MAMSrfOnlineEmiss final : public MAMGenericInterface { const_view_2d dust_fluxes_; // Constituent fluxes of species in [kg/m2/s] - view_2d constituent_fluxes_; + Field constituent_fluxes_; // Runtime scale factors for online emissions from namelist. Real dust_emis_scale_factor; Real seasalt_emis_scale_factor; - // Work array to store fluxes after unit conversions to kg/m2/s - view_1d fluxes_in_mks_units_; - // Unified atomic mass unit used for unit conversion (BAD constant) static constexpr Real amufac = 1.65979e-23; // 1.e4* kg / amu @@ -61,7 +58,6 @@ class MAMSrfOnlineEmiss final : public MAMGenericInterface { public: // For reading surface emissions and marine organics file - using srfEmissFunc = mam_coupling::srfEmissFunctions; using marineOrganicsFunc = marine_organics::marineOrganicsFunctions; @@ -153,12 +149,9 @@ class MAMSrfOnlineEmiss final : public MAMGenericInterface { // Species-specific scale factor Real scale_factor = 1.0; - // Data structure for reading interpolation - std::shared_ptr horizInterp_; - std::shared_ptr dataReader_; - srfEmissFunc::srfEmissTimeState timeState_; - srfEmissFunc::srfEmissInput data_start_, data_end_; - srfEmissFunc::srfEmissOutput data_out_; + // Data interpolation object and local output fields for each file sector. + std::shared_ptr data_interp_; + std::vector emiss_sector_fields_; }; // A vector for carrying emissions for all the species diff --git a/components/eamxx/src/physics/mam/readfiles/photo_table_utils.cpp b/components/eamxx/src/physics/mam/readfiles/photo_table_utils.cpp index 101761d363b9..e9a24aca37b2 100644 --- a/components/eamxx/src/physics/mam/readfiles/photo_table_utils.cpp +++ b/components/eamxx/src/physics/mam/readfiles/photo_table_utils.cpp @@ -6,6 +6,8 @@ namespace scream::impl { using mam4::mo_photo::phtcnt; using HostView1D = mam4::DeviceType::view_1d::host_mirror_type; +using HostView5D = mam4::DeviceType::view::host_mirror_type; +using HostView3D = mam4::DeviceType::view::host_mirror_type; using HostViewInt1D = mam4::DeviceType::view_1d::host_mirror_type; //------------------------------------------------------------------------- @@ -19,29 +21,33 @@ std::vector populate_etfphot_from_e3sm_case() { // We obtained these values from an e3sm simulations. // We should only use this function on Host. std::vector etfphot_data = { - 7.5691227E+11, 8.6525905E+11, 1.0355749E+12, 1.1846288E+12, 2.1524405E+12, - 3.2362584E+12, 3.7289849E+12, 4.4204330E+12, 4.6835350E+12, 6.1217728E+12, - 4.5575051E+12, 5.3491446E+12, 4.7016063E+12, 5.4281722E+12, 4.5023968E+12, - 6.8931981E+12, 6.2012647E+12, 6.1430771E+12, 5.7820385E+12, 7.6770646E+12, - 1.3966509E+13, 1.2105348E+13, 2.8588980E+13, 3.2160821E+13, 2.4978066E+13, - 2.7825401E+13, 2.3276451E+13, 3.6343684E+13, 6.1787886E+13, 7.8009914E+13, - 7.6440824E+13, 7.6291458E+13, 9.4645085E+13, 1.0124628E+14, 1.0354111E+14, - 1.0999650E+14, 1.0889946E+14, 1.1381912E+14, 1.3490042E+14, 1.5941519E+14, - 1.4983265E+14, 1.5184267E+14, 1.5991420E+14, 1.6976697E+14, 1.8771840E+14, - 1.6434367E+14, 1.8371960E+14, 2.1966369E+14, 1.9617879E+14, 2.2399700E+14, - 1.8429912E+14, 2.0129736E+14, 2.0541588E+14, 2.4334962E+14, 3.5077122E+14, - 3.4517894E+14, 3.5749668E+14, 3.6624304E+14, 3.4975113E+14, 3.5566025E+14, - 4.2825273E+14, 4.8406375E+14, 4.9511159E+14, 5.2695368E+14, 5.2401611E+14, - 5.0877746E+14, 4.8780853E+14}; + 0.75691227453241626E+012, 0.86525904597344678E+012, 0.10355748678445210E+013, 0.11846288453143215E+013, 0.21524405047611838E+013, + 0.32362583636383438E+013, 0.37289849127086353E+013, 0.44204330229059023E+013, 0.46835350139683008E+013, 0.61217728454045146E+013, + 0.45575051094967529E+013, 0.53491446243876533E+013, 0.47016062694342764E+013, 0.54281722298247529E+013, 0.45023968313414365E+013, + 0.68931981401230361E+013, 0.62012647462481055E+013, 0.61430770669364131E+013, 0.57820384729408037E+013, 0.76770646262530391E+013, + 0.13966508541416857E+014, 0.12105347510143980E+014, 0.28588979654418141E+014, 0.32160820948665508E+014, 0.24978065543030500E+014, + 0.27825400776036188E+014, 0.23276451219415352E+014, 0.36343683716296695E+014, 0.61787885646314477E+014, 0.78009914475741344E+014, + 0.76440824240882500E+014, 0.76291457600771391E+014, 0.94645085080390984E+014, 0.10124627769922270E+015, 0.10354111421691689E+015, + 0.10999649606948711E+015, 0.10889946060495367E+015, 0.11381912455165878E+015, 0.13490042469475880E+015, 0.15941519351184984E+015, + 0.14983265369952531E+015, 0.15184267258496494E+015, 0.15991419729740088E+015, 0.16976696691694741E+015, 0.18771840486614825E+015, + 0.16434366552645634E+015, 0.18371960453616509E+015, 0.21966368981040753E+015, 0.19617878628663241E+015, 0.22399700059898819E+015, + 0.18429911731380941E+015, 0.20129735694980109E+015, 0.20541588491339825E+015, 0.24334961879677731E+015, 0.35077121778312700E+015, + 0.34517894220011569E+015, 0.35749668154179594E+015, 0.36624304237331069E+015, 0.34975112547690056E+015, 0.35566025203681831E+015, + 0.42825273260963562E+015, 0.48406375456076200E+015, 0.49511158653410975E+015, 0.52695367706176038E+015, 0.52401610578239200E+015, + 0.50877746346978994E+015, 0.48780852943692825E+015}; return etfphot_data; } // This version uses eamxx_scorpio_interface to read netcdf files. mam4::mo_photo::PhotoTableData read_photo_table( - const std::string &rsf_file, const std::string &xs_long_file) { - // set up the lng_indexer and pht_alias_mult_1 views based on our - // (hardwired) chemical mechanism - HostViewInt1D lng_indexer_h("lng_indexer", phtcnt); + const std::string &rsf_file, const std::string &xs_long_file, + const std::vector &rxt_names, const int numj, + const HostViewInt1D &lng_indexer_h) { + + EKAT_REQUIRE_MSG(numj > 0, "Error: read_photo_table requires numj > 0.\n"); + EKAT_REQUIRE_MSG(lng_indexer_h.extent_int(0) == phtcnt, + "Error: read_photo_table requires lng_indexer_h sized by phtcnt.\n"); + int nw, nump, numsza, numcolo3, numalb, nt, np_xs; // table dimensions scorpio::register_file(rsf_file, scorpio::Read); @@ -56,15 +62,13 @@ mam4::mo_photo::PhotoTableData read_photo_table( nw = scorpio::get_dimlen(xs_long_file, "numwl"); np_xs = scorpio::get_dimlen(xs_long_file, "numprs"); - // FIXME: hard-coded for only one photo reaction. - std::string rxt_names[1] = {"jh2o2"}; - int numj = 1; - lng_indexer_h(0) = 0; // allocate the photolysis table auto table = mam4::mo_photo::create_photo_table_data( nw, nt, np_xs, numj, nump, numsza, numcolo3, numalb); // allocate host views for table data + HostView5D l_rsf_tab_h("rsf_tab_h",numalb,numcolo3,numsza,nump,nw); + HostView3D l_xsqy_h("xsqy_h",np_xs,nt,nw); auto rsf_tab_h = Kokkos::create_mirror_view(table.rsf_tab); auto xsqy_h = Kokkos::create_mirror_view(table.xsqy); auto sza_h = Kokkos::create_mirror_view(table.sza); @@ -72,8 +76,8 @@ mam4::mo_photo::PhotoTableData read_photo_table( auto press_h = Kokkos::create_mirror_view(table.press); auto colo3_h = Kokkos::create_mirror_view(table.colo3); auto o3rat_h = Kokkos::create_mirror_view(table.o3rat); - // auto etfphot_h = Kokkos::create_mirror_view(table.etfphot); auto prs_h = Kokkos::create_mirror_view(table.prs); + // read file data into our host views scorpio::read_var(rsf_file, "pm", press_h.data()); @@ -81,17 +85,19 @@ mam4::mo_photo::PhotoTableData read_photo_table( scorpio::read_var(rsf_file, "alb", alb_h.data()); scorpio::read_var(rsf_file, "colo3fact", o3rat_h.data()); scorpio::read_var(rsf_file, "colo3", colo3_h.data()); - // it produces an error. - scorpio::read_var(rsf_file, "RSF", rsf_tab_h.data()); + scorpio::read_var(rsf_file, "RSF", l_rsf_tab_h.data()); scorpio::read_var(xs_long_file, "pressure", prs_h.data()); // read xsqy data (using lng_indexer_h for the first index) - // FIXME: hard-coded for only one photo reaction. - for(int m = 0; m < phtcnt; ++m) { - auto xsqy_ndx_h = ekat::subview(xsqy_h, m); - scorpio::read_var(xs_long_file, rxt_names[m], xsqy_h.data()); + using policy_t3 = Kokkos::MDRangePolicy, Kokkos::DefaultHostExecutionSpace>; + for(int m = 0; m < numj; ++m) { + scorpio::read_var(xs_long_file, rxt_names[m], l_xsqy_h.data()); + Kokkos::parallel_for("xsqy_h", + policy_t3({0, 0, 0}, {xsqy_h.extent(1), xsqy_h.extent(2), xsqy_h.extent(3)}), + [&](const int i, const int j, const int k) { + xsqy_h(m, i, j, k) = l_xsqy_h(k,j,i); + }); } - // populate etfphot by rebinning solar data HostView1D wc_h("wc", nw), wlintv_h("wlintv", nw), we_h("we", nw + 1); @@ -106,6 +112,16 @@ mam4::mo_photo::PhotoTableData read_photo_table( auto etfphot_data = populate_etfphot_from_e3sm_case(); auto etfphot_h = HostView1D((Real *)etfphot_data.data(), nw); + using policy_t = Kokkos::MDRangePolicy, Kokkos::DefaultHostExecutionSpace>; + + Kokkos::parallel_for("scale_rsf_tab", + policy_t({0, 0, 0, 0}, {rsf_tab_h.extent(1), rsf_tab_h.extent(2), rsf_tab_h.extent(3), rsf_tab_h.extent(4)}), + [&](const int l, const int i, const int j, const int k) { + for (int w = 0; w < nw; ++w) { + rsf_tab_h(w,l, i, j, k) = l_rsf_tab_h(k,j,i,l,w)*wlintv_h(w); + } + }); + scorpio::release_file(rsf_file); scorpio::release_file(xs_long_file); @@ -148,4 +164,15 @@ mam4::mo_photo::PhotoTableData read_photo_table( return table; } +// MAM4xx E3SM v2 photolysis table reader. +mam4::mo_photo::PhotoTableData read_photo_table( + const std::string &rsf_file, const std::string &xs_long_file) { + + HostViewInt1D lng_indexer_h("lng_indexer", phtcnt); + std::vector rxt_names = {"jh2o2"}; + int numj = 1; + lng_indexer_h(0) = 0; + return read_photo_table(rsf_file, xs_long_file, rxt_names, numj, lng_indexer_h); +} + } // namespace scream::impl diff --git a/components/eamxx/src/physics/mam/tests/CMakeLists.txt b/components/eamxx/src/physics/mam/tests/CMakeLists.txt new file mode 100644 index 000000000000..66f4da18ba4a --- /dev/null +++ b/components/eamxx/src/physics/mam/tests/CMakeLists.txt @@ -0,0 +1,17 @@ +include(ScreamUtils) + +if (NOT SCREAM_ONLY_GENERATE_BASELINES) + CreateUnitTest(mam_photo_table_test + SOURCES mam_photo_table_test.cpp + LIBS mam eamxx_scorpio_interface eamxx_io ekat::YamlParser + LABELS "mam;physics" + MPI_RANKS 1 + THREADS 1 + ) + target_compile_definitions(mam_photo_table_test PRIVATE + SCREAM_DATA_DIR="${SCREAM_DATA_DIR}" + ) + # Ensure test input files are present in the data dir + GetInputFile(scream/mam4xx/photolysis/table_photo_input_ts_355.yaml) + GetInputFile(scream/mam4xx/photolysis/jlong_input_ts_355.yaml) +endif() diff --git a/components/eamxx/src/physics/mam/tests/mam_photo_table_test.cpp b/components/eamxx/src/physics/mam/tests/mam_photo_table_test.cpp new file mode 100644 index 000000000000..56ffb6f0bffe --- /dev/null +++ b/components/eamxx/src/physics/mam/tests/mam_photo_table_test.cpp @@ -0,0 +1,401 @@ +#include + +#include + +#include +#include + +#include "share/core/eamxx_types.hpp" + +#include +#include +#include + +#include "share/scorpio_interface/eamxx_scorpio_interface.hpp" + +namespace scream { +namespace impl { + +mam4::mo_photo::PhotoTableData read_photo_table( + const std::string& rsf_file, const std::string& xs_long_file); + +} // namespace impl +} // namespace scream + +namespace mam_photo_table { + +using Real = scream::Real; +using HostSpace = Kokkos::HostSpace; + +using Device = scream::DefaultDevice; +using ExecSpace = Device::execution_space; +using KT = ekat::KokkosTypes; +using view_1d = typename KT::template view_1d; +using view_2d = typename KT::template view_2d; +using view_3d = typename KT::template view_3d; +using TeamPolicy = Kokkos::TeamPolicy; +using MemberType = TeamPolicy::member_type; + +inline bool nearly_equal(const Real a, const Real b, + const Real rtol = 1e-8, + const Real atol = 1e-14) { + return std::abs(a - b) <= atol + rtol * std::abs(b); +} + +// Read a double-valued sequence from a ParameterList and convert to Real. +std::vector get_real_vec(const ekat::ParameterList& pl, + const std::string& key) { + const auto& dv = pl.get>(key); + return std::vector(dv.begin(), dv.end()); +} + +template struct PrecisionTolerance; + +template <> struct PrecisionTolerance { + static constexpr float relative_tol = 1e-5f; // Single precision tolerance + static constexpr float absolute_tol = 1e-8f; // Single precision tolerance +}; + +template <> struct PrecisionTolerance { + static constexpr double relative_tol = 1e-8; // Double precision tolerance + static constexpr double absolute_tol = 1e-12; // Double precision tolerance +}; + +} // namespace mam_photo_table + +using namespace mam_photo_table; + +TEST_CASE("mam_photo_table_yaml_reference_regression", + "[mam4][photo][kokkos]") { + if constexpr (mam4::nlev != 72) return; + using namespace scream; + + ekat::Comm comm(MPI_COMM_WORLD); + struct ScorpioGuard { + explicit ScorpioGuard(const ekat::Comm& comm) : comm_(comm) { + scorpio::init_subsystem(comm_); + } + ~ScorpioGuard() { + scorpio::finalize_subsystem(); + } + const ekat::Comm& comm_; + } scorpio_guard(comm); + + const std::string rsf_file = + std::string(SCREAM_DATA_DIR) + "/mam4xx/photolysis/RSF_GT200nm_v3.0_c080811.nc"; + const std::string xs_long_file = + std::string(SCREAM_DATA_DIR) + "/mam4xx/photolysis/temp_prs_GT200nm_JPL10_c130206.nc"; + const std::string input_yaml_file = std::string(SCREAM_DATA_DIR) + "/mam4xx/photolysis/jlong_input_ts_355.yaml"; + + const auto photo_table = scream::impl::read_photo_table(rsf_file, xs_long_file); + const auto root = ekat::parse_yaml_file(input_yaml_file); + REQUIRE(root.isSublist("input")); + REQUIRE(root.sublist("input").isSublist("fixed")); + const auto& fixed = root.sublist("input").sublist("fixed"); + + REQUIRE(photo_table.nw > 0); + REQUIRE(photo_table.numj == 1); + + auto sza_h = Kokkos::create_mirror_view_and_copy(HostSpace(), photo_table.sza); + auto del_sza_h = Kokkos::create_mirror_view_and_copy(HostSpace(), photo_table.del_sza); + auto alb_h = Kokkos::create_mirror_view_and_copy(HostSpace(), photo_table.alb); + auto del_alb_h = Kokkos::create_mirror_view_and_copy(HostSpace(), photo_table.del_alb); + auto colo3_h = Kokkos::create_mirror_view_and_copy(HostSpace(), photo_table.colo3); + auto o3rat_h = Kokkos::create_mirror_view_and_copy(HostSpace(), photo_table.o3rat); + auto del_o3rat_h = Kokkos::create_mirror_view_and_copy(HostSpace(), photo_table.del_o3rat); + auto press_h = Kokkos::create_mirror_view_and_copy(HostSpace(), photo_table.press); + auto prs_h = Kokkos::create_mirror_view_and_copy(HostSpace(), photo_table.prs); + auto dprs_h = Kokkos::create_mirror_view_and_copy(HostSpace(), photo_table.dprs); + auto rsf_tab_h = Kokkos::create_mirror_view_and_copy(HostSpace(), photo_table.rsf_tab); + auto xsqy_h = Kokkos::create_mirror_view_and_copy(HostSpace(), photo_table.xsqy); + auto etfphot_h = Kokkos::create_mirror_view_and_copy(HostSpace(), photo_table.etfphot); + auto lng_indexer_h = Kokkos::create_mirror_view_and_copy(HostSpace(), photo_table.lng_indexer); + auto pht_alias_mult_h = + Kokkos::create_mirror_view_and_copy(HostSpace(), photo_table.pht_alias_mult_1); + + const auto nw_ref = fixed.get>("nw")[0]; + const auto numj_ref = fixed.get>("numj")[0]; + const auto shape_ref = fixed.get>("shape_of_rsf_tab"); + REQUIRE(shape_ref.size() == 5); + const int nw_shape = shape_ref[0]; + const int nump_shape = shape_ref[1]; + const int numsza_shape = shape_ref[2]; + const int numcolo3_shape = shape_ref[3]; + const int numalb_shape = shape_ref[4]; + + const auto sza_ref = get_real_vec(fixed, "sza"); + const auto del_sza_ref = get_real_vec(fixed, "del_sza"); + const auto alb_ref = get_real_vec(fixed, "alb"); + const auto del_alb_ref = get_real_vec(fixed, "del_alb"); + const auto colo3_ref = get_real_vec(fixed, "colo3"); + const auto o3rat_ref = get_real_vec(fixed, "o3rat"); + const auto del_o3rat_ref = get_real_vec(fixed, "del_o3rat"); + const auto press_ref = fixed.isParameter("press") ? + get_real_vec(fixed, "press") : + get_real_vec(fixed, "pm"); + REQUIRE(!press_ref.empty()); + const auto etfphot_ref = get_real_vec(fixed, "etfphot"); + const auto prs_ref = get_real_vec(fixed, "prs"); + const auto dprs_ref = get_real_vec(fixed, "dprs"); + const auto rsf_tab_2d = get_real_vec(fixed, "rsf_tab_2d"); + const auto xsqy_2d = get_real_vec(fixed, "xsqy_2d"); + + SECTION("dimensions_match_expected_shapes") { + REQUIRE(photo_table.nw == nw_ref); + REQUIRE(photo_table.numj == numj_ref); + REQUIRE(photo_table.nw == nw_shape); + REQUIRE(photo_table.nump == nump_shape); + REQUIRE(photo_table.numsza == numsza_shape); + REQUIRE(photo_table.numcolo3 == numcolo3_shape); + REQUIRE(photo_table.numalb == numalb_shape); + + REQUIRE(photo_table.sza.extent_int(0) == photo_table.numsza); + REQUIRE(photo_table.alb.extent_int(0) == photo_table.numalb); + REQUIRE(photo_table.colo3.extent_int(0) == photo_table.nump); + REQUIRE(photo_table.o3rat.extent_int(0) == photo_table.numcolo3); + REQUIRE(photo_table.prs.extent_int(0) == photo_table.np_xs); + REQUIRE(photo_table.lng_indexer.extent_int(0) == mam4::mo_photo::phtcnt); + REQUIRE(photo_table.pht_alias_mult_1.extent_int(0) == mam4::mo_photo::phtcnt); + } + + SECTION("1d_tables_match_yaml_reference") { + for (int i = 0; i < photo_table.numsza; ++i) { + REQUIRE(nearly_equal(sza_h(i), sza_ref[i])); + } + for (int i = 0; i < photo_table.numalb; ++i) { + REQUIRE(nearly_equal(alb_h(i), alb_ref[i])); + } + for (int i = 0; i < photo_table.nump; ++i) { + REQUIRE(nearly_equal(colo3_h(i), colo3_ref[i])); + } + for (int i = 0; i < photo_table.numcolo3; ++i) { + REQUIRE(nearly_equal(o3rat_h(i), o3rat_ref[i])); + } + for (int i = 0; i < photo_table.nump; ++i) { + REQUIRE(nearly_equal(press_h(i), press_ref[i])); + } + for (int i = 0; i < photo_table.np_xs; ++i) { + REQUIRE(nearly_equal(prs_h(i), prs_ref[i])); + } + for (int i = 0; i < photo_table.numsza - 1; ++i) { + REQUIRE(nearly_equal(del_sza_h(i), del_sza_ref[i])); + } + for (int i = 0; i < photo_table.numalb - 1; ++i) { + REQUIRE(nearly_equal(del_alb_h(i), del_alb_ref[i])); + } + for (int i = 0; i < photo_table.numcolo3 - 1; ++i) { + REQUIRE(nearly_equal(del_o3rat_h(i), del_o3rat_ref[i])); + } + for (int i = 0; i < photo_table.np_xs - 1; ++i) { + REQUIRE(nearly_equal(dprs_h(i), dprs_ref[i])); + } + } + + SECTION("rsf_corner_slice_matches_reference") { + const int nw = photo_table.nw; + const int nump = photo_table.nump; + int count = 0; + for (int d2 = 0; d2 < nump; ++d2) { + for (int d1 = 0; d1 < nw; ++d1) { + const auto computed = rsf_tab_h(d1, d2, 0, 0, 0);; + const auto expected = rsf_tab_2d[count]; + count++; + INFO("rsf_tab mismatch at (i=" << d1 << ", j=" << d2 + << "), computed=" << computed << ", expected=" << expected); + REQUIRE(nearly_equal(computed, expected,1e-6)); + } + } + } + + SECTION("xsqy_table_matches_yaml_reference") { + REQUIRE(xsqy_2d.size() == + static_cast(photo_table.numj) * + static_cast(photo_table.nw)); + int count = 0; + for (int w = 0; w < photo_table.nw; ++w) { + REQUIRE(nearly_equal(xsqy_h(0, w, 0, 0), xsqy_2d[count], 0, 0)); + ++count; + } + } + + SECTION("indexing_and_alias_arrays_are_initialized") { + REQUIRE(lng_indexer_h(0) == 0); + for (int i = 0; i < mam4::mo_photo::phtcnt; ++i) { + REQUIRE(nearly_equal(pht_alias_mult_h(i), 1.0)); + } + } + + SECTION("etfphot_is_finite") { + REQUIRE(static_cast(etfphot_ref.size()) == photo_table.nw); + for (int i = 0; i < photo_table.nw; ++i) { + REQUIRE(nearly_equal(etfphot_h(i), etfphot_ref[i])); + REQUIRE(std::isfinite(etfphot_h(i))); + } + } + +} + +TEST_CASE("mam_photo_table_kernel_single_column_nlev72_regression", + "[mam4][photo][kokkos]") { + if constexpr (mam4::nlev != 72) return; + constexpr int ncol = 1; + constexpr int nlev = mam4::nlev; + constexpr int nref = 1; + using namespace scream; + + ekat::Comm comm(MPI_COMM_WORLD); + struct ScorpioGuard { + explicit ScorpioGuard(const ekat::Comm& comm) : comm_(comm) { + scorpio::init_subsystem(comm_); + } + ~ScorpioGuard() { scorpio::finalize_subsystem(); } + const ekat::Comm& comm_; + } scorpio_guard(comm); + + const std::string rsf_file = + std::string(SCREAM_DATA_DIR) + "/mam4xx/photolysis/RSF_GT200nm_v3.0_c080811.nc"; + const std::string xs_long_file = + std::string(SCREAM_DATA_DIR) + "/mam4xx/photolysis/temp_prs_GT200nm_JPL10_c130206.nc"; + const std::string input_yaml_file = std::string(SCREAM_DATA_DIR) + "/mam4xx/photolysis/table_photo_input_ts_355.yaml"; + + const auto root = ekat::parse_yaml_file(input_yaml_file); + REQUIRE(root.isSublist("input")); + REQUIRE(root.sublist("input").isSublist("fixed")); + const auto& fixed = root.sublist("input").sublist("fixed"); + + const auto photo_table = scream::impl::read_photo_table(rsf_file, xs_long_file); + const int work_len = mam4::mo_photo::get_photo_table_work_len(photo_table); + const int npht = mam4::mo_photo::phtcnt; + + REQUIRE(work_len > 0); + REQUIRE(npht >= nref); + + // Allocate device views. + view_2d work_photo_table("work_photo_table", ncol, work_len); + view_2d pmid("pmid", ncol, nlev); + view_2d pdel("pdel", ncol, nlev); + view_2d temper("temper", ncol, nlev); + view_2d o3col("o3col", ncol, nlev); + view_1d zen_angle("zen_angle", ncol); + view_1d srf_alb("srf_alb", ncol); + view_2d qc("qc", ncol, nlev); + view_2d cld("cld", ncol, nlev); + view_3d photo("photo", ncol, nlev, npht); + + Kokkos::deep_copy(work_photo_table, 0.0); + Kokkos::deep_copy(photo, 0.0); + + // Read atmospheric-state reference data from ParameterList. + const auto pmid_vals = get_real_vec(fixed, "pmid"); + const auto pdel_vals = get_real_vec(fixed, "pdel"); + const auto temper_vals = get_real_vec(fixed, "temper"); + const auto o3col_vals = get_real_vec(fixed, "col_dens_1"); + const auto lwc_vals = get_real_vec(fixed, "lwc"); + const auto cloud_vals = get_real_vec(fixed, "clouds"); + const auto zen_vals = get_real_vec(fixed, "zen_angle"); + const auto alb_vals = get_real_vec(fixed, "srf_alb"); + const auto esfact_vals = get_real_vec(fixed, "esfact"); + const auto photo_ref = get_real_vec(fixed, "photos"); + + REQUIRE(pmid_vals.size() >= static_cast(nlev)); + REQUIRE(pdel_vals.size() >= static_cast(nlev)); + REQUIRE(temper_vals.size() >= static_cast(nlev)); + REQUIRE(o3col_vals.size() >= static_cast(nlev)); + REQUIRE(lwc_vals.size() >= static_cast(nlev)); + REQUIRE(cloud_vals.size() >= static_cast(nlev)); + REQUIRE(zen_vals.size() >= 1); + REQUIRE(alb_vals.size() >= 1); + REQUIRE(esfact_vals.size() >= 1); + + const Real zen_val = zen_vals[0]; + const Real alb_val = alb_vals[0]; + const Real esfact = esfact_vals[0]; + + // Fill host mirrors and copy to device. + auto pmid_h = Kokkos::create_mirror_view(pmid); + auto pdel_h = Kokkos::create_mirror_view(pdel); + auto temper_h = Kokkos::create_mirror_view(temper); + auto o3col_h = Kokkos::create_mirror_view(o3col); + auto zen_h = Kokkos::create_mirror_view(zen_angle); + auto alb_h = Kokkos::create_mirror_view(srf_alb); + auto qc_h = Kokkos::create_mirror_view(qc); + auto cld_h = Kokkos::create_mirror_view(cld); + + for (int k = 0; k < nlev; ++k) { + pmid_h(0, k) = pmid_vals[k]; + pdel_h(0, k) = pdel_vals[k]; + temper_h(0, k) = temper_vals[k]; + o3col_h(0, k) = o3col_vals[k]; + qc_h(0, k) = lwc_vals[k]; + cld_h(0, k) = cloud_vals[k]; + } + zen_h(0) = zen_val; + alb_h(0) = alb_val; + + Kokkos::deep_copy(pmid, pmid_h); + Kokkos::deep_copy(pdel, pdel_h); + Kokkos::deep_copy(temper, temper_h); + Kokkos::deep_copy(o3col, o3col_h); + Kokkos::deep_copy(zen_angle, zen_h); + Kokkos::deep_copy(srf_alb, alb_h); + Kokkos::deep_copy(qc, qc_h); + Kokkos::deep_copy(cld, cld_h); + + // Launch one-column photolysis kernel. + TeamPolicy policy(ncol, Kokkos::AUTO()); + Kokkos::parallel_for( + "unit_test_table_photo_nlev72", policy, + KOKKOS_LAMBDA(const MemberType& team) { + const int icol = team.league_rank(); + + const auto work_icol = ekat::subview(work_photo_table, icol); + mam4::mo_photo::PhotoTableWorkArrays photo_work_arrays; + mam4::mo_photo::set_photo_table_work_arrays(photo_table, work_icol, + photo_work_arrays); + team.team_barrier(); + + mam4::mo_photo::table_photo( + team, + ekat::subview(photo, icol), + ekat::subview(pmid, icol), + ekat::subview(pdel, icol), + ekat::subview(temper, icol), + ekat::subview(o3col, icol), + zen_angle(icol), srf_alb(icol), + ekat::subview(qc, icol), + ekat::subview(cld, icol), + esfact, photo_table, photo_work_arrays); + }); + Kokkos::fence(); + + auto photo_h = Kokkos::create_mirror_view_and_copy(HostSpace(), photo); + + SECTION("all_outputs_are_finite") { + for (int k = 0; k < nlev; ++k) { + for (int j = 0; j < nref; ++j) { + INFO("Non-finite output at k=" << k << ", j=" << j + << ", value=" << photo_h(0, k, j)); + REQUIRE(std::isfinite(photo_h(0, k, j))); + } + } + } + + SECTION("compare_against_reference_when_available") { + REQUIRE(photo_ref.size() == static_cast(nlev * nref)); + constexpr Real relative_tol = PrecisionTolerance::relative_tol; + constexpr Real absolute_tol = PrecisionTolerance::absolute_tol; + + int count = 0; + for (int d2 = 0; d2 < nref; ++d2) { + for (int d1 = 0; d1 < nlev; ++d1) { + const auto computed = photo_h(0, d1, d2); + const auto expected = photo_ref[count]; + count++; + INFO("Reference mismatch at d1=" << d1 << ", d2=" << d2 + << ", computed=" << computed + << ", expected=" << expected); + REQUIRE(nearly_equal(computed, expected, relative_tol, absolute_tol)); + } + } + } +} diff --git a/components/eamxx/src/physics/register_physics.hpp b/components/eamxx/src/physics/register_physics.hpp index 75b88bd73022..2cd884232227 100644 --- a/components/eamxx/src/physics/register_physics.hpp +++ b/components/eamxx/src/physics/register_physics.hpp @@ -53,6 +53,12 @@ #ifdef EAMXX_HAS_CLD_FRAC_NET #include "physics/cld_fraction/cld_frac_net/eamxx_cld_frac_net_process_interface.hpp" #endif +#ifdef EAMXX_HAS_WATER_TRACERS +#include "physics/specialized_tracers/water_tracers/eamxx_water_tracers_process_interface.hpp" +#endif +#ifdef EAMXX_HAS_WATER_ISOTOPES +#include "physics/specialized_tracers/water_isotopes/eamxx_water_isotopes_process_interface.hpp" +#endif namespace scream { @@ -106,6 +112,12 @@ inline void register_physics () { #ifdef EAMXX_HAS_CLD_FRAC_NET proc_factory.register_product("cld_frac_net",&create_atmosphere_process); #endif +#ifdef EAMXX_HAS_WATER_TRACERS + proc_factory.register_product("water_tracers",&create_atmosphere_process); +#endif +#ifdef EAMXX_HAS_WATER_ISOTOPES + proc_factory.register_product("water_isotopes",&create_atmosphere_process); +#endif // If no physics was enabled, silence compile warning about unused var (void) proc_factory; diff --git a/components/eamxx/src/physics/shoc/CMakeLists.txt b/components/eamxx/src/physics/shoc/CMakeLists.txt index 68bcbf595a86..d7ed3926e83a 100644 --- a/components/eamxx/src/physics/shoc/CMakeLists.txt +++ b/components/eamxx/src/physics/shoc/CMakeLists.txt @@ -24,6 +24,7 @@ if (NOT EAMXX_ENABLE_GPU) eti/shoc_compute_shoc_mix_shoc_length.cpp eti/shoc_compute_shoc_vapor.cpp eti/shoc_compute_shoc_temperature.cpp + eti/shoc_compute_shear_strain3d.cpp eti/shoc_compute_shr_prod.cpp eti/shoc_compute_tmpi.cpp eti/shoc_diag_obklen.cpp diff --git a/components/eamxx/src/physics/shoc/disp/shoc_tke_disp.cpp b/components/eamxx/src/physics/shoc/disp/shoc_tke_disp.cpp index f8d730068e50..fb56def8e94b 100644 --- a/components/eamxx/src/physics/shoc/disp/shoc_tke_disp.cpp +++ b/components/eamxx/src/physics/shoc/disp/shoc_tke_disp.cpp @@ -20,7 +20,10 @@ ::shoc_tke_disp( const Scalar& Ckh, const Scalar& Ckm, const bool& shoc_1p5tke, + const bool& do_3d_turb, const view_2d& wthv_sec, + const view_3d& shear_strain3d_components, + const view_2d& shear_strain3d, const view_2d& shoc_mix, const view_2d& dz_zi, const view_2d& dz_zt, @@ -28,6 +31,7 @@ ::shoc_tke_disp( const view_2d& tabs, const view_2d& u_wind, const view_2d& v_wind, + const view_2d& w_field, const view_2d& brunt, const view_2d& zt_grid, const view_2d& zi_grid, @@ -47,11 +51,17 @@ ::shoc_tke_disp( const Int i = team.league_rank(); auto workspace = workspace_mgr.get_workspace(team); + uview_2d shear_strain3d_components_s; + if (do_3d_turb) { + shear_strain3d_components_s = ekat::subview(shear_strain3d_components, i); + } shoc_tke(team, nlev, nlevi, dtime, lambda_low, lambda_high, lambda_slope, lambda_thresh, - Ckh, Ckm, shoc_1p5tke, + Ckh, Ckm, shoc_1p5tke, do_3d_turb, ekat::subview(wthv_sec, i), + shear_strain3d_components_s, + ekat::subview(shear_strain3d, i), ekat::subview(shoc_mix, i), ekat::subview(dz_zi, i), ekat::subview(dz_zt, i), @@ -59,6 +69,7 @@ ::shoc_tke_disp( ekat::subview(tabs, i), ekat::subview(u_wind, i), ekat::subview(v_wind, i), + ekat::subview(w_field, i), ekat::subview(brunt, i), ekat::subview(zt_grid, i), ekat::subview(zi_grid, i), diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index dd57640bb728..52188b3748bb 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -54,6 +54,8 @@ void SHOCMacrophysics::create_requests() const auto m2 = pow(m,2); const auto s2 = pow(s,2); + const auto nondim = none; + const bool do_3d_turb = m_params.get("do_3d_turbulence_shoc", false); // These variables are needed by the interface, but not actually passed to shoc_main. add_field("omega", scalar3d_mid, Pa/s, grid_name, ps); @@ -74,6 +76,11 @@ void SHOCMacrophysics::create_requests() add_field("p_int", scalar3d_int, Pa, grid_name, ps); add_field("pseudo_density", scalar3d_mid, Pa, grid_name, ps); add_field("phis", scalar2d , m2/s2, grid_name); + if (do_3d_turb) { + const auto vector3d_mid_6 = m_grid->get_3d_vector_layout(LEV,6); + add_field("tke_shear_strain3d_components", vector3d_mid_6,nondim/s, grid_name, ps); + add_field("tke_shear_strain3d", scalar3d_mid,nondim/s2, grid_name, ps); + } // Input/Output variables add_field("horiz_winds", vector3d_mid, m/s, grid_name, ps); @@ -169,7 +176,7 @@ size_t SHOCMacrophysics::requested_buffer_size_in_bytes() const const auto policy = TPF::get_default_team_policy(m_num_cols, nlev_packs); const int n_wind_slots = ekat::npack(2)*Pack::n; const int n_trac_slots = ekat::npack(m_num_tracers+3)*Pack::n; - const size_t wsm_request= WSM::get_total_bytes_needed(nlevi_packs, 14+(2*n_wind_slots+n_trac_slots), policy); + const size_t wsm_request= WSM::get_total_bytes_needed(nlevi_packs, 20+(2*n_wind_slots+n_trac_slots), policy); return interface_request + wsm_request; } @@ -205,6 +212,8 @@ void SHOCMacrophysics::init_buffers(const ATMBufferManager &buffer_manager) const int nlev_packs = ekat::npack(m_num_levs); const int nlevi_packs = ekat::npack(m_num_levs+1); const int num_tracer_packs = ekat::npack(m_num_tracers); + m_dummy_shear_strain3d = view_2d("dummy_shear_strain3d", m_num_cols, nlev_packs); + Kokkos::deep_copy(m_dummy_shear_strain3d, 0); m_buffer.pref_mid = decltype(m_buffer.pref_mid)(s_mem, nlev_packs); s_mem += m_buffer.pref_mid.size(); @@ -249,7 +258,7 @@ void SHOCMacrophysics::init_buffers(const ATMBufferManager &buffer_manager) const auto policy = TPF::get_default_team_policy(m_num_cols, nlev_packs); const int n_wind_slots = ekat::npack(2)*Pack::n; const int n_trac_slots = ekat::npack(m_num_tracers+3)*Pack::n; - const int wsm_size = WSM::get_total_bytes_needed(nlevi_packs, 14+(2*n_wind_slots+n_trac_slots), policy)/sizeof(Pack); + const int wsm_size = WSM::get_total_bytes_needed(nlevi_packs, 20+(2*n_wind_slots+n_trac_slots), policy)/sizeof(Pack); s_mem += wsm_size; size_t used_mem = (reinterpret_cast(s_mem) - buffer_manager.get_memory())*sizeof(Real); @@ -275,6 +284,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) runtime_options.Ckh = m_params.get("coeff_kh"); runtime_options.Ckm = m_params.get("coeff_km"); runtime_options.shoc_1p5tke = m_params.get("shoc_1p5tke"); + runtime_options.do_3d_turb = m_params.get("do_3d_turbulence_shoc", false); runtime_options.extra_diags = m_params.get("extra_shoc_diags"); // Initialize all of the structures that are passed to shoc_main in run_impl. // Note: Some variables in the structures are not stored in the field manager. For these @@ -287,6 +297,14 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) const auto& surf_sens_flux = get_field_in("surf_sens_flux").get_view(); const auto& surf_evap = get_field_in("surf_evap").get_view(); const auto& surf_mom_flux = get_field_in("surf_mom_flux").get_view(); + const auto shear_strain3d = + runtime_options.do_3d_turb + ? get_field_out("tke_shear_strain3d").get_view() + : view_2d(m_dummy_shear_strain3d); + view_3d_const shear_strain3d_components; + if (runtime_options.do_3d_turb) { + shear_strain3d_components = get_field_in("tke_shear_strain3d_components").get_view(); + } const auto& qtracers = get_group_out("turbulence_advected_tracers").m_monolithic_field->get_strided_view(); const auto& qc = get_field_out("qc").get_view(); const auto& qv = get_field_out("qv").get_view(); @@ -335,6 +353,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) if (run_type==RunType::Initial){ Kokkos::deep_copy(sgs_buoy_flux,0.0); Kokkos::deep_copy(tk,0.0); + Kokkos::deep_copy(shear_strain3d,0.0); Kokkos::deep_copy(tke,0.0004); Kokkos::deep_copy(tke_copy,0.0004); Kokkos::deep_copy(cldfrac_liq,0.0); @@ -346,7 +365,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) shoc_preprocess.set_variables(m_num_cols,m_num_levs,z_surf, T_mid,p_mid,p_int,pseudo_density,omega,phis,surf_sens_flux,surf_evap, - surf_mom_flux,qtracers,qv,qc,qc_copy,tke,tke_copy,z_mid,z_int, + surf_mom_flux,qtracers,qv,shear_strain3d_components,shear_strain3d,qc,qc_copy,tke,tke_copy,z_mid,z_int, dse,rrho,rrho_i,thv,dz,zt_grid,zi_grid,wpthlp_sfc,wprtp_sfc,upwp_sfc,vpwp_sfc, wtracer_sfc,wm_zt,inv_exner,thlm,qw, cldfrac_liq, cldfrac_liq_prev, upwp_sfc_pert, vpwp_sfc_pert, um_pert, vm_pert, @@ -369,6 +388,8 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) input.wtracer_sfc = shoc_preprocess.wtracer_sfc; input.inv_exner = shoc_preprocess.inv_exner; input.phis = phis; + input.shear_strain3d_components = shear_strain3d_components; + input.shear_strain3d = shear_strain3d; // Input/Output Variables input_output.host_dse = shoc_preprocess.shoc_s; @@ -471,7 +492,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) const int n_wind_slots = ekat::npack(2)*Pack::n; const int n_trac_slots = ekat::npack(m_num_tracers+3)*Pack::n; const auto default_policy = TPF::get_default_team_policy(m_num_cols, nlev_packs); - workspace_mgr.setup(m_buffer.wsm_data, nlevi_packs, 14+(2*n_wind_slots+n_trac_slots), default_policy); + workspace_mgr.setup(m_buffer.wsm_data, nlevi_packs, 20+(2*n_wind_slots+n_trac_slots), default_policy); // Calculate pref_mid, and use that to calculate // maximum number of levels in pbl from surface diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp index a6f86eb52bc7..b97df6023dad 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp @@ -194,6 +194,8 @@ class SHOCMacrophysics : public scream::AtmosphereProcess sview_2d_const surf_mom_flux; view_3d_strided qtracers; view_2d qv; + view_3d_const shear_strain3d_components; + view_2d shear_strain3d; view_2d_const qc; view_2d qc_copy; view_2d z_mid; @@ -232,7 +234,8 @@ class SHOCMacrophysics : public scream::AtmosphereProcess const view_1d_const& phis_, const view_1d_const& surf_sens_flux_, const view_1d_const& surf_evap_, const sview_2d_const& surf_mom_flux_, const view_3d_strided& qtracers_, - const view_2d& qv_, const view_2d_const& qc_, const view_2d& qc_copy_, + const view_2d& qv_, const view_3d_const& shear_strain3d_components_, + const view_2d& shear_strain3d_, const view_2d_const& qc_, const view_2d& qc_copy_, const view_2d& tke_, const view_2d& tke_copy_, const view_2d& z_mid_, const view_2d& z_int_, const view_2d& dse_, const view_2d& rrho_, const view_2d& rrho_i_, @@ -258,6 +261,8 @@ class SHOCMacrophysics : public scream::AtmosphereProcess surf_evap = surf_evap_; surf_mom_flux = surf_mom_flux_; qv = qv_; + shear_strain3d_components = shear_strain3d_components_; + shear_strain3d = shear_strain3d_; // OUT qtracers = qtracers_; qc = qc_; @@ -555,6 +560,7 @@ class SHOCMacrophysics : public scream::AtmosphereProcess // Struct which contains local variables Buffer m_buffer; + view_2d m_dummy_shear_strain3d; // Store the structures for each argument to shoc_main; SHF::SHOCInput input; diff --git a/components/eamxx/src/physics/shoc/eti/shoc_compute_shear_strain3d.cpp b/components/eamxx/src/physics/shoc/eti/shoc_compute_shear_strain3d.cpp new file mode 100644 index 000000000000..2d40eadafed1 --- /dev/null +++ b/components/eamxx/src/physics/shoc/eti/shoc_compute_shear_strain3d.cpp @@ -0,0 +1,14 @@ +#include "shoc_compute_shear_strain3d_impl.hpp" + +namespace scream { +namespace shoc { + +/* + * Explicit instantiation for doing compute_shear_strain3d on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace shoc +} // namespace scream diff --git a/components/eamxx/src/physics/shoc/impl/shoc_adv_sgs_tke_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_adv_sgs_tke_impl.hpp index d98933000736..9ecc0220f0bc 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_adv_sgs_tke_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_adv_sgs_tke_impl.hpp @@ -19,11 +19,13 @@ ::adv_sgs_tke( const Int& nlev, const Real& dtime, const bool& shoc_1p5tke, + const bool& do_3d_turb, const uview_1d& shoc_mix, const uview_1d& wthv_sec, const uview_1d& sterm_zt, const uview_1d& tk, const uview_1d& brunt, + const uview_1d& shear_strain3d, const uview_1d& tke, const uview_1d& a_diss) { @@ -33,7 +35,6 @@ ::adv_sgs_tke( static constexpr Scalar basetemp = C::basetemp; static constexpr Scalar mintke = scream::shoc::Constants::mintke; static constexpr Scalar maxtke = scream::shoc::Constants::maxtke; - Pack a_prod_bu; //declare some constants static constexpr Scalar Cs = 0.15; @@ -45,6 +46,8 @@ ::adv_sgs_tke( const Int nlev_pack = ekat::npack(nlev); Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_pack), [&] (const Int& k) { + Pack a_prod_bu; + Pack a_prod_sh; // Compute buoyant production term if (shoc_1p5tke){ @@ -59,7 +62,12 @@ ::adv_sgs_tke( tke(k) = ekat::max(0,tke(k)); // Shear production term, use diffusivity from previous timestep - const Pack a_prod_sh = tk(k)*sterm_zt(k); + if (do_3d_turb){ + a_prod_sh = Ck*tk(k)*shear_strain3d(k); + } + else{ + a_prod_sh = tk(k)*sterm_zt(k); + } // Dissipation term a_diss(k)=Cee/shoc_mix(k)*ekat::pow(tke(k),sp(1.5)); diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assemble_shoc_shear_strain3d_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assemble_shoc_shear_strain3d_impl.hpp new file mode 100644 index 000000000000..4cf0d249650a --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assemble_shoc_shear_strain3d_impl.hpp @@ -0,0 +1,56 @@ +#ifndef SHOC_ASSEMBLE_SHOC_SHEAR_STRAIN3D_IMPL_HPP +#define SHOC_ASSEMBLE_SHOC_SHEAR_STRAIN3D_IMPL_HPP + +#include "shoc_functions.hpp" + +namespace scream { +namespace shoc { + +template +KOKKOS_FUNCTION +void Functions::assemble_shoc_shear_strain3d( + const MemberType& team, + const Int& nlev, + const uview_2d& shear_strain3d_components, + const uview_1d& du_dz_m, + const uview_1d& dv_dz_m, + const uview_1d& dw_dz_m, + const uview_1d& shear_strain3d) +{ + const Int nlev_pack = ekat::npack(nlev); + + // Assemble the full local velocity-gradient tensor from dycore horizontal + // components and SHOC-computed vertical components, then form the symmetric strain invariant. + team.team_barrier(); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_pack), [&] (const Int& k) { + constexpr Scalar one_half = Scalar(0.5); + constexpr Scalar two = Scalar(2.0); + + const Pack A00 = shear_strain3d_components(0,k); + const Pack A01 = shear_strain3d_components(1,k); + const Pack A10 = shear_strain3d_components(2,k); + const Pack A11 = shear_strain3d_components(3,k); + const Pack A20 = shear_strain3d_components(4,k); + const Pack A21 = shear_strain3d_components(5,k); + + const Pack A02 = du_dz_m(k); + const Pack A12 = dv_dz_m(k); + const Pack A22 = dw_dz_m(k); + + const Pack S00 = A00; + const Pack S11 = A11; + const Pack S22 = A22; + const Pack S01 = one_half * (A01 + A10); + const Pack S02 = one_half * (A02 + A20); + const Pack S12 = one_half * (A12 + A21); + + shear_strain3d(k) = + two * (S00*S00 + S11*S11 + S22*S22 + + two*S01*S01 + two*S02*S02 + two*S12*S12); + }); +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_compute_shear_strain3d_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_compute_shear_strain3d_impl.hpp new file mode 100644 index 000000000000..b950e3fff1c8 --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_compute_shear_strain3d_impl.hpp @@ -0,0 +1,63 @@ +#ifndef SHOC_COMPUTE_SHEAR_STRAIN3D_IMPL_HPP +#define SHOC_COMPUTE_SHEAR_STRAIN3D_IMPL_HPP + +#include "shoc_assemble_shoc_shear_strain3d_impl.hpp" +#include "shoc_compute_vertical_shear_terms_impl.hpp" + +namespace scream { +namespace shoc { + +#ifdef SCREAM_SHOC_SMALL_KERNELS +template +void Functions::compute_shear_strain3d_disp( + const Int& shcol, + const Int& nlev, + const Int& nlevi, + const view_3d& shear_strain3d_components, + const view_2d& dz_zi, + const view_2d& u_wind, + const view_2d& v_wind, + const view_2d& w_field, + const view_2d& zt_grid, + const view_2d& zi_grid, + const WorkspaceMgr& workspace_mgr, + const view_2d& shear_strain3d) +{ + using ExeSpace = typename KT::ExeSpace; + using TPF = ekat::TeamPolicyFactory; + + const auto nlev_packs = ekat::npack(nlev); + const auto policy = TPF::get_default_team_policy(shcol, nlev_packs); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { + const Int i = team.league_rank(); + auto workspace = workspace_mgr.get_workspace(team); + uview_1d du_dz_m, dv_dz_m, dw_dz_m; + workspace.template take_many_contiguous_unsafe<3>( + {"du_dz_m", "dv_dz_m", "dw_dz_m"}, + {&du_dz_m, &dv_dz_m, &dw_dz_m}); + + compute_vertical_shear_terms(team, nlev, nlevi, + Kokkos::subview(dz_zi, i, Kokkos::ALL()), + Kokkos::subview(u_wind, i, Kokkos::ALL()), + Kokkos::subview(v_wind, i, Kokkos::ALL()), + Kokkos::subview(w_field, i, Kokkos::ALL()), + Kokkos::subview(zt_grid, i, Kokkos::ALL()), + Kokkos::subview(zi_grid, i, Kokkos::ALL()), + workspace, + du_dz_m, dv_dz_m, dw_dz_m); + + assemble_shoc_shear_strain3d(team, nlev, + Kokkos::subview(shear_strain3d_components, i, Kokkos::ALL(), Kokkos::ALL()), + du_dz_m, dv_dz_m, dw_dz_m, + Kokkos::subview(shear_strain3d, i, Kokkos::ALL())); + + workspace.template release_many_contiguous<3>( + {&du_dz_m, &dv_dz_m, &dw_dz_m}); + }); +} +#endif + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_compute_vertical_shear_terms_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_compute_vertical_shear_terms_impl.hpp new file mode 100644 index 000000000000..8b16651b82e1 --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_compute_vertical_shear_terms_impl.hpp @@ -0,0 +1,101 @@ +#ifndef SHOC_COMPUTE_VERTICAL_SHEAR_TERMS_IMPL_HPP +#define SHOC_COMPUTE_VERTICAL_SHEAR_TERMS_IMPL_HPP + +#include "shoc_functions.hpp" + +namespace scream { +namespace shoc { + +template +KOKKOS_FUNCTION +void Functions::compute_vertical_shear_terms( + const MemberType& team, + const Int& nlev, + const Int& nlevi, + const uview_1d& dz_zi, + const uview_1d& u_wind, + const uview_1d& v_wind, + const uview_1d& w_field, + const uview_1d& zt_grid, + const uview_1d& zi_grid, + const Workspace& workspace, + const uview_1d& du_dz_m, + const uview_1d& dv_dz_m, + const uview_1d& dw_dz_m) +{ + // Compute the SHOC-column vertical gradients on interfaces, then + // interpolate them back to midpoint levels. + uview_1d du_dz_i, dv_dz_i, dw_dz_i; + workspace.template take_many_contiguous_unsafe<3>( + {"du_dz_i", "dv_dz_i", "dw_dz_i"}, + {&du_dz_i, &dv_dz_i, &dw_dz_i}); + + const Int nlev_pack = ekat::npack(nlev); + const Int nlevi_pack = ekat::npack(nlevi); + + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlevi_pack), [&] (const Int& k) { + du_dz_i(k) = 0; + dv_dz_i(k) = 0; + dw_dz_i(k) = 0; + }); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_pack), [&] (const Int& k) { + du_dz_m(k) = 0; + dv_dz_m(k) = 0; + dw_dz_m(k) = 0; + }); + team.team_barrier(); + + const auto s_u_wind = scalarize(u_wind); + const auto s_v_wind = scalarize(v_wind); + const auto s_w_field = scalarize(w_field); + + // Form the vertical gradients on the interface grid first so they are + // consistent with SHOC's native staggered-grid treatment of shear production. + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_pack), [&] (const Int& k) { + auto range_pack = ekat::range(k*Pack::n); + const auto active_range = range_pack > 0 && range_pack < nlev; + + if (active_range.any()) { + const Pack inv_dz = 1 / dz_zi(k); + + auto range_pack_safe = range_pack; + range_pack_safe.set(range_pack < 1, 1); + + Pack u_grid, u_up_grid; + Pack v_grid, v_up_grid; + Pack w_grid, w_up_grid; + ekat::index_and_shift<-1>(s_u_wind, range_pack_safe, u_grid, u_up_grid); + ekat::index_and_shift<-1>(s_v_wind, range_pack_safe, v_grid, v_up_grid); + ekat::index_and_shift<-1>(s_w_field, range_pack_safe, w_grid, w_up_grid); + + du_dz_i(k).set(active_range, inv_dz*(u_up_grid - u_grid)); + dv_dz_i(k).set(active_range, inv_dz*(v_up_grid - v_grid)); + dw_dz_i(k).set(active_range, inv_dz*(w_up_grid - w_grid)); + } + }); + + auto s_du_dz_i = scalarize(du_dz_i); + auto s_dv_dz_i = scalarize(dv_dz_i); + auto s_dw_dz_i = scalarize(dw_dz_i); + s_du_dz_i(0) = 0; + s_dv_dz_i(0) = 0; + s_dw_dz_i(0) = 0; + s_du_dz_i(nlevi-1) = 0; + s_dv_dz_i(nlevi-1) = 0; + s_dw_dz_i(nlevi-1) = 0; + + // Interpolate the interface-grid vertical gradients back to midpoint levels, + // where SHOC carries thermodynamic variables and TKE. + team.team_barrier(); + linear_interp(team, zi_grid, zt_grid, du_dz_i, du_dz_m, nlevi, nlev, 0); + linear_interp(team, zi_grid, zt_grid, dv_dz_i, dv_dz_m, nlevi, nlev, 0); + linear_interp(team, zi_grid, zt_grid, dw_dz_i, dw_dz_m, nlevi, nlev, 0); + + workspace.template release_many_contiguous<3>( + {&du_dz_i, &dv_dz_i, &dw_dz_i}); +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp index 54ea8e231af9..cda67cdb3878 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp @@ -89,6 +89,7 @@ void Functions::shoc_main_internal( const Scalar& Ckh, const Scalar& Ckm, const bool& shoc_1p5tke, + const bool& do_3d_turb, const bool& extra_diags, // Input Variables const Scalar& dx, @@ -107,6 +108,8 @@ void Functions::shoc_main_internal( const uview_1d& wtracer_sfc, const uview_1d& inv_exner, const Scalar& phis, + const uview_2d& shear_strain3d_components, + const uview_1d& shear_strain3d, // Workspace/Local Variables const Workspace& workspace, // Input/Output Variables @@ -228,9 +231,11 @@ void Functions::shoc_main_internal( shoc_tke(team,nlev,nlevi,dtime, // Input lambda_low,lambda_high,lambda_slope, // Runtime options lambda_thresh,Ckh,Ckm,shoc_1p5tke, // Runtime options - wthv_sec, // Input + do_3d_turb, // Runtime options + wthv_sec,shear_strain3d_components, // Input + shear_strain3d, // Input/Output shoc_mix,dz_zi,dz_zt,pres,shoc_tabs, // Input - u_wind,v_wind,brunt,zt_grid, // Input + u_wind,v_wind,w_field,brunt,zt_grid, // Input zi_grid,pblh, // Input workspace, // Workspace tke,tk,tkh, // Input/Output @@ -356,6 +361,7 @@ void Functions::shoc_main_internal( const Scalar& Ckh, const Scalar& Ckm, const bool& shoc_1p5tke, + const bool& do_3d_turb, const bool& extra_diags, // Input Variables const view_1d& dx, @@ -374,6 +380,8 @@ void Functions::shoc_main_internal( const view_2d& wtracer_sfc, const view_2d& inv_exner, const view_1d& phis, + const view_3d& shear_strain3d_components, + const view_2d& shear_strain3d, // Workspace Manager WorkspaceMgr& workspace_mgr, // Input/Output Variables @@ -496,13 +504,14 @@ void Functions::shoc_main_internal( workspace_mgr, // Workspace mgr brunt,shoc_mix); // Output - // Advance the SGS TKE equation shoc_tke_disp(shcol,nlev,nlevi,dtime, // Input - lambda_low,lambda_high,lambda_slope, // Runtime options - lambda_thresh,Ckh,Ckm,shoc_1p5tke, // Runtime options - wthv_sec, // Input + lambda_low,lambda_high,lambda_slope, // Runtime options + lambda_thresh,Ckh,Ckm,shoc_1p5tke, // Runtime options + do_3d_turb, // Runtime options + wthv_sec,shear_strain3d_components, // Input + shear_strain3d, // Input/Output shoc_mix,dz_zi,dz_zt,pres,shoc_tabs, // Input - u_wind,v_wind,brunt,zt_grid, // Input + u_wind,v_wind,w_field,brunt,zt_grid, // Input zi_grid,pblh, // Input workspace_mgr, // Workspace mgr tke,tk,tkh, // Input/Output @@ -636,6 +645,7 @@ Int Functions::shoc_main( const Scalar Ckm = shoc_runtime.Ckm; const bool shoc_1p5tke = shoc_runtime.shoc_1p5tke; const bool extra_diags = shoc_runtime.extra_diags; + const bool do_3d_turb = shoc_runtime.do_3d_turb; #ifndef SCREAM_SHOC_SMALL_KERNELS using ExeSpace = typename KT::ExeSpace; @@ -671,6 +681,12 @@ Int Functions::shoc_main( const auto w_field_s = ekat::subview(shoc_input.w_field, i); const auto wtracer_sfc_s = ekat::subview(shoc_input.wtracer_sfc, i); const auto inv_exner_s = ekat::subview(shoc_input.inv_exner, i); + uview_2d shear_strain3d_components_s; + if (do_3d_turb) { + shear_strain3d_components_s = + Kokkos::subview(shoc_input.shear_strain3d_components, i, Kokkos::ALL(), Kokkos::ALL()); + } + const auto shear_strain3d_s = ekat::subview(shoc_input.shear_strain3d, i); const auto host_dse_s = ekat::subview(shoc_input_output.host_dse, i); const auto tke_s = ekat::subview(shoc_input_output.tke, i); const auto thetal_s = ekat::subview(shoc_input_output.thetal, i); @@ -707,11 +723,13 @@ Int Functions::shoc_main( shoc_main_internal(team, nlev, nlevi, npbl, nadv, num_qtracers, dtime, lambda_low, lambda_high, lambda_slope, lambda_thresh, // Runtime options thl2tune, qw2tune, qwthl2tune, w2tune, length_fac, // Runtime options - c_diag_3rd_mom, Ckh, Ckm, shoc_1p5tke, extra_diags, // Runtime options + c_diag_3rd_mom, Ckh, Ckm, shoc_1p5tke, // Runtime options + do_3d_turb, extra_diags, // Runtime options dx_s, dy_s, zt_grid_s, zi_grid_s, // Input pres_s, presi_s, pdel_s, thv_s, w_field_s, // Input wthl_sfc_s, wqw_sfc_s, uw_sfc_s, vw_sfc_s, // Input wtracer_sfc_s, inv_exner_s, phis_s, // Input + shear_strain3d_components_s, shear_strain3d_s, // Input/Output workspace, // Workspace host_dse_s, tke_s, thetal_s, qw_s, u_wind_s, v_wind_s, // Input/Output wthv_sec_s, qtracers_s, tk_s, shoc_cldfrac_s, // Input/Output @@ -735,11 +753,12 @@ Int Functions::shoc_main( shoc_main_internal(shcol, nlev, nlevi, npbl, nadv, num_qtracers, dtime, lambda_low, lambda_high, lambda_slope, lambda_thresh, // Runtime options thl2tune, qw2tune, qwthl2tune, w2tune, length_fac, // Runtime options - c_diag_3rd_mom, Ckh, Ckm, shoc_1p5tke, extra_diags, // Runtime options + c_diag_3rd_mom, Ckh, Ckm, shoc_1p5tke, do_3d_turb, extra_diags, // Runtime options shoc_input.dx, shoc_input.dy, shoc_input.zt_grid, shoc_input.zi_grid, // Input shoc_input.pres, shoc_input.presi, shoc_input.pdel, shoc_input.thv, shoc_input.w_field, // Input shoc_input.wthl_sfc, shoc_input.wqw_sfc, shoc_input.uw_sfc, shoc_input.vw_sfc, // Input - shoc_input.wtracer_sfc, shoc_input.inv_exner, shoc_input.phis, // Input + shoc_input.wtracer_sfc, shoc_input.inv_exner, shoc_input.phis, + shoc_input.shear_strain3d_components, shoc_input.shear_strain3d, // Input/Output workspace_mgr, // Workspace Manager shoc_input_output.host_dse, shoc_input_output.tke, shoc_input_output.thetal, shoc_input_output.qw, u_wind_s, v_wind_s, // Input/Output shoc_input_output.wthv_sec, shoc_input_output.qtracers, shoc_input_output.tk, shoc_input_output.shoc_cldfrac, // Input/Output diff --git a/components/eamxx/src/physics/shoc/impl/shoc_tke_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_tke_impl.hpp index 9f5fa5f8b7ab..159265cef0d8 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_tke_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_tke_impl.hpp @@ -31,7 +31,10 @@ void Functions::shoc_tke( const Scalar& Ckh, const Scalar& Ckm, const bool& shoc_1p5tke, + const bool& do_3d_turb, const uview_1d& wthv_sec, + const uview_2d& shear_strain3d_components, + const uview_1d& shear_strain3d, const uview_1d& shoc_mix, const uview_1d& dz_zi, const uview_1d& dz_zt, @@ -39,6 +42,7 @@ void Functions::shoc_tke( const uview_1d& tabs, const uview_1d& u_wind, const uview_1d& v_wind, + const uview_1d& w_field, const uview_1d& brunt, const uview_1d& zt_grid, const uview_1d& zi_grid, @@ -50,25 +54,57 @@ void Functions::shoc_tke( const uview_1d& isotropy) { // Define temporary variables - uview_1d sterm_zt, a_diss, sterm; - workspace.template take_many_contiguous_unsafe<3>( - {"sterm_zt", "a_diss", "sterm"}, - {&sterm_zt, &a_diss, &sterm}); + uview_1d sterm_zt, a_diss, sterm, du_dz_m, dv_dz_m, dw_dz_m; + workspace.template take_many_contiguous_unsafe<6>( + {"sterm_zt", "a_diss", "sterm", "du_dz_m", "dv_dz_m", "dw_dz_m"}, + {&sterm_zt, &a_diss, &sterm, &du_dz_m, &dv_dz_m, &dw_dz_m}); // Compute integrated column stability in lower troposphere Scalar brunt_int(0); integ_column_stability(team,nlev,dz_zt,pres,brunt,brunt_int); - // Compute shear production term, which is on interface levels - // This follows the methods of Bretheron and Park (2010) - compute_shr_prod(team,nlevi,nlev,dz_zi,u_wind,v_wind,sterm); + // If not using 3d turbulence then use the default 1D calculation for shear production + if (!do_3d_turb){ + // Compute shear production term, which is on interface levels + // This follows the methods of Bretheron and Park (2010) + compute_shr_prod(team,nlevi,nlev,dz_zi,u_wind,v_wind,sterm); - // Interpolate shear term from interface to thermo grid - team.team_barrier(); - linear_interp(team,zi_grid,zt_grid,sterm,sterm_zt,nlevi,nlev,0); + // Interpolate shear term from interface to thermo grid + team.team_barrier(); + linear_interp(team,zi_grid,zt_grid,sterm,sterm_zt,nlevi,nlev,0); + + // In the legacy 1D path this diagnostic is not used to drive TKE, but the + // field is still part of SHOC's output contract and must not retain stale data. + const Int nlev_pack = ekat::npack(nlev); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_pack), [&] (const Int& k) { + shear_strain3d(k) = 0; + }); + } else { + compute_vertical_shear_terms(team,nlev,nlevi, + dz_zi,u_wind,v_wind,w_field, + zt_grid,zi_grid,workspace, + du_dz_m,dv_dz_m,dw_dz_m); + team.team_barrier(); + + assemble_shoc_shear_strain3d(team,nlev, + shear_strain3d_components, + du_dz_m,dv_dz_m,dw_dz_m, + shear_strain3d); + team.team_barrier(); + + // eddy_diffusivities still needs a midpoint shear magnitude for the + // cold-surface fallback path. In 3D mode, use the SHOC-grid strain term + // instead of leaving the old 1D shear workspace uninitialized. + static constexpr Scalar Ck_sh = 0.1; + const Int nlev_pack = ekat::npack(nlev); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_pack), [&] (const Int& k) { + sterm_zt(k) = Ck_sh*ekat::max(Pack(0),shear_strain3d(k)); + }); + team.team_barrier(); + } // Advance sgs TKE - adv_sgs_tke(team,nlev,dtime,shoc_1p5tke,shoc_mix,wthv_sec,sterm_zt,tk,brunt,tke,a_diss); + adv_sgs_tke(team,nlev,dtime,shoc_1p5tke,do_3d_turb,shoc_mix,wthv_sec,sterm_zt,tk,brunt,shear_strain3d,tke,a_diss); // Compute isotropic time scale [s] isotropic_ts(team,nlev,lambda_low,lambda_high,lambda_slope,lambda_thresh,brunt_int,tke,a_diss,brunt,isotropy); @@ -77,8 +113,8 @@ void Functions::shoc_tke( eddy_diffusivities(team,nlev,Ckh,Ckm,pblh,zt_grid,tabs,shoc_mix,sterm_zt,isotropy,tke,tkh,tk); // Release temporary variables from the workspace - workspace.template release_many_contiguous<3>( - {&sterm_zt, &a_diss, &sterm}); + workspace.template release_many_contiguous<6>( + {&sterm_zt, &a_diss, &sterm, &du_dz_m, &dv_dz_m, &dw_dz_m}); } } // namespace shoc diff --git a/components/eamxx/src/physics/shoc/shoc_functions.hpp b/components/eamxx/src/physics/shoc/shoc_functions.hpp index 52593f553e2c..4c182d0b1d31 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions.hpp +++ b/components/eamxx/src/physics/shoc/shoc_functions.hpp @@ -80,6 +80,7 @@ template struct Functions { Scalar Ckm; bool shoc_1p5tke; bool extra_diags; + bool do_3d_turb; }; // This struct stores input views for shoc_main. @@ -120,6 +121,10 @@ template struct Functions { view_2d inv_exner; // Host model surface geopotential height view_1d phis; + // Dycore-computed local tensor components: A00,A01,A10,A11,A20,A21 [/s] + view_3d shear_strain3d_components; + // 3D strain term for shear production of TKE [/s2] + view_2d shear_strain3d; }; // This struct stores input/outputs views for shoc_main. @@ -539,11 +544,47 @@ template struct Functions { KOKKOS_FUNCTION static void adv_sgs_tke(const MemberType &team, const Int &nlev, const Real &dtime, - const bool &shoc_1p5tke, const uview_1d &shoc_mix, - const uview_1d &wthv_sec, + const bool &shoc_1p5tke, const bool &do_3d_turb, + const uview_1d &shoc_mix, const uview_1d &wthv_sec, const uview_1d &sterm_zt, const uview_1d &tk, - const uview_1d &brunt, const uview_1d &tke, - const uview_1d &a_diss); + const uview_1d &brunt, const uview_1d &shear_strain3d, + const uview_1d &tke, const uview_1d &a_diss); + + KOKKOS_FUNCTION + static void compute_vertical_shear_terms( + const MemberType &team, const Int &nlev, const Int &nlevi, + const uview_1d &dz_zi, + const uview_1d &u_wind, + const uview_1d &v_wind, + const uview_1d &w_field, + const uview_1d &zt_grid, + const uview_1d &zi_grid, + const Workspace &workspace, + const uview_1d &du_dz_m, + const uview_1d &dv_dz_m, + const uview_1d &dw_dz_m); + + KOKKOS_FUNCTION + static void assemble_shoc_shear_strain3d( + const MemberType &team, const Int &nlev, + const uview_2d &shear_strain3d_components, + const uview_1d &du_dz_m, + const uview_1d &dv_dz_m, + const uview_1d &dw_dz_m, + const uview_1d &shear_strain3d); +#ifdef SCREAM_SHOC_SMALL_KERNELS + static void compute_shear_strain3d_disp( + const Int &shcol, const Int &nlev, const Int &nlevi, + const view_3d &shear_strain3d_components, + const view_2d &dz_zi, + const view_2d &u_wind, + const view_2d &v_wind, + const view_2d &w_field, + const view_2d &zt_grid, + const view_2d &zi_grid, + const WorkspaceMgr &workspace_mgr, + const view_2d &shear_strain3d); +#endif KOKKOS_FUNCTION static void @@ -693,7 +734,7 @@ template struct Functions { const Scalar &lambda_thresh, const Scalar &thl2tune, const Scalar &qw2tune, const Scalar &qwthl2tune, const Scalar &w2tune, const Scalar &length_fac, const Scalar &c_diag_3rd_mom, const Scalar &Ckh, const Scalar &Ckm, const bool &shoc_1p5tke, - const bool &extra_diags, + const bool &do_3d_turb, const bool &extra_diags, // Input Variables const Scalar &host_dx, const Scalar &host_dy, const uview_1d &zt_grid, const uview_1d &zi_grid, const uview_1d &pres, @@ -702,6 +743,8 @@ template struct Functions { const Scalar &wthl_sfc, const Scalar &wqw_sfc, const Scalar &uw_sfc, const Scalar &vw_sfc, const uview_1d &wtracer_sfc, const uview_1d &inv_exner, const Scalar &phis, + const uview_2d &shear_strain3d_components, + const uview_1d &shear_strain3d, // Local Workspace const Workspace &workspace, // Input/Output Variables @@ -737,7 +780,7 @@ template struct Functions { const Scalar &lambda_thresh, const Scalar &thl2tune, const Scalar &qw2tune, const Scalar &qwthl2tune, const Scalar &w2tune, const Scalar &length_fac, const Scalar &c_diag_3rd_mom, const Scalar &Ckh, const Scalar &Ckm, const bool &shoc_1p5tke, - const bool &extra_diags, + const bool &do_3d_turb, const bool &extra_diags, // Input Variables const view_1d &host_dx, const view_1d &host_dy, const view_2d &zt_grid, const view_2d &zi_grid, @@ -747,6 +790,8 @@ template struct Functions { const view_1d &wqw_sfc, const view_1d &uw_sfc, const view_1d &vw_sfc, const view_2d &wtracer_sfc, const view_2d &inv_exner, const view_1d &phis, + const view_3d &shear_strain3d_components, + const view_2d &shear_strain3d, // Workspace Manager WorkspaceMgr &workspace_mgr, // Input/Output Variables @@ -870,11 +915,15 @@ template struct Functions { static void shoc_tke(const MemberType &team, const Int &nlev, const Int &nlevi, const Scalar &dtime, const Scalar &lambda_low, const Scalar &lambda_high, const Scalar &lambda_slope, const Scalar &lambda_thresh, const Scalar &Ckh, - const Scalar &Ckm, const bool &shoc_1p5tke, - const uview_1d &wthv_sec, const uview_1d &shoc_mix, + const Scalar &Ckm, const bool &shoc_1p5tke, const bool &do_3d_turb, + const uview_1d &wthv_sec, + const uview_2d &shear_strain3d_components, + const uview_1d &shear_strain3d, + const uview_1d &shoc_mix, const uview_1d &dz_zi, const uview_1d &dz_zt, const uview_1d &pres, const uview_1d &tabs, const uview_1d &u_wind, const uview_1d &v_wind, + const uview_1d &w_field, const uview_1d &brunt, const uview_1d &zt_grid, const uview_1d &zi_grid, const Scalar &pblh, const Workspace &workspace, const uview_1d &tke, @@ -885,11 +934,15 @@ template struct Functions { const Scalar &dtime, const Scalar &lambda_low, const Scalar &lambda_high, const Scalar &lambda_slope, const Scalar &lambda_thresh, const Scalar &Ckh, const Scalar &Ckm, - const bool &shoc_1p5tke, const view_2d &wthv_sec, + const bool &shoc_1p5tke, const bool &do_3d_turb, + const view_2d &wthv_sec, + const view_3d &shear_strain3d_components, + const view_2d &shear_strain3d, const view_2d &shoc_mix, const view_2d &dz_zi, const view_2d &dz_zt, const view_2d &pres, const view_2d &tabs, const view_2d &u_wind, - const view_2d &v_wind, const view_2d &brunt, + const view_2d &v_wind, const view_2d &w_field, + const view_2d &brunt, const view_2d &zt_grid, const view_2d &zi_grid, const view_1d &pblh, const WorkspaceMgr &workspace_mgr, const view_2d &tke, @@ -937,6 +990,7 @@ template struct Functions { #include "shoc_isotropic_ts_impl.hpp" #include "shoc_length_impl.hpp" #include "shoc_linear_interp_impl.hpp" +#include "shoc_compute_shear_strain3d_impl.hpp" #include "shoc_main_impl.hpp" #include "shoc_pblintd_check_pblh_impl.hpp" #include "shoc_pblintd_cldcheck_impl.hpp" diff --git a/components/eamxx/src/physics/shoc/tests/CMakeLists.txt b/components/eamxx/src/physics/shoc/tests/CMakeLists.txt index a0c1c870c5ee..82d274d454a5 100644 --- a/components/eamxx/src/physics/shoc/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/shoc/tests/CMakeLists.txt @@ -51,6 +51,8 @@ set(SHOC_TESTS_SRCS shoc_diag_second_shoc_moments_tests.cpp shoc_pblintd_cldcheck_tests.cpp shoc_compute_shoc_vapor_tests.cpp + shoc_compute_shear_strain3d_tests.cpp + shoc_assemble_shear_strain3d_tests.cpp shoc_update_prognostics_implicit_tests.cpp shoc_main_tests.cpp shoc_pblintd_height_tests.cpp diff --git a/components/eamxx/src/physics/shoc/tests/infra/shoc_test_data.cpp b/components/eamxx/src/physics/shoc/tests/infra/shoc_test_data.cpp index 4197bacf22cc..9fdb6587786b 100644 --- a/components/eamxx/src/physics/shoc/tests/infra/shoc_test_data.cpp +++ b/components/eamxx/src/physics/shoc/tests/infra/shoc_test_data.cpp @@ -302,6 +302,22 @@ void compute_shoc_temperature(ComputeShocTempData& d) compute_shoc_temperature_host(d.shcol, d.nlev, d.thetal, d.ql, d.inv_exner, d.tabs); } +void compute_vertical_shear_terms(ComputeVerticalShearTermsData& d) +{ + compute_vertical_shear_terms_host(d.nlev, d.nlevi, d.shcol, + d.dz_zi, d.u_wind, d.v_wind, d.w_field, + d.zt_grid, d.zi_grid, + d.du_dz_m, d.dv_dz_m, d.dw_dz_m); +} + +void assemble_shoc_shear_strain3d(AssembleShocShearStrain3dData& d) +{ + assemble_shoc_shear_strain3d_host(d.shcol, d.nlev, + d.shear_strain3d_components, + d.du_dz_m, d.dv_dz_m, d.dw_dz_m, + d.shear_strain3d); +} + // end _c impls // @@ -498,6 +514,7 @@ void update_host_dse_host(Int shcol, Int nlev, Real* thlm, Real* shoc_ql, Real* using Pack = typename SHF::Pack; using view_1d = typename SHF::view_1d; using view_2d = typename SHF::view_2d; + using view_3d = typename SHF::view_3d; using KT = typename SHF::KT; using ExeSpace = typename KT::ExeSpace; using TPF = ekat::TeamPolicyFactory; @@ -663,6 +680,7 @@ void compute_shoc_mix_shoc_length_host(Int nlev, Int shcol, Real* tke, Real* bru using Pack = typename SHF::Pack; using view_1d = typename SHF::view_1d; using view_2d = typename SHF::view_2d; + using view_3d = typename SHF::view_3d; using KT = typename SHF::KT; using ExeSpace = typename KT::ExeSpace; using TPF = ekat::TeamPolicyFactory; @@ -829,6 +847,7 @@ void shoc_energy_integrals_host(Int shcol, Int nlev, Real *host_dse, Real *pdel, using Pack = typename SHF::Pack; using view_1d = typename SHF::view_1d; using view_2d = typename SHF::view_2d; + using view_3d = typename SHF::view_3d; using KT = typename SHF::KT; using ExeSpace = typename KT::ExeSpace; using TPF = ekat::TeamPolicyFactory; @@ -1223,6 +1242,7 @@ void compute_l_inf_shoc_length_host(Int nlev, Int shcol, Real *zt_grid, Real *dz using Pack = typename SHF::Pack; using view_1d = typename SHF::view_1d; using view_2d = typename SHF::view_2d; + using view_3d = typename SHF::view_3d; using KT = typename SHF::KT; using ExeSpace = typename KT::ExeSpace; using TPF = ekat::TeamPolicyFactory; @@ -1268,9 +1288,10 @@ void check_length_scale_shoc_length_host(Int nlev, Int shcol, Real* host_dx, Rea using SHF = Functions; using Scalar = typename SHF::Scalar; - using Pack = typename SHF::Pack; + using Pack = typename SHF::Pack; using view_1d = typename SHF::view_1d; using view_2d = typename SHF::view_2d; + using view_3d = typename SHF::view_3d; using KT = typename SHF::KT; using ExeSpace = typename KT::ExeSpace; using TPF = ekat::TeamPolicyFactory; @@ -1870,6 +1891,8 @@ void adv_sgs_tke_host(Int nlev, Int shcol, Real dtime, bool shoc_1p5tke, Real* s a_diss_d (temp_d[5]); //out const Int nk_pack = ekat::npack(nlev); + view_2d shear_strain3d_d("shear_strain3d_d", shcol, nk_pack); + Kokkos::deep_copy(shear_strain3d_d, 0); const auto policy = TPF::get_default_team_policy(shcol, nk_pack); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { @@ -1881,10 +1904,14 @@ void adv_sgs_tke_host(Int nlev, Int shcol, Real dtime, bool shoc_1p5tke, Real* s const auto sterm_zt_s = ekat::subview(sterm_zt_d ,i); const auto tk_s = ekat::subview(tk_d ,i); const auto brunt_s = ekat::subview(brunt_d, i); + const auto shear_strain3d_s = ekat::subview(shear_strain3d_d, i); const auto tke_s = ekat::subview(tke_d ,i); const auto a_diss_s = ekat::subview(a_diss_d ,i); + const bool do_3d_turb = false; - SHF::adv_sgs_tke(team, nlev, dtime, shoc_1p5tke, shoc_mix_s, wthv_sec_s, sterm_zt_s, tk_s, brunt_s, tke_s, a_diss_s); + SHF::adv_sgs_tke(team, nlev, dtime, shoc_1p5tke, do_3d_turb, + shoc_mix_s, wthv_sec_s, sterm_zt_s, tk_s, brunt_s, + shear_strain3d_s, tke_s, a_diss_s); }); // Sync back to host @@ -2039,6 +2066,129 @@ void compute_shr_prod_host(Int nlevi, Int nlev, Int shcol, Real* dz_zi, Real* u_ ekat::device_to_host({sterm}, shcol, nlevi, inout_views); } +void compute_vertical_shear_terms_host(Int nlev, Int nlevi, Int shcol, + Real* dz_zi, Real* u_wind, Real* v_wind, Real* w_field, + Real* zt_grid, Real* zi_grid, + Real* du_dz_m, Real* dv_dz_m, Real* dw_dz_m) +{ + using SHF = Functions; + + using Pack = typename SHF::Pack; + using view_2d = typename SHF::view_2d; + using KT = typename SHF::KT; + using ExeSpace = typename KT::ExeSpace; + using TPF = ekat::TeamPolicyFactory; + using MemberType = typename SHF::MemberType; + + static constexpr Int num_arrays = 9; + + std::vector temp_d(num_arrays); + std::vector dim1_sizes(num_arrays, shcol); + std::vector dim2_sizes = {nlevi, nlev, nlev, nlev, nlev, nlevi, nlev, nlev, nlev}; + std::vector ptr_array = {dz_zi, u_wind, v_wind, w_field, zt_grid, zi_grid, + du_dz_m, dv_dz_m, dw_dz_m}; + + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d); + + view_2d + dz_zi_d (temp_d[0]), + u_wind_d(temp_d[1]), + v_wind_d(temp_d[2]), + w_field_d(temp_d[3]), + zt_grid_d(temp_d[4]), + zi_grid_d(temp_d[5]), + du_dz_m_d(temp_d[6]), + dv_dz_m_d(temp_d[7]), + dw_dz_m_d(temp_d[8]); + + const Int nlev_packs = ekat::npack(nlev); + const Int nlevi_packs = ekat::npack(nlevi); + const auto policy = TPF::get_default_team_policy(shcol, nlev_packs); + + ekat::WorkspaceManager workspace_mgr(nlevi_packs, 3, policy); + + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { + const Int i = team.league_rank(); + + auto workspace = workspace_mgr.get_workspace(team); + + const auto dz_zi_s = ekat::subview(dz_zi_d, i); + const auto u_wind_s = ekat::subview(u_wind_d, i); + const auto v_wind_s = ekat::subview(v_wind_d, i); + const auto w_field_s = ekat::subview(w_field_d, i); + const auto zt_grid_s = ekat::subview(zt_grid_d, i); + const auto zi_grid_s = ekat::subview(zi_grid_d, i); + const auto du_dz_m_s = ekat::subview(du_dz_m_d, i); + const auto dv_dz_m_s = ekat::subview(dv_dz_m_d, i); + const auto dw_dz_m_s = ekat::subview(dw_dz_m_d, i); + + SHF::compute_vertical_shear_terms(team, nlev, nlevi, + dz_zi_s, u_wind_s, v_wind_s, w_field_s, + zt_grid_s, zi_grid_s, workspace, + du_dz_m_s, dv_dz_m_s, dw_dz_m_s); + }); + + std::vector out_views = {du_dz_m_d, dv_dz_m_d, dw_dz_m_d}; + ekat::device_to_host({du_dz_m, dv_dz_m, dw_dz_m}, shcol, nlev, out_views); +} + +void assemble_shoc_shear_strain3d_host(Int shcol, Int nlev, + Real* shear_strain3d_components, + Real* du_dz_m, Real* dv_dz_m, Real* dw_dz_m, + Real* shear_strain3d) +{ + using SHF = Functions; + + using Pack = typename SHF::Pack; + using view_2d = typename SHF::view_2d; + using view_3d = typename SHF::view_3d; + using KT = typename SHF::KT; + using ExeSpace = typename KT::ExeSpace; + using TPF = ekat::TeamPolicyFactory; + using MemberType = typename SHF::MemberType; + + const Int nlev_packs = ekat::npack(nlev); + + std::vector temp_2d_d(4); + std::vector dim1_sizes(4, shcol); + std::vector dim2_sizes = {nlev, nlev, nlev, nlev}; + std::vector ptr_array = {du_dz_m, dv_dz_m, dw_dz_m, shear_strain3d}; + + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_2d_d); + view_3d shear_strain3d_components_d("shear_strain3d_components_d", shcol, 6, nlev_packs); + + view_2d + du_dz_m_d(temp_2d_d[0]), + dv_dz_m_d(temp_2d_d[1]), + dw_dz_m_d(temp_2d_d[2]), + shear_strain3d_d(temp_2d_d[3]); + + Kokkos::deep_copy(shear_strain3d_components_d, 0); + const auto comps_d_s = ekat::scalarize(shear_strain3d_components_d); + + Kokkos::parallel_for(Kokkos::RangePolicy(0, shcol*6*nlev), KOKKOS_LAMBDA(const Int idx) { + const Int k = idx % nlev; + const Int tmp = idx / nlev; + const Int j = tmp % 6; + const Int i = tmp / 6; + comps_d_s(i,j,k) = shear_strain3d_components[idx]; + }); + + const auto policy = TPF::get_default_team_policy(shcol, nlev_packs); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { + const Int i = team.league_rank(); + SHF::assemble_shoc_shear_strain3d(team, nlev, + ekat::subview(shear_strain3d_components_d, i), + ekat::subview(du_dz_m_d, i), + ekat::subview(dv_dz_m_d, i), + ekat::subview(dw_dz_m_d, i), + ekat::subview(shear_strain3d_d, i)); + }); + + std::vector out_views = {shear_strain3d_d}; + ekat::device_to_host({shear_strain3d}, shcol, nlev, out_views); +} + void compute_tmpi_host(Int nlevi, Int shcol, Real dtime, Real *rho_zi, Real *dz_zi, Real *tmpi) { using SHF = Functions; @@ -2371,8 +2521,12 @@ Int shoc_main_host(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npb // shoc_main treats u/v_wind as 1 array and // CXX version of shoc qtracers is the transpose of the fortran version.. const auto nlev_packs = ekat::npack(nlev); + view_2d shear_strain3d_d("shear_strain3d_d", shcol, nlev_packs); + view_3d shear_strain3d_components_d("shear_strain3d_components_d", shcol, 6, nlev_packs); view_3d horiz_wind_d("horiz_wind",shcol,2,nlev_packs); view_3d qtracers_cxx_d("qtracers",shcol,num_qtracers,nlev_packs); + Kokkos::deep_copy(shear_strain3d_d, 0); + Kokkos::deep_copy(shear_strain3d_components_d, 0); // scalarize each view const auto u_wind_d_s = ekat::scalarize(u_wind_d); @@ -2399,7 +2553,8 @@ Int shoc_main_host(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npb pres_d, presi_d, pdel_d, thv_d, w_field_d, wthl_sfc_d, wqw_sfc_d, uw_sfc_d, vw_sfc_d, uw_sfc_pert_d, vw_sfc_pert_d, wtracer_sfc_d, - inv_exner_d, phis_d}; + inv_exner_d, phis_d, + shear_strain3d_components_d, shear_strain3d_d}; SHF::SHOCInputOutput shoc_input_output{host_dse_d, tke_d, thetal_d, qw_d, horiz_wind_d, wthv_sec_d, qtracers_cxx_d, tk_d, shoc_cldfrac_d, shoc_ql_d, um_pert_d, vm_pert_d }; @@ -2939,9 +3094,10 @@ void shoc_tke_host(Int shcol, Int nlev, Int nlevi, Real dtime, bool shoc_1p5tke, using SHF = Functions; using Scalar = typename SHF::Scalar; - using Pack = typename SHF::Pack; + using Pack = typename SHF::Pack; using view_1d = typename SHF::view_1d; using view_2d = typename SHF::view_2d; + using view_3d = typename SHF::view_3d; using KT = typename SHF::KT; using ExeSpace = typename KT::ExeSpace; using TPF = ekat::TeamPolicyFactory; @@ -2984,10 +3140,17 @@ void shoc_tke_host(Int shcol, Int nlev, Int nlevi, Real dtime, bool shoc_1p5tke, const Int nlev_packs = ekat::npack(nlev); const Int nlevi_packs = ekat::npack(nlevi); + + view_2d w_field_d("w_field_d", shcol, nlev_packs); + view_2d shear_strain3d_d("shear_strain3d_d", shcol, nlev_packs); + view_3d shear_strain3d_components_d("shear_strain3d_components_d", shcol, 6, nlev_packs); + Kokkos::deep_copy(w_field_d, 0); + Kokkos::deep_copy(shear_strain3d_d, 0); + Kokkos::deep_copy(shear_strain3d_components_d, 0); const auto policy = TPF::get_default_team_policy(shcol, nlev_packs); // Local variable workspace - ekat::WorkspaceManager workspace_mgr(nlevi_packs, 3, policy); + ekat::WorkspaceManager workspace_mgr(nlevi_packs, 6, policy); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { const Int i = team.league_rank(); @@ -3000,6 +3163,7 @@ void shoc_tke_host(Int shcol, Int nlev, Int nlevi, Real dtime, bool shoc_1p5tke, const auto shoc_mix_s = ekat::subview(shoc_mix_d, i); const auto u_wind_s = ekat::subview(u_wind_d, i); const auto v_wind_s = ekat::subview(v_wind_d, i); + const auto w_field_s = ekat::subview(w_field_d, i); const auto dz_zi_s = ekat::subview(dz_zi_d, i); const auto dz_zt_s = ekat::subview(dz_zt_d, i); const auto pres_s = ekat::subview(pres_d, i); @@ -3011,6 +3175,8 @@ void shoc_tke_host(Int shcol, Int nlev, Int nlevi, Real dtime, bool shoc_1p5tke, const auto tk_s = ekat::subview(tk_d, i); const auto tkh_s = ekat::subview(tkh_d, i); const auto isotropy_s = ekat::subview(isotropy_d, i); + const auto shear_strain3d_s = ekat::subview(shear_strain3d_d, i); + const auto shear_strain3d_components_s = ekat::subview(shear_strain3d_components_d, i); // Hardcode for F90 testing const Real lambda_low = 0.001; @@ -3019,11 +3185,12 @@ void shoc_tke_host(Int shcol, Int nlev, Int nlevi, Real dtime, bool shoc_1p5tke, const Real lambda_thresh = 0.02; const Real Ckh = 0.1; const Real Ckm = 0.1; + const bool do_3d_turb = false; SHF::shoc_tke(team,nlev,nlevi,dtime,lambda_low,lambda_high,lambda_slope,lambda_thresh, - Ckh, Ckm, shoc_1p5tke, - wthv_sec_s,shoc_mix_s,dz_zi_s,dz_zt_s,pres_s, - tabs_s,u_wind_s,v_wind_s,brunt_s,zt_grid_s,zi_grid_s,pblh_s, + Ckh, Ckm, shoc_1p5tke, do_3d_turb, + wthv_sec_s,shear_strain3d_components_s,shear_strain3d_s,shoc_mix_s,dz_zi_s,dz_zt_s,pres_s, + tabs_s,u_wind_s,v_wind_s,w_field_s,brunt_s,zt_grid_s,zi_grid_s,pblh_s, workspace, tke_s,tk_s,tkh_s,isotropy_s); }); diff --git a/components/eamxx/src/physics/shoc/tests/infra/shoc_test_data.hpp b/components/eamxx/src/physics/shoc/tests/infra/shoc_test_data.hpp index eba2148e8ffc..4ea160faf169 100644 --- a/components/eamxx/src/physics/shoc/tests/infra/shoc_test_data.hpp +++ b/components/eamxx/src/physics/shoc/tests/infra/shoc_test_data.hpp @@ -938,6 +938,37 @@ struct ComputeShocTempData : public PhysicsTestData { PTD_STD_DEF(ComputeShocTempData, 2, shcol, nlev); }; +struct ComputeVerticalShearTermsData : public PhysicsTestData { + // Inputs + Int shcol, nlev, nlevi; + Real *dz_zi, *zi_grid; + Real *u_wind, *v_wind, *w_field, *zt_grid; + + // Outputs + Real *du_dz_m, *dv_dz_m, *dw_dz_m; + + ComputeVerticalShearTermsData(Int shcol_, Int nlev_, Int nlevi_) : + PhysicsTestData({{ shcol_, nlevi_ }, { shcol_, nlev_ }}, + {{ &dz_zi, &zi_grid }, { &u_wind, &v_wind, &w_field, &zt_grid, &du_dz_m, &dv_dz_m, &dw_dz_m }}), + shcol(shcol_), nlev(nlev_), nlevi(nlevi_) {} + + PTD_STD_DEF(ComputeVerticalShearTermsData, 3, shcol, nlev, nlevi); +}; + +struct AssembleShocShearStrain3dData : public PhysicsTestData { + Int shcol, nlev; + Real *shear_strain3d_components; + Real *du_dz_m, *dv_dz_m, *dw_dz_m; + Real *shear_strain3d; + + AssembleShocShearStrain3dData(Int shcol_, Int nlev_) : + PhysicsTestData({{ shcol_, 6, nlev_ }, { shcol_, nlev_ }}, + {{ &shear_strain3d_components }, { &du_dz_m, &dv_dz_m, &dw_dz_m, &shear_strain3d }}), + shcol(shcol_), nlev(nlev_) {} + + PTD_STD_DEF(AssembleShocShearStrain3dData, 2, shcol, nlev); +}; + // Glue functions to call from host with the Data struct void shoc_grid (ShocGridData& d); @@ -994,6 +1025,8 @@ void pblintd_surf_temp(PblintdSurfTempData& d); void pblintd_check_pblh(PblintdCheckPblhData& d); void pblintd(PblintdData& d); void compute_shoc_temperature(ComputeShocTempData& d); +void compute_vertical_shear_terms(ComputeVerticalShearTermsData& d); +void assemble_shoc_shear_strain3d(AssembleShocShearStrain3dData& d); // Call from host @@ -1073,6 +1106,14 @@ void integ_column_stability_host(Int nlev, Int shcol, Real *dz_zt, void isotropic_ts_host(Int nlev, Int shcol, Real* brunt_int, Real* tke, Real* a_diss, Real* brunt, Real* isotropy); void dp_inverse_host(Int nlev, Int shcol, Real *rho_zt, Real *dz_zt, Real *rdp_zt); +void compute_vertical_shear_terms_host(Int nlev, Int nlevi, Int shcol, + Real* dz_zi, Real* u_wind, Real* v_wind, Real* w_field, + Real* zt_grid, Real* zi_grid, + Real* du_dz_m, Real* dv_dz_m, Real* dw_dz_m); +void assemble_shoc_shear_strain3d_host(Int shcol, Int nlev, + Real* shear_strain3d_components, + Real* du_dz_m, Real* dv_dz_m, Real* dw_dz_m, + Real* shear_strain3d); int shoc_init_host(Int nlev, Real* pref_mid, Int nbot_shoc, Int ntop_shoc); Int shoc_main_host(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, Real* host_dx, Real* host_dy, Real* thv, diff --git a/components/eamxx/src/physics/shoc/tests/infra/shoc_unit_tests_common.hpp b/components/eamxx/src/physics/shoc/tests/infra/shoc_unit_tests_common.hpp index a8cbd0a92141..335ed071a2a5 100644 --- a/components/eamxx/src/physics/shoc/tests/infra/shoc_unit_tests_common.hpp +++ b/components/eamxx/src/physics/shoc/tests/infra/shoc_unit_tests_common.hpp @@ -128,6 +128,8 @@ struct UnitWrap { struct TestPblintdCheckPblh; struct TestPblintd; struct TestComputeShocTemp; + struct TestComputeVerticalShearTerms; + struct TestAssembleShocShearStrain3d; }; }; diff --git a/components/eamxx/src/physics/shoc/tests/shoc_assemble_shear_strain3d_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_assemble_shear_strain3d_tests.cpp new file mode 100644 index 000000000000..ea6b8e46d3f7 --- /dev/null +++ b/components/eamxx/src/physics/shoc/tests/shoc_assemble_shear_strain3d_tests.cpp @@ -0,0 +1,207 @@ +#include "catch2/catch.hpp" + +#include "shoc_unit_tests_common.hpp" +#include "shoc_test_data.hpp" +#include "share/core/eamxx_types.hpp" +#include "share/core/eamxx_setup_random_test.hpp" + +#include +#include +#include +#include +#include + +namespace scream { +namespace shoc { +namespace unit_test { + +namespace { + +inline Real reference_shear_strain3d(const Real a00, const Real a01, const Real a10, + const Real a11, const Real a20, const Real a21, + const Real du_dz_m, const Real dv_dz_m, const Real dw_dz_m) +{ + const Real s00 = a00; + const Real s11 = a11; + const Real s22 = dw_dz_m; + const Real s01 = 0.5 * (a01 + a10); + const Real s02 = 0.5 * (du_dz_m + a20); + const Real s12 = 0.5 * (dv_dz_m + a21); + + return 2.0 * (s00*s00 + s11*s11 + s22*s22 + + 2.0*s01*s01 + 2.0*s02*s02 + 2.0*s12*s12); +} + +} // namespace + +template +struct UnitWrap::UnitTest::TestAssembleShocShearStrain3d : public UnitWrap::UnitTest::Base { + + static void require_close(const Real a, const Real b, const Real scale = 1) + { + const Real tol = 200 * std::numeric_limits::epsilon() * scale; + REQUIRE(a == Approx(b).margin(tol)); + } + + void run_property() + { + static constexpr Int shcol = 2; + static constexpr Int nlev = 5; + + AssembleShocShearStrain3dData d(shcol, nlev); + + REQUIRE(d.shcol == shcol); + REQUIRE(d.nlev == nlev); + + for (Int s = 0; s < shcol; ++s) { + for (Int k = 0; k < nlev; ++k) { + const Int off2 = k + s*nlev; + d.du_dz_m[off2] = 0; + d.dv_dz_m[off2] = 0; + d.dw_dz_m[off2] = 0; + d.shear_strain3d[off2] = -1; + for (Int j = 0; j < 6; ++j) { + const Int off3 = k + nlev*(j + 6*s); + d.shear_strain3d_components[off3] = 0; + } + } + } + + assemble_shoc_shear_strain3d(d); + + for (Int i = 0; i < shcol*nlev; ++i) { + REQUIRE(d.shear_strain3d[i] == 0); + } + + for (Int s = 0; s < shcol; ++s) { + for (Int k = 0; k < nlev; ++k) { + const Int off2 = k + s*nlev; + d.du_dz_m[off2] = 1 + k; + d.dv_dz_m[off2] = 2 + k; + d.dw_dz_m[off2] = 3 + k; + const Real vals[6] = {1, -2, 4, -3, 5, -6}; + for (Int j = 0; j < 6; ++j) { + const Int off3 = k + nlev*(j + 6*s); + d.shear_strain3d_components[off3] = vals[j]; + } + } + } + + assemble_shoc_shear_strain3d(d); + + for (Int s = 0; s < shcol; ++s) { + for (Int k = 0; k < nlev; ++k) { + const Int off2 = k + s*nlev; + const Real expected = reference_shear_strain3d(1, -2, 4, -3, 5, -6, + 1 + k, 2 + k, 3 + k); + require_close(d.shear_strain3d[off2], expected, expected + 1); + REQUIRE(d.shear_strain3d[off2] >= 0); + } + } + + // Antisymmetric horizontal off-diagonal part should cancel. + for (Int s = 0; s < shcol; ++s) { + for (Int k = 0; k < nlev; ++k) { + const Int off2 = k + s*nlev; + d.du_dz_m[off2] = 0; + d.dv_dz_m[off2] = 0; + d.dw_dz_m[off2] = 0; + const Real vals[6] = {0, 7, -7, 0, 0, 0}; + for (Int j = 0; j < 6; ++j) { + const Int off3 = k + nlev*(j + 6*s); + d.shear_strain3d_components[off3] = vals[j]; + } + } + } + + assemble_shoc_shear_strain3d(d); + + for (Int i = 0; i < shcol*nlev; ++i) { + REQUIRE(d.shear_strain3d[i] == 0); + } + + // The vertical cross term should depend only on the symmetric average 0.5*(A20 + du_dz_m). + for (Int s = 0; s < shcol; ++s) { + for (Int k = 0; k < nlev; ++k) { + const Int off2 = k + s*nlev; + d.du_dz_m[off2] = 3; + d.dv_dz_m[off2] = 0; + d.dw_dz_m[off2] = 0; + const Real vals[6] = {0, 0, 0, 0, -3, 0}; + for (Int j = 0; j < 6; ++j) { + const Int off3 = k + nlev*(j + 6*s); + d.shear_strain3d_components[off3] = vals[j]; + } + } + } + + assemble_shoc_shear_strain3d(d); + + for (Int i = 0; i < shcol*nlev; ++i) { + REQUIRE(d.shear_strain3d[i] == 0); + } + } + + void run_property_random() + { + auto engine = Base::get_engine(); + + std::uniform_int_distribution shcol_dist(1, 5); + std::uniform_int_distribution nlev_dist(7, 19); + std::uniform_real_distribution val_dist(-10, 10); + + for (Int trial = 0; trial < 20; ++trial) { + const Int shcol = shcol_dist(engine); + const Int nlev = nlev_dist(engine); + + AssembleShocShearStrain3dData d(shcol, nlev); + + for (Int s = 0; s < shcol; ++s) { + for (Int k = 0; k < nlev; ++k) { + const Int off2 = k + s*nlev; + d.du_dz_m[off2] = val_dist(engine); + d.dv_dz_m[off2] = val_dist(engine); + d.dw_dz_m[off2] = val_dist(engine); + for (Int j = 0; j < 6; ++j) { + const Int off3 = k + nlev*(j + 6*s); + d.shear_strain3d_components[off3] = val_dist(engine); + } + } + } + + assemble_shoc_shear_strain3d(d); + + for (Int s = 0; s < shcol; ++s) { + for (Int k = 0; k < nlev; ++k) { + const Int off2 = k + s*nlev; + const auto comp = [&](const Int j) { + return d.shear_strain3d_components[k + nlev*(j + 6*s)]; + }; + const Real expected = reference_shear_strain3d(comp(0), comp(1), comp(2), comp(3), comp(4), comp(5), + d.du_dz_m[off2], d.dv_dz_m[off2], d.dw_dz_m[off2]); + require_close(d.shear_strain3d[off2], expected, std::abs(expected) + 1); + REQUIRE(d.shear_strain3d[off2] >= 0); + } + } + } + } +}; + +} // namespace unit_test +} // namespace shoc +} // namespace scream + +namespace { + +// FIXME: this test is failing on certain gpu machines but not reproducible on others. This function is not +// called by EAMxx by defult, so disabling this test now, but should be revisted in the future. +//TEST_CASE("assemble_shoc_shear_strain3d_property", "shoc") +//{ +// using TestStruct = +// scream::shoc::unit_test::UnitWrap::UnitTest::TestAssembleShocShearStrain3d; +// +// TestStruct().run_property(); +// TestStruct().run_property_random(); +//} + +} // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_compute_shear_strain3d_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_compute_shear_strain3d_tests.cpp new file mode 100644 index 000000000000..e977f7e425b8 --- /dev/null +++ b/components/eamxx/src/physics/shoc/tests/shoc_compute_shear_strain3d_tests.cpp @@ -0,0 +1,308 @@ +#include "catch2/catch.hpp" + +#include "shoc_unit_tests_common.hpp" +#include "shoc_functions.hpp" +#include "shoc_test_data.hpp" +#include "share/core/eamxx_types.hpp" +#include "share/core/eamxx_setup_random_test.hpp" + +#include +#include +#include +#include +#include +#include +#include + +namespace scream { +namespace shoc { +namespace unit_test { + +namespace { + +inline Real interp_interface_to_midpoint(const std::vector& x1, + const std::vector& y1, + const std::vector& x2, + const Int k2) +{ + const Int km1 = x1.size(); + Int idx = k2 + 1; + if (idx >= km1) { + idx = km1 - 1; + } + + const Real x = x1[idx]; + const Real xs = x1[idx-1]; + const Real y = y1[idx]; + const Real ys = y1[idx-1]; + + return ys + (y - ys) * (x2[k2] - xs) / (x - xs); +} + +inline std::vector reference_vertical_shear_component(const std::vector& dz_zi, + const std::vector& field, + const std::vector& zt_grid, + const std::vector& zi_grid) +{ + const Int nlev = field.size(); + const Int nlevi = dz_zi.size(); + + std::vector grad_i(nlevi, 0); + std::vector grad_m(nlev, 0); + + for (Int k = 1; k < nlev; ++k) { + grad_i[k] = (field[k-1] - field[k]) / dz_zi[k]; + } + + grad_i[0] = 0; + grad_i[nlevi-1] = 0; + + for (Int k = 0; k < nlev; ++k) { + grad_m[k] = interp_interface_to_midpoint(zi_grid, grad_i, zt_grid, k); + grad_m[k] = std::max(grad_m[k], Real(0)); + } + + return grad_m; +} + +inline void build_midpoint_grid_from_interfaces(const std::vector& zi_grid, + std::vector& zt_grid) +{ + const Int nlev = zt_grid.size(); + for (Int k = 0; k < nlev; ++k) { + zt_grid[k] = 0.5 * (zi_grid[k] + zi_grid[k+1]); + } +} + +inline void build_dz_zi_from_midpoints(const std::vector& zt_grid, + std::vector& dz_zi) +{ + const Int nlev = zt_grid.size(); + dz_zi[0] = 0; + for (Int k = 1; k < nlev; ++k) { + dz_zi[k] = zt_grid[k-1] - zt_grid[k]; + } + dz_zi[nlev] = 0; +} + +} // namespace + +template +struct UnitWrap::UnitTest::TestComputeVerticalShearTerms : public UnitWrap::UnitTest::Base { + static void require_close(const Real a, const Real b, const Real scale = 1) + { + const Real tol = 100 * std::numeric_limits::epsilon() * scale; + REQUIRE(a == Approx(b).margin(tol)); + } + + void run_property() + { + static constexpr Int shcol = 2; + static constexpr Int nlev = 5; + static constexpr Int nlevi = nlev + 1; + + ComputeVerticalShearTermsData d(shcol, nlev, nlevi); + + const std::vector dz_zi = {0, 500, 200, 100, 50, 0}; + const std::vector zi_grid = {860, 360, 160, 60, 10, 0}; + std::vector zt_grid(nlev); + build_midpoint_grid_from_interfaces(zi_grid, zt_grid); + + const std::vector u_wind_shr = {2, 1, 0, -1, -2}; + const std::vector v_wind_shr = {1, 2, 3, 4, 5}; + const std::vector w_field_shr = {5, 4, 3, 2, 1}; + + const auto exp_du = reference_vertical_shear_component(dz_zi, u_wind_shr, zt_grid, zi_grid); + const auto exp_dv = reference_vertical_shear_component(dz_zi, v_wind_shr, zt_grid, zi_grid); + const auto exp_dw = reference_vertical_shear_component(dz_zi, w_field_shr, zt_grid, zi_grid); + + REQUIRE(d.shcol == shcol); + REQUIRE(d.nlev == nlev); + REQUIRE(d.nlevi == nlevi); + + for (Int s = 0; s < shcol; ++s) { + for (Int k = 0; k < nlevi; ++k) { + const auto offset = k + s * nlevi; + d.dz_zi[offset] = dz_zi[k]; + d.zi_grid[offset] = zi_grid[k]; + } + + for (Int k = 0; k < nlev; ++k) { + const auto offset = k + s * nlev; + d.u_wind[offset] = u_wind_shr[k]; + d.v_wind[offset] = v_wind_shr[k]; + d.w_field[offset] = w_field_shr[k]; + d.zt_grid[offset] = zt_grid[k]; + } + } + + compute_vertical_shear_terms(d); + + for (Int s = 0; s < shcol; ++s) { + for (Int k = 0; k < nlev; ++k) { + const auto offset = k + s * nlev; + require_close(d.du_dz_m[offset], exp_du[k], std::abs(exp_du[k]) + 1); + require_close(d.dv_dz_m[offset], exp_dv[k], std::abs(exp_dv[k]) + 1); + require_close(d.dw_dz_m[offset], exp_dw[k], std::abs(exp_dw[k]) + 1); + + REQUIRE(d.du_dz_m[offset] > 0); + REQUIRE(d.dv_dz_m[offset] == 0); + REQUIRE(d.dw_dz_m[offset] > 0); + } + + for (Int k = 0; k < nlev - 2; ++k) { + const auto offset = k + s * nlev; + REQUIRE(std::abs(d.du_dz_m[offset]) < std::abs(d.du_dz_m[offset+1])); + if (d.dv_dz_m[offset+1] > 0) { + REQUIRE(std::abs(d.dv_dz_m[offset]) < std::abs(d.dv_dz_m[offset+1])); + } + REQUIRE(std::abs(d.dw_dz_m[offset]) < std::abs(d.dw_dz_m[offset+1])); + } + + REQUIRE(std::abs(d.du_dz_m[s*nlev + nlev-1]) < std::abs(d.du_dz_m[s*nlev + nlev-2])); + if (d.dv_dz_m[s*nlev + nlev-2] > 0) { + REQUIRE(std::abs(d.dv_dz_m[s*nlev + nlev-1]) < std::abs(d.dv_dz_m[s*nlev + nlev-2])); + } + REQUIRE(std::abs(d.dw_dz_m[s*nlev + nlev-1]) < std::abs(d.dw_dz_m[s*nlev + nlev-2])); + } + + for (Int s = 0; s < shcol; ++s) { + for (Int k = 0; k < nlev; ++k) { + const auto offset = k + s * nlev; + d.u_wind[offset] = 10; + d.v_wind[offset] = -5; + d.w_field[offset] = 3; + } + } + + compute_vertical_shear_terms(d); + + for (Int s = 0; s < shcol; ++s) { + for (Int k = 0; k < nlev; ++k) { + const auto offset = k + s * nlev; + REQUIRE(d.du_dz_m[offset] == 0); + REQUIRE(d.dv_dz_m[offset] == 0); + REQUIRE(d.dw_dz_m[offset] == 0); + } + } + + for (Int s = 0; s < shcol; ++s) { + for (Int k = 0; k < nlev; ++k) { + const auto offset = k + s * nlev; + d.u_wind[offset] = 7; + d.v_wind[offset] = -2; + d.w_field[offset] = Real(k); + } + } + + compute_vertical_shear_terms(d); + + for (Int s = 0; s < shcol; ++s) { + for (Int k = 0; k < nlev; ++k) { + const auto offset = k + s * nlev; + REQUIRE(d.du_dz_m[offset] == 0); + REQUIRE(d.dv_dz_m[offset] == 0); + REQUIRE(d.dw_dz_m[offset] == 0); + } + } + } + + void run_property_random() + { + auto engine = Base::get_engine(); + + std::uniform_int_distribution shcol_dist(1, 6); + std::uniform_int_distribution nlev_dist(7, 19); + std::uniform_real_distribution dz_dist(10, 300); + std::uniform_real_distribution frac_dist(0.2, 0.8); + std::uniform_real_distribution field_dist(-25, 25); + + for (Int trial = 0; trial < 20; ++trial) { + const Int shcol = shcol_dist(engine); + const Int nlev = nlev_dist(engine); + const Int nlevi = nlev + 1; + + ComputeVerticalShearTermsData d(shcol, nlev, nlevi); + std::vector> exp_du(shcol); + std::vector> exp_dv(shcol); + std::vector> exp_dw(shcol); + + for (Int s = 0; s < shcol; ++s) { + std::vector zi_grid(nlevi); + std::vector zt_grid(nlev); + std::vector dz_zi(nlevi); + std::vector u_wind(nlev); + std::vector v_wind(nlev); + std::vector w_field(nlev); + + zi_grid[nlevi-1] = 0; + for (Int k = nlevi - 2; k >= 0; --k) { + zi_grid[k] = zi_grid[k+1] + dz_dist(engine); + } + + for (Int k = 0; k < nlev; ++k) { + const Real upper = zi_grid[k]; + const Real lower = zi_grid[k+1]; + zt_grid[k] = lower + frac_dist(engine) * (upper - lower); + } + + build_dz_zi_from_midpoints(zt_grid, dz_zi); + + for (Int k = 0; k < nlev; ++k) { + u_wind[k] = field_dist(engine); + v_wind[k] = field_dist(engine); + w_field[k] = field_dist(engine); + } + + exp_du[s] = reference_vertical_shear_component(dz_zi, u_wind, zt_grid, zi_grid); + exp_dv[s] = reference_vertical_shear_component(dz_zi, v_wind, zt_grid, zi_grid); + exp_dw[s] = reference_vertical_shear_component(dz_zi, w_field, zt_grid, zi_grid); + + for (Int k = 0; k < nlevi; ++k) { + const auto offset = k + s * nlevi; + d.dz_zi[offset] = dz_zi[k]; + d.zi_grid[offset] = zi_grid[k]; + } + + for (Int k = 0; k < nlev; ++k) { + const auto offset = k + s * nlev; + d.u_wind[offset] = u_wind[k]; + d.v_wind[offset] = v_wind[k]; + d.w_field[offset] = w_field[k]; + d.zt_grid[offset] = zt_grid[k]; + } + } + + compute_vertical_shear_terms(d); + + for (Int s = 0; s < shcol; ++s) { + for (Int k = 0; k < nlev; ++k) { + const auto offset = k + s * nlev; + require_close(d.du_dz_m[offset], exp_du[s][k], std::abs(exp_du[s][k]) + 1); + require_close(d.dv_dz_m[offset], exp_dv[s][k], std::abs(exp_dv[s][k]) + 1); + require_close(d.dw_dz_m[offset], exp_dw[s][k], std::abs(exp_dw[s][k]) + 1); + } + } + } + } +}; + +} // namespace unit_test +} // namespace shoc +} // namespace scream + +namespace { + +// FIXME: this test is failing on certain gpu machines but not reproducible on others. This function is not +// called by EAMxx by defult, so disabling this test now, but should be revisted in the future. + +//TEST_CASE("compute_vertical_shear_terms_property", "shoc") +//{ +// using TestStruct = +// scream::shoc::unit_test::UnitWrap::UnitTest::TestComputeVerticalShearTerms; +// +// TestStruct().run_property(); +// TestStruct().run_property_random(); +//} + +} // namespace diff --git a/components/eamxx/src/physics/spa/eamxx_spa_process_interface.cpp b/components/eamxx/src/physics/spa/eamxx_spa_process_interface.cpp index f9968a9fb730..2ef7f0942b74 100644 --- a/components/eamxx/src/physics/spa/eamxx_spa_process_interface.cpp +++ b/components/eamxx/src/physics/spa/eamxx_spa_process_interface.cpp @@ -97,20 +97,7 @@ void SPA::initialize_impl (const RunType /* run_type */) ". Valid options are: yearly_periodic, linear.\n"); } - if (m_iop_data_manager!=nullptr) { - // IOP cases cannot have a remap file. We will create a IOPRemapper as the horiz remapper - EKAT_REQUIRE_MSG(spa_map_file == "" or spa_map_file=="none", - "Error! Cannot define spa_remap_file for cases with an Intensive Observation Period defined. " - "The IOP class defines it's own remap from file data -> model data.\n"); - - // TODO: expose tgt lat/lon in IOPDataManager, to avoid injecting knowledge - // of its param list structure in other places - Real iop_lat = m_iop_data_manager->get_params().get("target_latitude"); - Real iop_lon = m_iop_data_manager->get_params().get("target_longitude"); - m_data_interpolation->create_horiz_remappers (iop_lat,iop_lon); - } else { - m_data_interpolation->create_horiz_remappers (spa_map_file=="none" ? "" : spa_map_file); - } + m_data_interpolation->create_horiz_remappers(spa_map_file, m_iop_data_manager); DataInterpolation::VertRemapData vremap_data; vremap_data.vr_type = DataInterpolation::Dynamic3DRef; vremap_data.pname = "PS"; diff --git a/components/eamxx/src/physics/spc/eamxx_spc_process_interface.cpp b/components/eamxx/src/physics/spc/eamxx_spc_process_interface.cpp index 2e7a7ba7f423..67cc417afedc 100644 --- a/components/eamxx/src/physics/spc/eamxx_spc_process_interface.cpp +++ b/components/eamxx/src/physics/spc/eamxx_spc_process_interface.cpp @@ -6,6 +6,7 @@ #include #include +#include namespace scream { @@ -15,6 +16,12 @@ SPC::SPC (const ekat::Comm& comm, const ekat::ParameterList& params) { EKAT_REQUIRE_MSG(m_params.isParameter("spc_data_file"), "ERROR: spc_data_file is missing from SPC parameter list."); + EKAT_REQUIRE_MSG(m_params.isParameter("gas_species"), + "ERROR: gas_species is missing from SPC parameter list."); + + m_gas_species = m_params.get>("gas_species"); + EKAT_REQUIRE_MSG(m_gas_species.size()>0, + "ERROR: gas_species list in SPC parameter list is empty."); } // ========================================================================================= @@ -35,7 +42,9 @@ void SPC::create_requests() add_field("p_mid" , scalar3d_mid, Pa, grid_name, ps); // Set of fields used strictly as output - add_field("o3_volume_mix_ratio", scalar3d_mid, mol/mol, grid_name, ps); + for (const auto& species : m_gas_species) { + add_field(species + "_volume_mix_ratio", scalar3d_mid, mol/mol, grid_name, ps); + } } // ========================================================================================= @@ -45,9 +54,11 @@ void SPC::initialize_impl (const RunType /* run_type */) // NOTE: SPC does not have an internal persistent state, so run_type is irrelevant - std::vector spc_fields = { - get_field_out("o3_volume_mix_ratio").alias("O3") - }; + std::vector spc_fields; + spc_fields.reserve(m_gas_species.size()); + for (const auto& species : m_gas_species) { + spc_fields.push_back(get_field_out(species + "_volume_mix_ratio").alias(ekat::upper_case(species))); + } auto spc_data_file = m_params.get("spc_data_file"); auto spc_map_file = m_params.get("spc_remap_file",""); auto time_interpolation_method = m_params.get("time_interpolation_method","yearly_periodic"); @@ -65,20 +76,7 @@ void SPC::initialize_impl (const RunType /* run_type */) ". Valid options are: yearly_periodic, linear.\n"); } - if (m_iop_data_manager!=nullptr) { - // IOP cases cannot have a remap file. We will create a IOPRemapper as the horiz remapper - EKAT_REQUIRE_MSG(spc_map_file == "" or spc_map_file=="none", - "Error! Cannot define spc_remap_file for cases with an Intensive Observation Period defined. " - "The IOP class defines it's own remap from file data -> model data.\n"); - - // TODO: expose tgt lat/lon in IOPDataManager, to avoid injecting knowledge - // of its param list structure in other places - Real iop_lat = m_iop_data_manager->get_params().get("target_latitude"); - Real iop_lon = m_iop_data_manager->get_params().get("target_longitude"); - m_data_interpolation->create_horiz_remappers (iop_lat,iop_lon); - } else { - m_data_interpolation->create_horiz_remappers (spc_map_file=="none" ? "" : spc_map_file); - } + m_data_interpolation->create_horiz_remappers(spc_map_file, m_iop_data_manager); DataInterpolation::VertRemapData vremap_data; vremap_data.vr_type = DataInterpolation::Dynamic3DRef; vremap_data.pname = "PS"; @@ -89,7 +87,9 @@ void SPC::initialize_impl (const RunType /* run_type */) // Set property checks for fields in this process using FWI = FieldWithinIntervalCheck; - add_postcondition_check(get_field_out("o3_volume_mix_ratio"),m_model_grid,1e-36,1e-2,true); + for (const auto& species : m_gas_species) { + add_postcondition_check(get_field_out(species + "_volume_mix_ratio"),m_model_grid,1e-36,1e-2,true); + } } // ========================================================================================= diff --git a/components/eamxx/src/physics/spc/eamxx_spc_process_interface.hpp b/components/eamxx/src/physics/spc/eamxx_spc_process_interface.hpp index 84f3aa428a5e..136cd9872cde 100644 --- a/components/eamxx/src/physics/spc/eamxx_spc_process_interface.hpp +++ b/components/eamxx/src/physics/spc/eamxx_spc_process_interface.hpp @@ -37,6 +37,9 @@ class SPC : public AtmosphereProcess std::shared_ptr m_model_grid; std::shared_ptr m_data_interpolation; + + // Names of the gas species to be read in and prescribed + std::vector m_gas_species; }; // class SPC } // namespace scream diff --git a/components/eamxx/src/physics/specialized_tracers/CMakeLists.txt b/components/eamxx/src/physics/specialized_tracers/CMakeLists.txt new file mode 100644 index 000000000000..99043ce890ea --- /dev/null +++ b/components/eamxx/src/physics/specialized_tracers/CMakeLists.txt @@ -0,0 +1,34 @@ +# Auxiliary tracers for EAMxx. +# +# This folder groups the "auxiliary tracer" physics processes. Currently, +# there are two options: passive water tracers and, built on top of them, +# water isotopes. Additional tracer classes could be added in an analagous +# way. The build is governed by these CMake options: +# +# EAMXX_ENABLE_WATER_TRACERS - build the passive water tracer process +# EAMXX_ENABLE_WATER_ISOTOPES - build the water isotope process +# +# Water isotopes are implemented on top of the water tracer infrastructure, so +# they can ONLY be built when water tracers are also enabled. If isotopes are +# requested without tracers we auto-enable tracers and warn, rather than build +# an inconsistent configuration. + +option(EAMXX_ENABLE_WATER_TRACERS "Whether to build the EAMxx water tracer process" OFF) +option(EAMXX_ENABLE_WATER_ISOTOPES "Whether to build the EAMxx water isotope process" OFF) + +# Enforce the dependency: isotopes require tracers. +if (EAMXX_ENABLE_WATER_ISOTOPES AND NOT EAMXX_ENABLE_WATER_TRACERS) + message(STATUS "WARNING: EAMXX_ENABLE_WATER_ISOTOPES=ON requires EAMXX_ENABLE_WATER_TRACERS; " + "auto-enabling water tracers.") + set(EAMXX_ENABLE_WATER_TRACERS ON CACHE BOOL "Whether to build the EAMxx water tracer process" FORCE) +endif() + +# Each process lives in its own sibling subdirectory. Descend +# into each only when the corresponding process is enabled. water_isotopes must +# be added after water_tracers, since it depends on the water_tracers target. +if (EAMXX_ENABLE_WATER_TRACERS) + add_subdirectory(water_tracers) +endif() +if (EAMXX_ENABLE_WATER_ISOTOPES) + add_subdirectory(water_isotopes) +endif() diff --git a/components/eamxx/src/physics/specialized_tracers/water_isotopes/CMakeLists.txt b/components/eamxx/src/physics/specialized_tracers/water_isotopes/CMakeLists.txt new file mode 100644 index 000000000000..cb69f08fc21e --- /dev/null +++ b/components/eamxx/src/physics/specialized_tracers/water_isotopes/CMakeLists.txt @@ -0,0 +1,28 @@ +# Water isotope physics process. +# +# This directory is only entered when EAMXX_ENABLE_WATER_ISOTOPES is ON, +# which in turn guarantees EAMXX_ENABLE_WATER_TRACERS is ON +# (isotopes are layered on top of the water tracer infrastructure). Following +# the EAMxx one-library-per-process pattern, the water isotope process gets its +# own library. + +add_library(water_isotopes + eamxx_water_isotopes_process_interface.cpp +) + +# Public compile definition so the registration guard in register_physics.hpp +# becomes active. +target_compile_definitions(water_isotopes PUBLIC EAMXX_HAS_WATER_ISOTOPES) + +# Expose this directory +target_include_directories(water_isotopes PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR} +) + +target_link_libraries(water_isotopes PUBLIC scream_share) +target_link_libraries(water_isotopes PRIVATE water_tracers) + +# Link into the eamxx_physics INTERFACE target. +if (TARGET eamxx_physics) + target_link_libraries(eamxx_physics INTERFACE water_isotopes) +endif() diff --git a/components/eamxx/src/physics/specialized_tracers/water_isotopes/eamxx_water_isotopes_process_interface.cpp b/components/eamxx/src/physics/specialized_tracers/water_isotopes/eamxx_water_isotopes_process_interface.cpp new file mode 100644 index 000000000000..dffd1a98ac2a --- /dev/null +++ b/components/eamxx/src/physics/specialized_tracers/water_isotopes/eamxx_water_isotopes_process_interface.cpp @@ -0,0 +1,23 @@ +#include "eamxx_water_isotopes_process_interface.hpp" + +namespace scream +{ + +// ========================================================================================= +WaterIsotopes::WaterIsotopes(const ekat::Comm& comm, const ekat::ParameterList& params) + : WaterTracers(comm, params) +{ + // Water isotopes will inherit all tracer handling from WaterTracers + // No additional initialization needed at this stage +} + +// ========================================================================================= +void WaterIsotopes::run_impl(const double dt) +{ + // Call base class tracer physics (currently a no-op) + WaterTracers::run_impl(dt); + + // TODO: Add fractionation physics here +} + +} // namespace scream diff --git a/components/eamxx/src/physics/specialized_tracers/water_isotopes/eamxx_water_isotopes_process_interface.hpp b/components/eamxx/src/physics/specialized_tracers/water_isotopes/eamxx_water_isotopes_process_interface.hpp new file mode 100644 index 000000000000..5eba4a1e0d97 --- /dev/null +++ b/components/eamxx/src/physics/specialized_tracers/water_isotopes/eamxx_water_isotopes_process_interface.hpp @@ -0,0 +1,46 @@ +#ifndef SCREAM_WATER_ISOTOPES_HPP +#define SCREAM_WATER_ISOTOPES_HPP + +#include "physics/specialized_tracers/water_tracers/eamxx_water_tracers_process_interface.hpp" +#include "ekat/ekat_parameter_list.hpp" + +#include + +namespace scream +{ +/* + * The class responsible to handle water isotope transport and fractionation + * + * This process extends WaterTracers to add equilibrium and kinetic fractionation + * during phase changes for water isotope species. + * + * By inheriting from WaterTracers, this class reuses all tracer field handling + * and only needs to override specific fractionation hooks. + * + * Note: This is a stub implementation that registers the process. Fractionation + * physics will be added later. +*/ + +class WaterIsotopes : public WaterTracers +{ +public: + // Constructors + WaterIsotopes (const ekat::Comm& comm, const ekat::ParameterList& params); + + // Override the name to distinguish from base WaterTracers + std::string name () const override { return "water_isotopes"; } + +protected: + + // Override run_impl to add fractionation physics + // For now, just calls base class implementation + void run_impl (const double dt) override; + + // TODO: Add virtual hooks for fractionation processes + // e.g., apply_equilibrium_fractionation(), apply_kinetic_fractionation() + +}; // class WaterIsotopes + +} // namespace scream + +#endif // SCREAM_WATER_ISOTOPES_HPP diff --git a/components/eamxx/src/physics/specialized_tracers/water_tracers/CMakeLists.txt b/components/eamxx/src/physics/specialized_tracers/water_tracers/CMakeLists.txt new file mode 100644 index 000000000000..5607cd5036ae --- /dev/null +++ b/components/eamxx/src/physics/specialized_tracers/water_tracers/CMakeLists.txt @@ -0,0 +1,28 @@ +# Water tracer physics process. +# +# This directory is only entered when EAMXX_ENABLE_WATER_TRACERS is ON. +# Following the EAMxx one-library-per-process pattern, the +# water tracer process gets its own library. The sibling water_isotopes library +# depends on this one. + +add_library(water_tracers + eamxx_water_tracers_process_interface.cpp +) + +# Public compile definition so the registration guard in register_physics.hpp +# becomes active. +target_compile_definitions(water_tracers PUBLIC EAMXX_HAS_WATER_TRACERS) + +# Expose this directory to allow includes of the +# process interface headers. +target_include_directories(water_tracers PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR} +) + +# Link required EAMxx libraries. +target_link_libraries(water_tracers scream_share) + +# Link into the eamxx_physics INTERFACE target. +if (TARGET eamxx_physics) + target_link_libraries(eamxx_physics INTERFACE water_tracers) +endif() diff --git a/components/eamxx/src/physics/specialized_tracers/water_tracers/eamxx_water_tracers_process_interface.cpp b/components/eamxx/src/physics/specialized_tracers/water_tracers/eamxx_water_tracers_process_interface.cpp new file mode 100644 index 000000000000..729ed701edfb --- /dev/null +++ b/components/eamxx/src/physics/specialized_tracers/water_tracers/eamxx_water_tracers_process_interface.cpp @@ -0,0 +1,51 @@ +#include "eamxx_water_tracers_process_interface.hpp" + +#include + +namespace scream +{ + +// ========================================================================================= +WaterTracers::WaterTracers(const ekat::Comm& comm, const ekat::ParameterList& params) + : AtmosphereProcess(comm, params) + , m_tracer_count(0) +{ + // Read tracer count from parameter list (default 0) + m_tracer_count = m_params.get("tracer_count", 0); +} + +// ========================================================================================= +void WaterTracers::create_requests() +{ + // Get the grid for this process + m_grid = m_grids_manager->get_grid("physics"); + m_num_cols = m_grid->get_num_local_dofs(); // Number of columns on this rank + m_num_levs = m_grid->get_num_vertical_levels(); // Number of levels per column + + // TODO: Define field requests for water tracer arrays + // For now, this process requires no fields and computes no fields + // Field definitions will be added later +} + +// ========================================================================================= +void WaterTracers::initialize_impl(const RunType /* run_type */) +{ + // TODO: Initialize tracer field arrays and any precomputed data + // For now, this is a no-op since no fields are defined yet +} + +// ========================================================================================= +void WaterTracers::run_impl(const double /* dt */) +{ + // TODO: Implement tracer transport and physics + // This stub implementation performs no operations + // Tracer physics will be added in subsequent PRs +} + +// ========================================================================================= +void WaterTracers::finalize_impl() +{ + // Nothing to finalize at this stage +} + +} // namespace scream diff --git a/components/eamxx/src/physics/specialized_tracers/water_tracers/eamxx_water_tracers_process_interface.hpp b/components/eamxx/src/physics/specialized_tracers/water_tracers/eamxx_water_tracers_process_interface.hpp new file mode 100644 index 000000000000..6d8e36e9b2fe --- /dev/null +++ b/components/eamxx/src/physics/specialized_tracers/water_tracers/eamxx_water_tracers_process_interface.hpp @@ -0,0 +1,60 @@ +#ifndef SCREAM_WATER_TRACERS_HPP +#define SCREAM_WATER_TRACERS_HPP + +#include "share/atm_process/atmosphere_process.hpp" +#include "ekat/ekat_parameter_list.hpp" + +#include + +namespace scream +{ +/* + * The class responsible to handle water tracer transport through the atmosphere + * + * This process manages additional water species that track through the model + * without undergoing fractionation. Water isotopes (a special case with + * fractionation) are handled by the WaterIsotopes subclass. + * + * Note: This is a stub implementation that registers the process and sets up + * the basic infrastructure. Field definitions and physics implementation are + * deferred to later specs in the water isotope campaign. +*/ + +class WaterTracers : public AtmosphereProcess +{ +public: + using KT = ekat::KokkosTypes; + + // Constructors + WaterTracers (const ekat::Comm& comm, const ekat::ParameterList& params); + + // The type of subcomponent + AtmosphereProcessType type () const { return AtmosphereProcessType::Physics; } + + // The name of the subcomponent + virtual std::string name () const { return "water_tracers"; } + + // Create grid-dependent field requests + void create_requests (); + +protected: + + // The three main overrides for the subcomponent + void initialize_impl (const RunType run_type); + void run_impl (const double dt); + void finalize_impl (); + + // Keep track of field dimensions + int m_num_cols; + int m_num_levs; + + // Number of tracers to track (from parameter list) + int m_tracer_count; + + std::shared_ptr m_grid; + +}; // class WaterTracers + +} // namespace scream + +#endif // SCREAM_WATER_TRACERS_HPP diff --git a/components/eamxx/src/physics/zm/eamxx_zm_process_interface.cpp b/components/eamxx/src/physics/zm/eamxx_zm_process_interface.cpp index 76f96ec1f687..3b76490285f5 100644 --- a/components/eamxx/src/physics/zm/eamxx_zm_process_interface.cpp +++ b/components/eamxx/src/physics/zm/eamxx_zm_process_interface.cpp @@ -78,8 +78,8 @@ void ZMDeepConvection::create_requests () add_field ("precip_ice_surf_mass", scalar2d, kg/m2, grid_name, "ACCUMULATED"); // T/qv from previous time step for DCAPE - add_field("zm_t_prev", scalar3d_mid, K, grid_name); - add_field("zm_q_prev", scalar3d_mid, kg/kg, grid_name); + add_field("zm_t_prev", scalar3d_mid, K, grid_name, pack_size); + add_field("zm_q_prev", scalar3d_mid, kg/kg, grid_name, pack_size); // Diagnostic Outputs add_field("zm_prec", scalar2d, m/s, grid_name); @@ -576,6 +576,10 @@ void ZMDeepConvection::run_impl (const double dt) qi (i,k) += zm_detr_qi(i,k) * dt; nc (i,k) += zm_detr_nc(i,k) * dt; ni (i,k) += zm_detr_ni(i,k) * dt; + // latent heat of fusion released when detrained condensate, which is + // liquid-only in ZM without zm_microp, is partitioned into ice + // (mirrors EAM: ptend%s = dlf * ice_frac * latice in clubb_intr.F90) + T_mid(i,k) += zm_detr_qi(i,k) * PC::LatIce.value / PC::CP.value * dt; } winds_v(i,0,k) += loc_zm_output_tend_out_u (i,k) * dt; winds_v(i,1,k) += loc_zm_output_tend_out_v (i,k) * dt; diff --git a/components/eamxx/src/share/algorithm/eamxx_data_interpolation.cpp b/components/eamxx/src/share/algorithm/eamxx_data_interpolation.cpp index ae60ff3d0bb2..769a39c64b04 100644 --- a/components/eamxx/src/share/algorithm/eamxx_data_interpolation.cpp +++ b/components/eamxx/src/share/algorithm/eamxx_data_interpolation.cpp @@ -5,6 +5,7 @@ #include "share/remap/horizontal_remapper.hpp" #include "share/remap/iop_remapper.hpp" #include "share/grid/point_grid.hpp" +#include "share/data_managers/IOPDataManager.hpp" #include "share/scorpio_interface/eamxx_scorpio_interface.hpp" #include "share/field/field_reader.hpp" #include "share/util/eamxx_universal_constants.hpp" @@ -641,39 +642,54 @@ create_horiz_remappers (const std::string& map_file) } void DataInterpolation:: -create_horiz_remappers (const Real iop_lat, const Real iop_lon) +create_horiz_remappers (const std::string& map_file, + const std::shared_ptr& iop_data_manager) { - using namespace ShortFieldTagsNames; + // IOP cases cannot have a remap file. We will create a IOPRemapper as the horiz remapper + if (iop_data_manager!=nullptr) { + EKAT_REQUIRE_MSG(map_file == "" || map_file=="none", + "[DataInterpolation] Error! Cannot define map_file for cases with an Intensive Observation Period defined. " + "The IOP class defines its own remap from file data to model data.\n"); - EKAT_REQUIRE_MSG (m_horiz_remapper_beg==nullptr, - "[DataInterpolation] Error! Horizontal remappers were already setup.\n"); + // TODO: expose tgt lat/lon in IOPDataManager, to avoid injecting knowledge + // of its parameter list structure in other places + Real iop_lat = iop_data_manager->get_params().get("target_latitude"); + Real iop_lon = iop_data_manager->get_params().get("target_longitude"); - EKAT_REQUIRE_MSG (not std::isnan(iop_lat) and not std::isnan(iop_lon), - "[DataInterpolation] Error! At least one between iop_lat and iop_lon appears to be invalid.\n" - " - iop_lat: " << iop_lat << "\n" - " - iop_lon: " << iop_lon << "\n"); - - int ncols_model = m_model_grid->get_num_global_dofs(); - int nlevs_model = m_model_grid->get_num_vertical_levels(); - - // Create hremap tgt grid - int nlevs_data = m_fields_have_lev_dim ? get_input_files_dimlen (m_input_files_dimnames[e2str(LEV)]) : nlevs_model; - int ncols_data = m_fields_have_col_dim ? get_input_files_dimlen (m_input_files_dimnames[e2str(COL)]) : ncols_model; - - // Create grid for IO and load lat/lon field in IO grid from any data file - m_data_grid = create_point_grid(m_name+"_data",ncols_data,nlevs_data,m_model_grid->get_comm()); - auto lat = m_data_grid->create_geometry_data("lat",m_data_grid->get_2d_scalar_layout()); - auto lon = m_data_grid->create_geometry_data("lon",m_data_grid->get_2d_scalar_layout()); - auto gids = m_data_grid->get_partitioned_dim_gids(); - auto comm = m_data_grid->get_comm(); - read_fields(m_time_database.files.front(),{lat,lon},gids,comm); + using namespace ShortFieldTagsNames; - // Create iop remap tgt grid - m_grid_after_hremap = m_model_grid->clone(m_name+"_post_hremap",true); - m_grid_after_hremap->reset_vertical_configuration(nlevs_data, AbstractGrid::VKind::Model); - m_horiz_remapper_beg = std::make_shared(m_data_grid,m_grid_after_hremap,iop_lat,iop_lon); - if (m_time_dependent) { - m_horiz_remapper_end = std::make_shared(m_data_grid,m_grid_after_hremap,iop_lat,iop_lon); + EKAT_REQUIRE_MSG (m_horiz_remapper_beg==nullptr, + "[DataInterpolation] Error! Horizontal remappers were already setup.\n"); + + EKAT_REQUIRE_MSG (not std::isnan(iop_lat) and not std::isnan(iop_lon), + "[DataInterpolation] Error! At least one between iop_lat and iop_lon appears to be invalid.\n" + " - iop_lat: " << iop_lat << "\n" + " - iop_lon: " << iop_lon << "\n"); + + int ncols_model = m_model_grid->get_num_global_dofs(); + int nlevs_model = m_model_grid->get_num_vertical_levels(); + + // Create hremap tgt grid + int nlevs_data = m_fields_have_lev_dim ? get_input_files_dimlen (m_input_files_dimnames[e2str(LEV)]) : nlevs_model; + int ncols_data = m_fields_have_col_dim ? get_input_files_dimlen (m_input_files_dimnames[e2str(COL)]) : ncols_model; + + // Create grid for IO and load lat/lon field in IO grid from any data file + m_data_grid = create_point_grid(m_name+"_data",ncols_data,nlevs_data,m_model_grid->get_comm()); + auto lat = m_data_grid->create_geometry_data("lat",m_data_grid->get_2d_scalar_layout()); + auto lon = m_data_grid->create_geometry_data("lon",m_data_grid->get_2d_scalar_layout()); + auto gids = m_data_grid->get_partitioned_dim_gids(); + auto comm = m_data_grid->get_comm(); + read_fields(m_time_database.files.front(),{lat,lon},gids,comm); + + // Create iop remap tgt grid + m_grid_after_hremap = m_model_grid->clone(m_name+"_post_hremap",true); + m_grid_after_hremap->reset_vertical_configuration(nlevs_data, AbstractGrid::VKind::Model); + m_horiz_remapper_beg = std::make_shared(m_data_grid,m_grid_after_hremap,iop_lat,iop_lon); + if (m_time_dependent) { + m_horiz_remapper_end = std::make_shared(m_data_grid,m_grid_after_hremap,iop_lat,iop_lon); + } + } else { + create_horiz_remappers(map_file=="none" ? "" : map_file); } } diff --git a/components/eamxx/src/share/algorithm/eamxx_data_interpolation.hpp b/components/eamxx/src/share/algorithm/eamxx_data_interpolation.hpp index ab3ae7a1f498..319f2b054725 100644 --- a/components/eamxx/src/share/algorithm/eamxx_data_interpolation.hpp +++ b/components/eamxx/src/share/algorithm/eamxx_data_interpolation.hpp @@ -12,6 +12,9 @@ namespace scream { +// Forward declaration +class IOPDataManager; + class DataInterpolation { public: @@ -74,7 +77,8 @@ class DataInterpolation int time_index = -1); void create_horiz_remappers (const std::string& map_file = ""); - void create_horiz_remappers (const Real iop_lat, const Real iop_lon); + void create_horiz_remappers (const std::string& map_file, + const std::shared_ptr& iop_data_manager); void create_vert_remapper (); void create_vert_remapper (const VertRemapData& data); diff --git a/components/eamxx/src/share/io/eamxx_output_manager.cpp b/components/eamxx/src/share/io/eamxx_output_manager.cpp index d93bf8a836d5..02f194291c93 100644 --- a/components/eamxx/src/share/io/eamxx_output_manager.cpp +++ b/components/eamxx/src/share/io/eamxx_output_manager.cpp @@ -928,7 +928,9 @@ setup_file ( IOFileSpecs& filespecs, filespecs.is_open = true; if (filespecs.storage.type!=NumSnaps) { - filespecs.storage.set_time_idx(control.next_write_ts); + // We want the next write timestamp for instant, but the last write timestamp for average/min/max, + // since the latter is the start of the averaging window, and its control hasn't progressed yet + filespecs.storage.set_time_idx(m_avg_type==OutputAvgType::Instant ? control.next_write_ts : control.last_write_ts); } m_resume_output_file = false; diff --git a/components/eamxx/src/share/io/tests/io_monthly.cpp b/components/eamxx/src/share/io/tests/io_monthly.cpp index d54be13135e9..61edc66c3e5c 100644 --- a/components/eamxx/src/share/io/tests/io_monthly.cpp +++ b/components/eamxx/src/share/io/tests/io_monthly.cpp @@ -195,6 +195,107 @@ void read (const int seed, const ekat::Comm& comm) } } +// Write 12 months of data using the given averaging type with monthly storage. +// This tests that the timestamp index in the file is taken from last_write_ts +// (start of the averaging window) rather than next_write_ts (end of window) +// for non-Instant averaging types. +void write_avg_type (const std::string& avg_type, const int seed, const ekat::Comm& comm) +{ + auto gm = get_gm(comm); + auto grid = gm->get_grid("point_grid"); + + auto t0 = get_t0(); + const int dt = 86400*30; // 30 days + + auto fm = get_fm(grid,t0,seed); + std::vector fnames; + for (auto it : fm->get_repo()) { + fnames.push_back(it.second->name()); + } + + ekat::ParameterList om_pl; + om_pl.set("filename_prefix",std::string("io_monthly_avg")); + om_pl.set("field_names",fnames); + om_pl.set("averaging_type", avg_type); + om_pl.set("file_max_storage_type",std::string("one_month")); + om_pl.set("floating_point_precision",std::string("single")); + auto& ctrl_pl = om_pl.sublist("output_control"); + ctrl_pl.set("frequency_units",std::string("nsteps")); + ctrl_pl.set("frequency",1); + ctrl_pl.set("save_grid_data",false); + + OutputManager om; + om.initialize(comm,om_pl,t0,false); + om.setup(fm,gm->get_grid_names()); + + // Run 12 steps: one per month + const int nsteps = 12; + auto t = t0; + for (int n=0; nget_field(name); + add(f,1); + } + om.run(t); + } + om.finalize(); +} + +// Verify that for AVERAGE/MIN/MAX averaging with monthly storage: +// - 12 separate files were created (one per month) +// - each file contains exactly 1 time snapshot +// - each file contains the correct averaged/min/max data +void read_avg_type (const std::string& avg_type, const int seed, const ekat::Comm& comm) +{ + auto t0 = get_t0(); + int dt = 86400*30; + + auto gm = get_gm(comm); + auto grid = gm->get_grid("point_grid"); + auto gids = grid->get_partitioned_dim_gids(); + + auto fm0 = get_fm(grid,t0,seed); + std::vector fields; + for (auto it : fm0->get_repo()) { + fields.push_back(it.second->clone()); + } + + // For non-Instant averaging, the filename is derived from last_write_ts + // (the start of the averaging window). With freq=1 nstep, the n-th window + // starts at t0 + n*dt and ends at t0 + (n+1)*dt. + // So the n-th output file is named using the timestamp t0 + n*dt. + std::string casename = "io_monthly_avg"; + auto get_filename = [&](const util::TimeStamp& t) { + auto t_str = t.to_string().substr(0,7); // YYYY-MM + std::string fname = casename + + "." + avg_type + ".nsteps_x1" + + ".np" + std::to_string(comm.size()) + + "." + t_str + + ".nc"; + return fname; + }; + + for (int n=0; n<12; ++n) { + auto window_start = t0 + n*dt; + auto filename = get_filename(window_start); + + // Each file must hold exactly one snapshot (one per month). + REQUIRE(scorpio::get_dimlen(filename,"time")==1); + + read_fields(filename,fields,gids,comm); + + // With 1 sample per averaging window, AVERAGE=MIN=MAX equals the single sample. + // At window n (0-indexed), the field was incremented n+1 times before the write. + for (const auto& f : fields) { + auto f0 = fm0->get_field(f.name()).clone(CloneFlags::CopyData); + add(f0,n+1); + REQUIRE(views_are_equal(f,f0)); + } + } +} + TEST_CASE ("io_monthly") { ekat::Comm comm(MPI_COMM_WORLD); scorpio::init_subsystem(comm); @@ -202,13 +303,30 @@ TEST_CASE ("io_monthly") { auto seed = get_random_test_seed(&comm); if (comm.am_i_root()) { - std::cout << " -> Testing output with one file per month ...\n"; + std::cout << " -> Testing monthly output with INSTANT averaging ...\n"; } write(seed,comm); read (seed,comm); if (comm.am_i_root()) { - std::cout << " -> Testing output with one file per month ... PASS\n"; + std::cout << " -> Testing monthly output with INSTANT averaging ... PASS\n"; } + + // Test that for AVERAGE/MIN/MAX averaging with monthly storage, each month's + // output goes into a separate correctly-named file (one snapshot per file). + // This tests the fix for the timestamp handling bug in setup_file, where + // set_time_idx now uses last_write_ts instead of next_write_ts for non-Instant + // averaging types. + for (const auto& avg_type : {"AVERAGE","MIN","MAX"}) { + if (comm.am_i_root()) { + std::cout << " -> Testing monthly output with " << avg_type << " averaging ...\n"; + } + write_avg_type(avg_type,seed,comm); + read_avg_type (avg_type,seed,comm); + if (comm.am_i_root()) { + std::cout << " -> Testing monthly output with " << avg_type << " averaging ... PASS\n"; + } + } + scorpio::finalize_subsystem(); } diff --git a/components/eamxx/src/share/physics/tests/CMakeLists.txt b/components/eamxx/src/share/physics/tests/CMakeLists.txt index d40aa6ec291e..d18c7bc52ecd 100644 --- a/components/eamxx/src/share/physics/tests/CMakeLists.txt +++ b/components/eamxx/src/share/physics/tests/CMakeLists.txt @@ -17,6 +17,7 @@ endif() CreateUnitTest(physics_saturation_run_and_cmp SOURCES physics_saturation_run_and_cmp.cpp + EXCLUDE_MAIN_CPP LIBS eamxx_physics_share EXE_ARGS "${SCREAM_BASELINE_FILE_ARG}" THREADS ${SCREAM_BASELINE_TEST_THREADS} diff --git a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml index b3df701e6f56..ef66493941d8 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml @@ -14,15 +14,15 @@ eamxx: mam4_srf_online_emiss: # MAM4xx-Surface-Emissions srf_remap_file: "" - srf_emis_specifier_for_dms: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc - srf_emis_specifier_for_so2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc - srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_dms: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20260730.nc + srf_emis_specifier_for_so2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20260730.nc + srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20260730.nc + srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20260730.nc + srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20260730.nc + srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20260730.nc + srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20260730.nc + srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20260730.nc + srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20260730.nc srf_emis_scale_factor_for_dust: 1.5 soil_erodibility_file: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/dst_ne2np4_c20241028.nc diff --git a/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt index 65f377b4db1f..0c45342a4b69 100644 --- a/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt @@ -27,15 +27,15 @@ GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) # Ensure test input files are present in the data dir set (TEST_INPUT_FILES - scream/mam4xx/emissions/ne2np4/surface/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc - scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc - scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20260730.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20260730.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20260730.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20260730.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20260730.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20260730.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20260730.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20260730.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20260730.nc scream/mam4xx/emissions/ne2np4/dst_ne2np4_c20241028.nc scream/mam4xx/emissions/ne2np4/monthly_macromolecules_0.1deg_bilinear_year01_merge_ne2np4_c20241030.nc ) diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml index 9c2abc300bea..5f76796b9e4f 100644 --- a/components/eamxx/tests/single-process/mam/emissions/input.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -14,15 +14,15 @@ eamxx: # MAM4xx-Surface-Emissions create_fields_interval_checks: true srf_remap_file: "" - srf_emis_specifier_for_dms: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc - srf_emis_specifier_for_so2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc - srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_dms: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20260730.nc + srf_emis_specifier_for_so2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20260730.nc + srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20260730.nc + srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20260730.nc + srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20260730.nc + srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20260730.nc + srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20260730.nc + srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20260730.nc + srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20260730.nc srf_emis_scale_factor_for_dust: 1.5 soil_erodibility_file: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/dst_ne2np4_c20241028.nc diff --git a/components/elm/src/biogeochem/FireMod.F90 b/components/elm/src/biogeochem/FireMod.F90 index adfffc4e7d40..57881d444344 100644 --- a/components/elm/src/biogeochem/FireMod.F90 +++ b/components/elm/src/biogeochem/FireMod.F90 @@ -66,7 +66,7 @@ module FireMod ! !PRIVATE MEMBER DATA: real(r8), pointer :: forc_lnfm(:) ! Lightning frequency - real(r8), pointer :: forc_hdm(:) ! Human population density + real(r8), public, pointer :: forc_hdm(:) ! Human population density !$acc declare create(forc_lnfm) !$acc declare create(forc_hdm ) real(r8), parameter :: secsphr = 3600._r8 ! Seconds in an hour diff --git a/components/elm/src/biogeophys/SoilStateType.F90 b/components/elm/src/biogeophys/SoilStateType.F90 index ff4a100141ed..6e8604f95709 100644 --- a/components/elm/src/biogeophys/SoilStateType.F90 +++ b/components/elm/src/biogeophys/SoilStateType.F90 @@ -6,7 +6,7 @@ module SoilStateType use shr_log_mod , only : errMsg => shr_log_errMsg use decompMod , only : bounds_type use abortutils , only : endrun - use spmdMod , only : mpicom, MPI_INTEGER, masterproc + use spmdMod , only : mpicom, MPI_INTEGER, masterproc, iam use ncdio_pio , only : file_desc_t, ncd_defvar, ncd_io, ncd_double, ncd_int, ncd_inqvdlen use ncdio_pio , only : ncd_pio_openfile, ncd_inqfdims, ncd_pio_closefile, ncd_inqdid, ncd_inqdlen use elm_varpar , only : more_vertlayers, numpft, numrad @@ -117,6 +117,10 @@ subroutine Init(this, bounds) call this%InitHistory(bounds) call this%InitCold(bounds) +#ifdef MOAB_LATERAL + call this%InitColdGhost(bounds) +#endif + end subroutine Init !------------------------------------------------------------------------ @@ -163,7 +167,7 @@ subroutine InitAllocate(this, bounds) allocate(this%watopt_col (begc:endc,nlevgrnd)) ; this%watopt_col (:,:) = spval allocate(this%watfc_col (begc:endc,nlevgrnd)) ; this%watfc_col (:,:) = spval allocate(this%watmin_col (begc:endc,nlevgrnd)) ; this%watmin_col (:,:) = spval - allocate(this%sucsat_col (begc:endc,nlevgrnd)) ; this%sucsat_col (:,:) = spval + allocate(this%sucsat_col (begc_all:endc_all,nlevgrnd)) ; this%sucsat_col (:,:) = spval allocate(this%sucmin_col (begc:endc,nlevgrnd)) ; this%sucmin_col (:,:) = spval allocate(this%soilbeta_col (begc:endc)) ; this%soilbeta_col (:) = spval allocate(this%soilalpha_col (begc:endc)) ; this%soilalpha_col (:) = spval @@ -1080,6 +1084,155 @@ subroutine InitColdGhost(this, bounds_proc) end subroutine InitColdGhost +#else + +#ifdef MOAB_LATERAL + + !------------------------------------------------------------------------ + subroutine PackOwnedGridLevelDataForMOAB(bounds_proc, col_itype, data_c_in, data_g_out) + ! + implicit none + ! + type(bounds_type) , intent(in) :: bounds_proc + integer , intent(in) :: col_itype + real(r8), pointer , intent(in) :: data_c_in(:,:) + real(r8), pointer , intent(inout) :: data_g_out(:,:) + ! + integer :: c, g, j + integer :: ncols_per_gcell(bounds_proc%begg:bounds_proc%endg) + + data_g_out(:,:) = 0._r8 + ncols_per_gcell(:) = 0 + + do c = bounds_proc%begc, bounds_proc%endc + if (col_pp%itype(c) == col_itype) then + g = col_pp%gridcell(c) + ncols_per_gcell(g) = ncols_per_gcell(g) + 1 + if (ncols_per_gcell(g) > 1) then + call endrun('PackOwnedGridLevelDataForMOAB: more than one matching '// & + 'column per grid cell; one-nat-veg-column invariant violated.') + end if + do j = 1, nlevgrnd + data_g_out(g, j) = data_c_in(c, j) + end do + end if + end do + + end subroutine PackOwnedGridLevelDataForMOAB + + !------------------------------------------------------------------------ + subroutine UnpackGhostGridLevelDataFromMOAB(bounds_proc, col_itype, data_g_in, data_c_out) + ! + implicit none + ! + type(bounds_type) , intent(in) :: bounds_proc + integer , intent(in) :: col_itype + real(r8) , pointer, intent(in) :: data_g_in(:,:) + real(r8) , pointer, intent(inout) :: data_c_out(:,:) + ! + integer :: c, g, j + + do c = bounds_proc%endc + 1, bounds_proc%endc_all + if (col_pp%itype(c) == col_itype) then + g = col_pp%gridcell(c) + do j = 1, nlevgrnd + data_c_out(c, j) = data_g_in(g, j) + end do + end if + end do + + end subroutine UnpackGhostGridLevelDataFromMOAB + + !------------------------------------------------------------------------ + subroutine BatchExchangeFieldsUsingMOAB(bounds_proc, col_itype, watsat, hksat, bsw, sucsat) + ! + ! Pack all four fields into a single grid-level buffer, perform one MPI + ! round via GridLevelRealDataHaloExchange, then unpack. + ! Field layout: field f (1..4), soil layer j (1..nlevgrnd) → + ! component index (f-1)*nlevgrnd + j. + ! + use domainLateralMod , only : GridLevelRealDataHaloExchange + use domainLateralMod , only : setup_twoD_real_data_for_moab, twoD_real_data_for_moab + use MOABGridType , only : moab_gcell, mlndghostid + ! + implicit none + ! + ! !ARGUMENTS: + type(bounds_type) , intent(in) :: bounds_proc + integer , intent(in) :: col_itype + real(r8), pointer , intent(inout) :: watsat(:,:) + real(r8), pointer , intent(inout) :: hksat(:,:) + real(r8), pointer , intent(inout) :: bsw(:,:) + real(r8), pointer , intent(inout) :: sucsat(:,:) + ! + integer, parameter :: nfields = 4 + real(r8), pointer :: data(:,:) ! (begg:endg_all, nfields*nlevgrnd) + type(twoD_real_data_for_moab) :: data_moab + integer :: c, g, j + + ! allocate grid-level buffer for all fields + allocate(data(bounds_proc%begg:bounds_proc%endg_all, nfields*nlevgrnd)) + data(:,:) = 0._r8 + + ! --- pack owned columns --- + do c = bounds_proc%begc, bounds_proc%endc + if (col_pp%itype(c) == col_itype) then + g = col_pp%gridcell(c) + do j = 1, nlevgrnd + data(g, 0*nlevgrnd + j) = watsat(c, j) + data(g, 1*nlevgrnd + j) = hksat(c, j) + data(g, 2*nlevgrnd + j) = bsw(c, j) + data(g, 3*nlevgrnd + j) = sucsat(c, j) + end do + end if + end do + + ! --- single MPI halo exchange --- + call setup_twoD_real_data_for_moab(mlndghostid, 'batch_soil_data', nfields*nlevgrnd, & + moab_gcell%num_ghosted, data_moab) + call GridLevelRealDataHaloExchange(data_moab, bounds_proc%begg, bounds_proc%endg, & + bounds_proc%endg_all, data) + + ! --- unpack ghost columns --- + do c = bounds_proc%endc + 1, bounds_proc%endc_all + if (col_pp%itype(c) == col_itype) then + g = col_pp%gridcell(c) + do j = 1, nlevgrnd + watsat(c, j) = data(g, 0*nlevgrnd + j) + hksat(c, j) = data(g, 1*nlevgrnd + j) + bsw(c, j) = data(g, 2*nlevgrnd + j) + sucsat(c, j) = data(g, 3*nlevgrnd + j) + end do + end if + end do + + ! free memory + deallocate(data_moab%values) + deallocate(data) + + end subroutine BatchExchangeFieldsUsingMOAB + + !------------------------------------------------------------------------ + subroutine InitColdGhost(this, bounds_proc) + ! + ! !DESCRIPTION: + ! Assign soil properties for ghost/halo columns + ! + ! !USES: + ! + implicit none + ! + ! !ARGUMENTS: + class(soilstate_type) :: this + type(bounds_type), intent(in) :: bounds_proc + ! + integer, parameter :: nat_veg_col_itype = 1 + + call BatchExchangeFieldsUsingMOAB(bounds_proc, nat_veg_col_itype, & + this%watsat_col, this%hksat_col, this%bsw_col, this%sucsat_col) + + end subroutine InitColdGhost + #else !------------------------------------------------------------------------ @@ -1102,6 +1255,7 @@ subroutine InitColdGhost(this, bounds_proc) end subroutine InitColdGhost +#endif #endif !------------------------------------------------------------------------ diff --git a/components/elm/src/cpl/elm_cpl_indices.F90 b/components/elm/src/cpl/elm_cpl_indices.F90 index 0347469e396f..85c753f2f374 100644 --- a/components/elm/src/cpl/elm_cpl_indices.F90 +++ b/components/elm/src/cpl/elm_cpl_indices.F90 @@ -81,6 +81,9 @@ module elm_cpl_indices integer, public ::index_l2x_Sl_hr(0:iac_npft_max) = 0 integer, public ::index_l2x_Sl_npp(0:iac_npft_max) = 0 integer, public ::index_l2x_Sl_pftwgt(0:iac_npft_max) = 0 + integer, public ::index_l2x_Sl_hdd = 0 ! lnd->iac heating degree days + integer, public ::index_l2x_Sl_cdd = 0 ! lnd->iac cooling degree days + integer, public ::index_l2x_Sl_forc_hdm = 0 ! human population density (per-gridcell) ! drv -> lnd (required) @@ -391,6 +394,11 @@ subroutine elm_cpl_indices_set( ) index_x2l_Sz_harvest_frac(p) = mct_avect_indexra(x2l,trim(name)) end if enddo + ! Scalar per-gridcell field + index_l2x_Sl_forc_hdm = mct_avect_indexra(l2x, 'Sl_forc_hdm') + index_l2x_Sl_hdd = mct_avect_indexra(l2x,'Sl_hdd') + index_l2x_Sl_cdd = mct_avect_indexra(l2x,'Sl_cdd') + endif call mct_aVect_clean(x2l) diff --git a/components/elm/src/cpl/lnd_import_export.F90 b/components/elm/src/cpl/lnd_import_export.F90 index dc8bea16c2b6..bd0c02da3c51 100644 --- a/components/elm/src/cpl/lnd_import_export.F90 +++ b/components/elm/src/cpl/lnd_import_export.F90 @@ -1417,6 +1417,7 @@ subroutine lnd_export( bounds, lnd2atm_vars, lnd2glc_vars, lnd2iac_vars, l2x) use domainMod , only : ldomain use seq_drydep_mod , only : n_drydep use shr_megan_mod , only : shr_megan_mechcomps_n + use FireMod , only : forc_hdm ! ! !ARGUMENTS: implicit none @@ -1424,7 +1425,6 @@ subroutine lnd_export( bounds, lnd2atm_vars, lnd2glc_vars, lnd2iac_vars, l2x) type(lnd2atm_type), intent(inout) :: lnd2atm_vars ! elm land to atmosphere exchange data type type(lnd2glc_type), intent(inout) :: lnd2glc_vars ! elm land to atmosphere exchange data type type(lnd2iac_type), intent(inout) :: lnd2iac_vars ! elm lnd to gcam exchange vars - real(r8) , intent(out) :: l2x(:,:)! land to coupler export state on land grid ! ! !LOCAL VARIABLES: @@ -1546,10 +1546,18 @@ subroutine lnd_export( bounds, lnd2atm_vars, lnd2glc_vars, lnd2iac_vars, l2x) if (iac_present) then do p = 0,numpft - l2x(index_l2x_Sl_hr(p),i) = lnd2iac_vars%hr(g,p) - l2x(index_l2x_Sl_npp(p),i) = lnd2iac_vars%npp(g,p) - l2x(index_l2x_Sl_pftwgt(p),i) = lnd2iac_vars%pftwgt(g,p) + l2x(index_l2x_Sl_hr(p),i) = lnd2iac_vars%hr(g,p) + l2x(index_l2x_Sl_npp(p),i) = lnd2iac_vars%npp(g,p) + l2x(index_l2x_Sl_pftwgt(p),i) = lnd2iac_vars%pftwgt(g,p) end do + ! Scalar per-gridcell fields + ! forc_hdm is time-interpolated from stream file (FireMod in non-CPL_BYPASS, + ! or mirrored from atm2lnd_vars in CPL_BYPASS during lnd_import); + ! copy here into lnd2iac_vars and pack into the coupler vector. + lnd2iac_vars%forc_hdm(g) = forc_hdm(g) + l2x(index_l2x_Sl_forc_hdm, i) = lnd2iac_vars%forc_hdm(g) + l2x(index_l2x_Sl_hdd,i) = lnd2iac_vars%hdd(g) + l2x(index_l2x_Sl_cdd,i) = lnd2iac_vars%cdd(g) end if end do diff --git a/components/elm/src/data_types/ColumnConnectionSetType.F90 b/components/elm/src/data_types/ColumnConnectionSetType.F90 new file mode 100644 index 000000000000..e78559910539 --- /dev/null +++ b/components/elm/src/data_types/ColumnConnectionSetType.F90 @@ -0,0 +1,137 @@ +module ColumnConnectionSetType + + + use shr_kind_mod , only : r8 => shr_kind_r8 + use shr_infnan_mod , only : isnan => shr_infnan_isnan, nan => shr_infnan_nan, assignment(=) + use decompMod , only : bounds_type + use abortutils , only : endrun + use ColumnType , only : col_pp + implicit none + save + public + + type, public :: col_connection_set_type + Integer :: nconn ! number of connections + Integer, pointer :: col_id_up(:) => null() ! list of ids of upwind cells + Integer, pointer :: col_id_dn(:) => null() ! list of ids of downwind cells + Integer, pointer :: grid_id_up(:) => null() ! list of ids of upwind cells + Integer, pointer :: grid_id_dn(:) => null() ! list of ids of downwind cells + integer, pointer :: grid_id_up_norder(:) => null() ! list of ids of upwind cells in natural order + integer, pointer :: grid_id_dn_norder(:) => null() ! list of ids of downwind cells in natural order + integer, pointer :: col_up_forder(:) => null() ! the order in which the lateral flux should be added for upwind cells + integer, pointer :: col_dn_forder(:) => null() ! the order in which the lateral flux should be added for downwind cells + Real(r8), pointer :: dist(:) => null() ! list of distance vectors + Real(r8), pointer :: face_length(:) => null() ! list of edge of faces normal to distance vectors + Real(r8), pointer :: uparea(:) => null() ! list of up cell areas of horizaontal faces + Real(r8), pointer :: downarea(:) => null() ! list of down cell areas of horizaontal faces + Real(r8), pointer :: dzg(:) => null() ! list of areas of dz between downwind and upwind cells + Real(r8), pointer :: facecos(:) => null() ! dot product of the cell face normal vector and cell centroid vector + Real(r8), pointer :: vertcos(:) => null() ! dot product of the cell face normal vector and cell centroid vector for vertical flux, the rank for vertcos + ! is from 1 to column size which is different from rank of lateral faces + contains +#ifdef MOAB_LATERAL + procedure, public :: Init => InitViaMOAB +#endif + end type col_connection_set_type + + type (col_connection_set_type), public, target :: c2c_connections ! connection type + +contains + +#ifdef MOAB_LATERAL + !------------------------------------------------------------------------ + subroutine InitViaMOAB(this, bounds_proc) + ! + use MOABGridType, only : moab_edge_internal, moab_gcell + use decompMod , only : bounds_type + ! + implicit none + ! + class (col_connection_set_type) :: this + type(bounds_type), intent(in) :: bounds_proc ! bound information at processor level + ! + integer :: g, g_up_moab, g_dn_moab, g_up_elm, g_dn_elm + integer :: c, c_up, c_dn + integer :: iconn, nconn + integer, parameter :: nat_veg_col_itype = 1 + integer, pointer :: nat_col_id(:) + + + ! allocate memory and initialize + allocate(nat_col_id(bounds_proc%begg_all:bounds_proc%endg_all)) + nat_col_id(:) = -1 + + ! loop over columns to determine the naturally-vegetated column for each grid cell. + do c = bounds_proc%begc_all, bounds_proc%endc_all + if (col_pp%itype(c) == nat_veg_col_itype) then + g = col_pp%gridcell(c) + + if (nat_col_id(g) /= -1) then + call endrun('ERROR: More than one naturally vegetated column found.') + end if + + nat_col_id(g) = c + end if + end do + + ! loop over grid level connections and determine number of column level connections + nconn = 0 + do iconn = 1, moab_edge_internal%num + g_up_moab = moab_edge_internal%cell_ids(iconn, 1) + g_dn_moab = moab_edge_internal%cell_ids(iconn, 2) + + g_up_elm = moab_gcell%moab2elm(g_up_moab) + g_dn_elm = moab_gcell%moab2elm(g_dn_moab) + + if (nat_col_id(g_up_elm) /= -1 .and. nat_col_id(g_dn_elm) /= -1) then + nconn = nconn + 1 + end if + end do + + ! allocate and initialize data structure + this%nconn = nconn + allocate(this%col_id_up(nconn)) ; this%col_id_up(:) = 0 + allocate(this%col_id_dn(nconn)) ; this%col_id_dn(:) = 0 + allocate(this%grid_id_up(nconn)) ; this%grid_id_up(:) = 0 + allocate(this%grid_id_dn(nconn)) ; this%grid_id_dn(:) = 0 + allocate(this%grid_id_up_norder(nconn)) ; this%grid_id_up_norder(:) = 0 + allocate(this%grid_id_dn_norder(nconn)) ; this%grid_id_dn_norder(:) = 0 + allocate(this%col_up_forder(nconn)) ; this%col_up_forder(:) = 0 + allocate(this%col_dn_forder(nconn)) ; this%col_dn_forder(:) = 0 + allocate(this%face_length(nconn)) ; this%face_length(:) = 0 + allocate(this%uparea(nconn)) ; this%uparea(:) = 0 + allocate(this%downarea(nconn)) ; this%downarea(:) = 0 + allocate(this%dist(nconn)) ; this%dist(:) = 0 + allocate(this%dzg(nconn)) ; this%dzg(:) = 0 + allocate(this%facecos(nconn)) ; this%facecos(:) = 0 + + nconn = 0 + do iconn = 1, moab_edge_internal%num + g_up_moab = moab_edge_internal%cell_ids(iconn, 1) + g_dn_moab = moab_edge_internal%cell_ids(iconn, 2) + + g_up_elm = moab_gcell%moab2elm(g_up_moab) + g_dn_elm = moab_gcell%moab2elm(g_dn_moab) + + if (nat_col_id(g_up_elm) /= -1 .and. nat_col_id(g_dn_elm) /= -1) then + nconn = nconn + 1 + + this%col_id_up(nconn) = nat_col_id(g_up_elm) + this%col_id_dn(nconn) = nat_col_id(g_dn_elm) + + this%grid_id_up(nconn) = g_up_elm + this%grid_id_dn(nconn) = g_dn_elm + + this%grid_id_up_norder(nconn) = moab_gcell%natural_id(g_up_moab) + this%grid_id_dn_norder(nconn) = moab_gcell%natural_id(g_dn_moab) + end if + end do + + ! free up memory + deallocate(nat_col_id) + + end subroutine InitViaMOAB +#endif + +end module ColumnConnectionSetType + diff --git a/components/elm/src/data_types/ColumnDataType.F90 b/components/elm/src/data_types/ColumnDataType.F90 index 85e1bb7714cb..a46627ffe599 100644 --- a/components/elm/src/data_types/ColumnDataType.F90 +++ b/components/elm/src/data_types/ColumnDataType.F90 @@ -1101,7 +1101,7 @@ module ColumnDataType !------------------------------------------------------------------------ ! Subroutines to initialize and clean column energy state data structure !------------------------------------------------------------------------ - subroutine col_es_init(this, begc, endc) + subroutine col_es_init(this, begc, endc, endc_owned) ! ! !USES: use landunit_varcon, only : istice, istwet, istsoil, istdlak, istice_mec @@ -1111,7 +1111,7 @@ subroutine col_es_init(this, begc, endc) ! ! !ARGUMENTS: class(column_energy_state) :: this - integer, intent(in) :: begc,endc + integer, intent(in) :: begc,endc, endc_owned !------------------------------------------------------------------------ ! ! !LOCAL VARIABLES: @@ -1227,7 +1227,7 @@ subroutine col_es_init(this, begc, endc) !----------------------------------------------------------------------- ! Initialize soil+snow temperatures - do c = begc,endc + do c = begc,endc_owned l = col_pp%landunit(c) ! Snow level temperatures - all land points @@ -1385,12 +1385,12 @@ end subroutine col_es_clean !------------------------------------------------------------------------ ! Subroutines to initialize and clean column water state data structure !------------------------------------------------------------------------ - subroutine col_ws_init(this, begc, endc, h2osno_input, snow_depth_input, watsat_input) + subroutine col_ws_init(this, begc, endc, endc_owned, h2osno_input, snow_depth_input, watsat_input) ! use elm_varctl , only : use_lake_wat_storage, use_arctic_init ! !ARGUMENTS: class(column_water_state) :: this - integer , intent(in) :: begc,endc + integer , intent(in) :: begc,endc, endc_owned real(r8), intent(in) :: h2osno_input(begc:) real(r8), intent(in) :: snow_depth_input(begc:) real(r8), intent(in) :: watsat_input(begc:, 1:) ! volumetric soil water at saturation (porosity) @@ -1680,7 +1680,7 @@ subroutine col_ws_init(this, begc, endc, h2osno_input, snow_depth_input, watsat_ ! Arrays that are initialized from input arguments this%wslake_col(begc:endc) = 0._r8 - do c = begc,endc + do c = begc,endc_owned l = col_pp%landunit(c) this%h2osno(c) = h2osno_input(c) this%int_snow(c) = h2osno_input(c) @@ -5774,11 +5774,11 @@ end subroutine col_ef_clean !------------------------------------------------------------------------ ! Subroutines to initialize and clean column water flux data structure !------------------------------------------------------------------------ - subroutine col_wf_init(this, begc, endc) + subroutine col_wf_init(this, begc, endc, endc_owned) ! ! !ARGUMENTS: class(column_water_flux) :: this - integer, intent(in) :: begc,endc + integer, intent(in) :: begc,endc, endc_owned ! !LOCAL VARIABLES: integer :: l,c integer :: ncells @@ -6064,7 +6064,7 @@ subroutine col_wf_init(this, begc, endc) this%qflx_to_downhill(begc:endc) = 0._r8 ! needed for CNNLeaching - do c = begc, endc + do c = begc, endc_owned l = col_pp%landunit(c) if (col_pp%is_soil(c) .or. col_pp%is_crop(c)) then this%qflx_drain(c) = 0._r8 diff --git a/components/elm/src/main/accumulMod.F90 b/components/elm/src/main/accumulMod.F90 index a3e2185b1ed7..a4f315435735 100644 --- a/components/elm/src/main/accumulMod.F90 +++ b/components/elm/src/main/accumulMod.F90 @@ -278,7 +278,7 @@ subroutine extract_accum_field_sl (name, field, nstep) beg = accum(nf)%beg1d end = accum(nf)%end1d - if (size(field,dim=1) /= end-beg+1) then + if (size(field,dim=1) < end-beg+1) then write(iulog,*)'ERROR in extract_accum_field for field ',accum(nf)%name write(iulog,*)'size of first dimension of field is ',& size(field,dim=1),' and should be ',end-beg+1 @@ -341,7 +341,7 @@ subroutine extract_accum_field_ml (name, field, nstep) numlev = accum(nf)%numlev beg = accum(nf)%beg1d end = accum(nf)%end1d - if (size(field,dim=1) /= end-beg+1) then + if (size(field,dim=1) < end-beg+1) then write(iulog,*)'ERROR in extract_accum_field for field ',accum(nf)%name write(iulog,*)'size of first dimension of field is ',& size(field,dim=1),' and should be ',end-beg+1 @@ -406,7 +406,7 @@ subroutine update_accum_field_sl (name, field, nstep) beg = accum(nf)%beg1d end = accum(nf)%end1d - if (size(field,dim=1) /= end-beg+1) then + if (size(field,dim=1) < end-beg+1) then write(iulog,*)'ERROR in UPDATE_ACCUM_FIELD_SL for field ',accum(nf)%name write(iulog,*)'size of first dimension of field is ',size(field,dim=1),& ' and should be ',end-beg+1 @@ -500,7 +500,7 @@ subroutine update_accum_field_ml (name, field, nstep) numlev = accum(nf)%numlev beg = accum(nf)%beg1d end = accum(nf)%end1d - if (size(field,dim=1) /= end-beg+1) then + if (size(field,dim=1) < end-beg+1) then write(iulog,*)'ERROR in UPDATE_ACCUM_FIELD_ML for field ',accum(nf)%name write(iulog,*)'size of first dimension of field is ',size(field,dim=1),& ' and should be ',end-beg+1 diff --git a/components/elm/src/main/decompInitMod.F90 b/components/elm/src/main/decompInitMod.F90 index e00221a388c7..bee57b57ce7c 100644 --- a/components/elm/src/main/decompInitMod.F90 +++ b/components/elm/src/main/decompInitMod.F90 @@ -2380,50 +2380,6 @@ subroutine decompInit_ghosts(glcmask) if (.not.lateral_connectivity) then - ! No ghost cells - procinfo%ncells_ghost = 0 - procinfo%ntunits_ghost = 0 - procinfo%nlunits_ghost = 0 - procinfo%ncols_ghost = 0 - procinfo%npfts_ghost = 0 - procinfo%nCohorts_ghost = 0 - - procinfo%begg_ghost = 0 - procinfo%begt_ghost = 0 - procinfo%begl_ghost = 0 - procinfo%begc_ghost = 0 - procinfo%begp_ghost = 0 - procinfo%begCohort_ghost = 0 - procinfo%endg_ghost = 0 - procinfo%endt_ghost = 0 - procinfo%endl_ghost = 0 - procinfo%endc_ghost = 0 - procinfo%endp_ghost = 0 - procinfo%endCohort_ghost = 0 - - ! All = local (as no ghost cells) - procinfo%ncells_all = procinfo%ncells - procinfo%ntunits_all = procinfo%ntunits - procinfo%nlunits_all = procinfo%nlunits - procinfo%ncols_all = procinfo%ncols - procinfo%npfts_all = procinfo%npfts - procinfo%nCohorts_all = procinfo%nCohorts - - procinfo%begg_all = procinfo%begg - procinfo%begt_all = procinfo%begt - procinfo%begl_all = procinfo%begl - procinfo%begc_all = procinfo%begc - procinfo%begp_all = procinfo%begp - procinfo%begCohort_all = procinfo%begCohort - procinfo%endg_all = procinfo%endg - procinfo%endt_all = procinfo%endt - procinfo%endl_all = procinfo%endl - procinfo%endc_all = procinfo%endc - procinfo%endp_all = procinfo%endp - procinfo%endCohort_all = procinfo%endCohort - - else - #if defined(MOAB_LATERAL) call get_proc_bounds(begg, endg) @@ -2528,7 +2484,53 @@ subroutine decompInit_ghosts(glcmask) procinfo%endp_all = procinfo%endp + procinfo%npfts_ghost procinfo%endCohort_all = procinfo%endCohort + procinfo%nCohorts_ghost -#elif defined(USE_PETSC_LIB) +#else + ! No ghost cells + procinfo%ncells_ghost = 0 + procinfo%ntunits_ghost = 0 + procinfo%nlunits_ghost = 0 + procinfo%ncols_ghost = 0 + procinfo%npfts_ghost = 0 + procinfo%nCohorts_ghost = 0 + + procinfo%begg_ghost = 0 + procinfo%begt_ghost = 0 + procinfo%begl_ghost = 0 + procinfo%begc_ghost = 0 + procinfo%begp_ghost = 0 + procinfo%begCohort_ghost = 0 + procinfo%endg_ghost = 0 + procinfo%endt_ghost = 0 + procinfo%endl_ghost = 0 + procinfo%endc_ghost = 0 + procinfo%endp_ghost = 0 + procinfo%endCohort_ghost = 0 + + ! All = local (as no ghost cells) + procinfo%ncells_all = procinfo%ncells + procinfo%ntunits_all = procinfo%ntunits + procinfo%nlunits_all = procinfo%nlunits + procinfo%ncols_all = procinfo%ncols + procinfo%npfts_all = procinfo%npfts + procinfo%nCohorts_all = procinfo%nCohorts + + procinfo%begg_all = procinfo%begg + procinfo%begt_all = procinfo%begt + procinfo%begl_all = procinfo%begl + procinfo%begc_all = procinfo%begc + procinfo%begp_all = procinfo%begp + procinfo%begCohort_all = procinfo%begCohort + procinfo%endg_all = procinfo%endg + procinfo%endt_all = procinfo%endt + procinfo%endl_all = procinfo%endl + procinfo%endc_all = procinfo%endc + procinfo%endp_all = procinfo%endp + procinfo%endCohort_all = procinfo%endCohort +#endif + + else + +#if defined(USE_PETSC_LIB) call get_proc_bounds(begg, endg) diff --git a/components/elm/src/main/elm_driver.F90 b/components/elm/src/main/elm_driver.F90 index 94a4cc7eee1e..2dde98ed4b18 100644 --- a/components/elm/src/main/elm_driver.F90 +++ b/components/elm/src/main/elm_driver.F90 @@ -1469,7 +1469,7 @@ subroutine elm_drv(doalb, nextsw_cday, declinp1, declin, rstwr, nlend, rdate) !$OMP PARALLEL DO PRIVATE (nc, bounds_clump) do nc = 1,nclumps call get_clump_bounds(nc, bounds_clump) - call lnd2iac_vars%update_lnd2iac(bounds_clump) + call lnd2iac_vars%update_lnd2iac(bounds_clump, lnd2atm_vars) end do !$OMP END PARALLEL DO call t_stopf('lnd2iac') diff --git a/components/elm/src/main/elm_initializeMod.F90 b/components/elm/src/main/elm_initializeMod.F90 index b594011cb555..7401bacae1d7 100644 --- a/components/elm/src/main/elm_initializeMod.F90 +++ b/components/elm/src/main/elm_initializeMod.F90 @@ -23,6 +23,7 @@ module elm_initializeMod use ELMFatesInterfaceMod , only : ELMFatesGlobals1,ELMFatesGlobals2 use BeTRSimulationELM, only : create_betr_simulation_elm use SoilLittVertTranspMod, only : CreateLitterTransportList + use ColumnConnectionSetType, only : c2c_connections use iso_c_binding ! !----------------------------------------- @@ -228,11 +229,15 @@ subroutine initialize1( ) 'Unsupported domain_decomp_type = ' // trim(domain_decomp_type)) end select +#ifdef MOAB_LATERAL + call domainlateral_init(ldomain_lateral) +#else if (lateral_connectivity) then call domainlateral_init(ldomain_lateral, cellsOnCell, edgesOnCell, & nEdgesOnCell, areaCell, dcEdge, dvEdge, & nCells_loc, nEdges_loc, maxEdges) endif +#endif ! *** Get JUST gridcell processor bounds *** ! Remaining bounds (landunits, columns, patches) will be determined @@ -413,6 +418,10 @@ subroutine initialize1( ) ! This is needed here for the following call to decompInit_glcp call initGridCells() + call initGhostGridCells() +#ifdef MOAB_LATERAL + call c2c_connections%Init(bounds_proc) +#endif if (fsurdat /= " " .and. use_finetop_rad) then if (masterproc) then diff --git a/components/elm/src/main/elm_instMod.F90 b/components/elm/src/main/elm_instMod.F90 index bdf5147186dc..f28b942c510a 100644 --- a/components/elm/src/main/elm_instMod.F90 +++ b/components/elm/src/main/elm_instMod.F90 @@ -421,7 +421,7 @@ subroutine elm_inst_biogeophys(bounds_proc) call grc_es%Init(bounds_proc%begg_all, bounds_proc%endg_all) call lun_es%Init(bounds_proc%begl_all, bounds_proc%endl_all) - call col_es%Init(bounds_proc%begc_all, bounds_proc%endc_all) + call col_es%Init(bounds_proc%begc_all, bounds_proc%endc_all, bounds_proc%endc) call veg_es%Init(bounds_proc%begp_all, bounds_proc%endp_all) call canopystate_vars%init(bounds_proc) @@ -436,7 +436,7 @@ subroutine elm_inst_biogeophys(bounds_proc) call grc_ws%Init(bounds_proc%begg_all, bounds_proc%endg_all) call lun_ws%Init(bounds_proc%begl_all, bounds_proc%endl_all) - call col_ws%Init(bounds_proc%begc_all, bounds_proc%endc_all, & + call col_ws%Init(bounds_proc%begc_all, bounds_proc%endc_all, bounds_proc%endc, & h2osno_col(begc:endc), & snow_depth_col(begc:endc), & soilstate_vars%watsat_col(begc:endc, 1:)) @@ -445,7 +445,7 @@ subroutine elm_inst_biogeophys(bounds_proc) call waterflux_vars%init(bounds_proc) call grc_wf%Init(bounds_proc%begg_all, bounds_proc%endg_all, bounds_proc) - call col_wf%Init(bounds_proc%begc_all, bounds_proc%endc_all) + call col_wf%Init(bounds_proc%begc_all, bounds_proc%endc_all, bounds_proc%endc) call veg_wf%Init(bounds_proc%begp_all, bounds_proc%endp_all) call chemstate_vars%Init(bounds_proc) diff --git a/components/elm/src/main/initGridCellsMod.F90 b/components/elm/src/main/initGridCellsMod.F90 index a0e99dd2102d..d8c6954d2f4b 100644 --- a/components/elm/src/main/initGridCellsMod.F90 +++ b/components/elm/src/main/initGridCellsMod.F90 @@ -718,6 +718,9 @@ subroutine initGhostGridcells() call CheckGhostSubgridHierarchy() #endif +#ifdef MOAB_LATERAL + call initGhostColumnsMOAB() +#endif end subroutine initGhostGridcells #ifdef USE_PETSC_LIB @@ -1330,4 +1333,56 @@ end subroutine CheckGhostSubgridHierarchy #endif !^ifdef USE_PETSC_LIB + +#if MOAB_LATERAL + !------------------------------------------------------------------------ + subroutine initGhostColumnsMOAB() + ! + ! DESCRIPTION + ! + use decompMod , only : get_proc_bounds + use domainLateralMod , only : ldomain_lateral, GridLevelIntegerDataHaloExchange + ! + implicit none + ! + ! LOCAL VARIABLES + type(bounds_type) :: bounds_proc ! temporary + integer, pointer :: num_nat_veg_columns(:) + integer :: g, c + integer, parameter :: icol_nat_veg = 1 + + call get_proc_bounds(bounds_proc) + + allocate(num_nat_veg_columns(bounds_proc%begg:bounds_proc%endg_all)) + num_nat_veg_columns(:) = 0 + + ! for owned grid cells, save the number of naturally vegetated soil columns + do c = bounds_proc%begc, bounds_proc%endc + g = col_pp%gridcell(c) + + if (col_pp%itype(c) == icol_nat_veg) then + if (num_nat_veg_columns(g) /= 0) then + call endrun(msg='ERROR: initGhostColumnsMOAB only supports the one natural soil column per grid cell') + else + num_nat_veg_columns(g) = num_nat_veg_columns(g) + 1 + end if + end if + + end do + + ! do the halo exchange + call GridLevelIntegerDataHaloExchange(ldomain_lateral, bounds_proc%begg, bounds_proc%endg, bounds_proc%endg_all, num_nat_veg_columns) + + ! for ghost columns, set mapping to ghost grid cell + c = bounds_proc%endc + do g = bounds_proc%endg + 1, bounds_proc%endg_all + if (num_nat_veg_columns(g) > 0) then + c = c + 1 + col_pp%gridcell(c) = g + col_pp%itype(c) = icol_nat_veg + endif + end do + + end subroutine initGhostColumnsMOAB +#endif end module initGridCellsMod diff --git a/components/elm/src/main/lnd2iacMod.F90 b/components/elm/src/main/lnd2iacMod.F90 index be2f19a87aea..8ef43868214d 100644 --- a/components/elm/src/main/lnd2iacMod.F90 +++ b/components/elm/src/main/lnd2iacMod.F90 @@ -14,12 +14,16 @@ module lnd2iacMod use ColumnDataType , only : col_cf ! for hr use VegetationType , only : veg_pp ! pftwgt use VegetationDataType, only: veg_cf ! for npp + use lnd2atmType , only : lnd2atm_type ! for lnd2atm_vars ! ! !PUBLIC TYPES: implicit none private save + ! Base temperature for degree day calculation: 18C in Kelvin + real(r8), parameter :: t_base_K = 291.15_r8 + ! lnd -> iac variables structure ! Fields are dimensioned (ngrid,numpft+1) ! pftwgt is frac of actual grid cell (not frac of land) @@ -27,6 +31,9 @@ module lnd2iacMod real(r8), pointer :: hr(:,:) => null() real(r8), pointer :: npp(:,:) => null() real(r8), pointer :: pftwgt(:,:) => null() + real(r8), pointer :: forc_hdm(:) => null() + real(r8), pointer :: hdd(:) => null() + real(r8), pointer :: cdd(:) => null() contains ! This object oriented stuff... @@ -57,34 +64,39 @@ subroutine Init(this, bounds) allocate(this%hr(begg:endg,0:numpft)) allocate(this%npp(begg:endg,0:numpft)) allocate(this%pftwgt(begg:endg,0:numpft)) + allocate(this%forc_hdm(begg:endg)) + allocate(this%hdd(begg:endg)) + allocate(this%cdd(begg:endg)) this%hr(:,:)=0.0_r8 this%npp(:,:)=0.0_r8 this%pftwgt(:,:)=0.0_r8 + this%forc_hdm(:)= 0.0_r8 + this%hdd(:)= 0.0_r8 + this%cdd(:)= 0.0_r8 end subroutine Init !------------------------------------------------------ - subroutine update_lnd2iac(this, bounds) + subroutine update_lnd2iac(this, bounds, lnd2atm_vars) ! !DESCRIPTION: - ! Stuff values into lnd2iac + ! Stuff values into lnd2iac. + ! Instantaneous fields (hr, npp, pftwgt, hdd, cdd) are overwritten each call. ! ! !ARGUMENTS: class(lnd2iac_type) , intent(inout) :: this type(bounds_type) , intent(in) :: bounds + type(lnd2atm_type) , intent(in) :: lnd2atm_vars ! elm land to atmosphere exchange data type ! !LOCAL VARIABLES: character(len=*), parameter :: subname = 'update_lnd2iac' integer :: begg,endg - integer :: begc,endc integer :: begp,endp - integer :: c,p,g,pft begg = bounds%begg; endg = bounds%endg begp = bounds%begp; endp = bounds%endp - ! Fill everything with zeros by default this%hr(begg:endg,:)=0.0_r8 this%npp(begg:endg,:)=0.0_r8 this%pftwgt(begg:endg,:)=0.0_r8 @@ -93,17 +105,23 @@ subroutine update_lnd2iac(this, bounds) do p = begp, endp g=veg_pp%gridcell(p) pft=veg_pp%itype(p) - c=veg_pp%column(p) ! for hr if (veg_pp%active(p)) then - ! Assign values - this%hr(g,pft) = col_cf%hr(c) ! Every pft in this column gets this hr value - this%npp(g,pft) = veg_cf%npp(p) - ! this is the fraction of actual grid cell - this%pftwgt(g,pft) = veg_pp%wtgcell(p) * ldomain%frac(g) * & - ldomain%mask(g) + ! Instantaneous fields + this%hr(g,pft) = col_cf%hr(c) + this%npp(g,pft) = veg_cf%npp(p) + this%pftwgt(g,pft) = veg_pp%wtgcell(p) * ldomain%frac(g) * & + ldomain%mask(g) + end if end do + + do g = begg, endg + ! Estimate heating and cooling degree days + this%hdd(g) = max(t_base_K - lnd2atm_vars%t_ref2m_grc(g), 0.0_r8) + this%cdd(g) = max(lnd2atm_vars%t_ref2m_grc(g) - t_base_K, 0.0_r8) + end do + end subroutine update_lnd2iac -end module lnd2iacMod \ No newline at end of file +end module lnd2iacMod diff --git a/components/elm/src/utils/domainLateralMod.F90 b/components/elm/src/utils/domainLateralMod.F90 index b46299a13661..33a78b3ea83b 100644 --- a/components/elm/src/utils/domainLateralMod.F90 +++ b/components/elm/src/utils/domainLateralMod.F90 @@ -545,6 +545,295 @@ subroutine ExchangeColumnLevelGhostData(bounds_proc, nvals_per_col, & end subroutine ExchangeColumnLevelGhostData +#else + +#ifdef MOAB_LATERAL + + !----------------------------------------------------------------------- + ! This is a stub for the case when PETSc is unavailable + ! + use shr_kind_mod, only : r8 => shr_kind_r8 + use shr_sys_mod , only : shr_sys_abort + use spmdMod , only : masterproc + use elm_varctl , only : iulog + use spmdMod , only : masterproc, iam, npes, mpicom, comp_id + use abortutils , only : endrun + use MOABGridType, only : moab_gcell, mlndghostid + ! + ! !PUBLIC TYPES: + implicit none + private + ! + + type, public :: oneD_int_data_for_moab + integer :: moab_app_id ! ID of MAOB app + character(len=1024) :: tag_name ! MOAB tag name + integer :: tag_type ! type of MOAB tag: 0 = dense, int; 1 = dense, double + integer :: num_tags ! Number of tags + integer :: entity_type(1) ! vertex or element based type + integer :: tag_index(1) ! Index of tag after it is registered in MOAB + integer :: num_comp ! number of components + integer :: ngcells ! number of grid cells + integer, allocatable :: values(:) ! data + end type oneD_int_data_for_moab + + type, public :: twoD_real_data_for_moab + integer :: moab_app_id ! ID of MAOB app + character(len=1024) :: tag_name ! MOAB tag name + integer :: tag_type ! type of MOAB tag: 0 = dense, int; 1 = dense, double + integer :: num_tags ! Number of tags + integer :: entity_type(1) ! vertex or element based type + integer :: tag_index(1) ! Index of tag after it is registered in MOAB + integer :: num_comp ! number of components + integer :: ngcells ! number of grid cells + real(r8), allocatable :: values(:,:) ! data + end type twoD_real_data_for_moab + + type, public :: domainlateral_type + type(oneD_int_data_for_moab) :: grid_level_count + end type domainlateral_type + + type(domainlateral_type) , public :: ldomain_lateral + ! + ! !PUBLIC MEMBER FUNCTIONS: + public domainlateral_init ! initializes + public setup_twoD_real_data_for_moab + public GridLevelIntegerDataHaloExchange + public GridLevelRealDataHaloExchange + ! + !EOP + !------------------------------------------------------------------------------ + +contains + + !------------------------------------------------------------------------------ + subroutine setup_oneD_int_data_for_moab(moab_app_id, tag_name, num_cells_ghosted, data) + ! + ! DESCRIPTION: + ! Sets up 1D integer-type data structure for performing MOAB-based halo exchange + ! of data. This supports a single value per grid cell to be exchanged. + ! + use iso_c_binding + use iMOAB, only : iMOAB_DefineTagStorage + ! + implicit none + ! + ! ARGUMENTS: + integer , intent(in) :: moab_app_id + character(len=*) , intent(in) :: tag_name + integer , intent(in) :: num_cells_ghosted + type(oneD_int_data_for_moab) , intent(out) :: data + ! + ! LOCAL VARIABLES: + integer :: ierr + + data%moab_app_id = moab_app_id + data%tag_name = trim(tag_name) // C_NULL_CHAR ! name + data%tag_type = 0 ! 0 = dense, int + data%num_tags = 1 ! a single tag + data%entity_type(1) = 1 ! element (== cell) based data + data%num_comp = 1 ! number components in the tag + data%ngcells = num_cells_ghosted ! number of grid cells + + ! allocate memory + allocate(data%values(data%ngcells)) + + ! define the tag in MOAB + ierr = iMOAB_DefineTagStorage(data%moab_app_id, data%tag_name, data%tag_type, data%num_comp, data%tag_index(1)) + + end subroutine setup_oneD_int_data_for_moab + + !------------------------------------------------------------------------------ + subroutine setup_twoD_real_data_for_moab(moab_app_id, tag_name, num_comp, num_cells_ghosted, data) + ! + ! DESCRIPTION: + ! Sets up 2D real-type data structure for performing MOAB-based halo exchange + ! of data. This supports 'num_comp' values per grid cell to be exchanged. + ! + use iso_c_binding + use iMOAB, only : iMOAB_DefineTagStorage + ! + ! ARGUMENT: + integer , intent(in) :: moab_app_id + character(len=*) , intent(in) :: tag_name + integer , intent(in) :: num_comp + integer , intent(in) :: num_cells_ghosted + type(twoD_real_data_for_moab) , intent(out) :: data + ! + ! LOCAL VARIABLES: + integer :: ierr + + data%moab_app_id = moab_app_id + data%tag_name = trim(tag_name) // C_NULL_CHAR ! name + data%tag_type = 1 ! 1 = dense, double + data%num_tags = 1 ! a single tag + data%entity_type(1) = 1 ! element (== cell) based data + data%num_comp = num_comp ! number components in the tag + data%ngcells = num_cells_ghosted ! number of grid cells + + ! allocate memory + allocate(data%values(data%num_comp, data%ngcells)) + + ! define the tag in MOAB + ierr = iMOAB_DefineTagStorage(data%moab_app_id, data%tag_name, data%tag_type, data%num_comp, data%tag_index(1)) + + end subroutine setup_twoD_real_data_for_moab + + !------------------------------------------------------------------------------ + subroutine domainlateral_init(domain_l) + ! + ! DESCRIPTION: + ! Creates data structure for doing halo exchanges using MOAB + ! + implicit none + ! + ! ARGUMENTS: + type(domainlateral_type) :: domain_l ! domain datatype + + ! creates the MOAB tag 1D data at grid level + call setup_oneD_int_data_for_moab(mlndghostid, 'grid_level_count', moab_gcell%num_ghosted, domain_l%grid_level_count) + + end subroutine domainlateral_init + + !------------------------------------------------------------------------------ + subroutine do_haloexchange_oneD_integer_data_for_moab(data) + ! + ! DESCRIPTION: + ! Perform MOAB-based halo exchange + ! + use iMOAB, only : iMOAB_SetIntTagStorage, iMOAB_GetIntTagStorage, iMOAB_SynchronizeTags + ! + ! ARGUMENT: + type(oneD_int_data_for_moab) , intent(inout) :: data + ! + ! LOCAL VARIABLE: + integer :: ierr + + ! set the data in MOAB tag + ierr = iMOAB_SetIntTagStorage(data%moab_app_id, data%tag_name, data%ngcells * data%num_comp, data%entity_type(1), data%values) + if (ierr > 0) call endrun('Error: setting values in MOAB tag failed.') + + ! do the halo-exchange + ierr = iMOAB_SynchronizeTags(data%moab_app_id, data%num_tags, data%tag_index(1), data%entity_type(1)) + if (ierr > 0) call endrun('Error: synchronization of MOAB tag failed.') + + ! get the data from MOAB tag + ierr = iMOAB_GetIntTagStorage(data%moab_app_id, data%tag_name, data%ngcells * data%num_comp, data%entity_type(1), data%values) + if (ierr > 0) call endrun('Error: getting values in MOAB tag failed.') + + end subroutine do_haloexchange_oneD_integer_data_for_moab + + !------------------------------------------------------------------------------ + subroutine do_haloexchange_twoD_real_data_for_moab(data) + ! + ! DESCRIPTION: + ! Perform MOAB-based halo exchange + ! + use iMOAB, only : iMOAB_SetDoubleTagStorage, iMOAB_GetDoubleTagStorage, iMOAB_SynchronizeTags + ! + ! INPUT ARGUMENT: + type(twoD_real_data_for_moab) , intent(inout) :: data + ! + ! LOCAL VARIABLE: + integer :: ierr + + ! set the data in MOAB tag + ierr = iMOAB_SetDoubleTagStorage(data%moab_app_id, data%tag_name, data%ngcells * data%num_comp, data%entity_type(1), data%values) + if (ierr > 0) call endrun('Error: setting values in MOAB tag failed.') + + ! do the halo-exchange + ierr = iMOAB_SynchronizeTags(data%moab_app_id, data%num_tags, data%tag_index(1), data%entity_type(1)) + if (ierr > 0) call endrun('Error: synchronization of MOAB tag failed.') + + ! get the data from MOAB tag + ierr = iMOAB_GetDoubleTagStorage(data%moab_app_id, data%tag_name, data%ngcells * data%num_comp, data%entity_type(1), data%values) + if (ierr > 0) call endrun('Error: getting values in MOAB tag failed.') + + end subroutine do_haloexchange_twoD_real_data_for_moab + + !------------------------------------------------------------------------------ + subroutine GridLevelIntegerDataHaloExchange(domain_l, begg, endg_owned, endg_all, elm_data) + ! + ! DESCRIPTION: + ! Performs halo exchange of integer data. It is assumed that there is only one value + ! per grid cell. elm_data has data in ELM-format such that owned grid cells at the beginning + ! followed by ghost grid cells. After MOAB-based halo exchange values are filled in + ! elm_data corresponding to ghost cells. + ! + implicit none + ! + ! ARGUMENTS: + type(domainlateral_type) :: domain_l ! domain datatype + integer, intent(in) :: begg ! beginning index of grid cell + integer, intent(in) :: endg_owned ! ending index for owned grid cells + integer, intent(in) :: endg_all ! ending index for all (owned + ghost) grid cells + integer, intent(inout) , pointer :: elm_data(:) ! data packed in ELM's format + ! + ! LOCAL VARAIBLES: + integer :: g, j, idx + integer :: ierr + + ! convert data from ELM format to MOAB format + do g = begg, endg_owned + idx = moab_gcell%elm2moab(g) + domain_l%grid_level_count%values(idx) = elm_data(g) + end do + + ! perform halo exchange + call do_haloexchange_oneD_integer_data_for_moab(domain_l%grid_level_count) + + ! convert data from MOAB format to ELM format + do idx = 1, moab_gcell%num_ghosted + if (.not.moab_gcell%is_owned(idx)) then + g = moab_gcell%moab2elm(idx) + elm_data(g) = domain_l%grid_level_count%values(idx) + end if + end do + + end subroutine GridLevelIntegerDataHaloExchange + + !------------------------------------------------------------------------------ + subroutine GridLevelRealDataHaloExchange(data_moab, begg, endg_owned, endg_all, elm_data) + ! + ! DESCRIPTION: + ! Performs halo exchange of real data. elm_data has data in ELM-format such that + ! owned grid cells are at the beginning followed by ghost grid cells. After + ! MOAB-based halo exchange, values in elm_data are filled for ghost cells. + ! data_moab must be pre-set up by the caller via setup_twoD_real_data_for_moab. + ! + implicit none + ! + type(twoD_real_data_for_moab) , intent(inout) :: data_moab ! caller-owned MOAB tag struct + integer, intent(in) :: begg ! beginning index of grid cell + integer, intent(in) :: endg_owned ! ending index for owned grid cells + integer, intent(in) :: endg_all ! ending index for all (owned + ghost) grid cells + real(r8), intent(inout), pointer :: elm_data(:,:) ! data packed in ELM's format + ! + integer :: g, j, idx + + ! convert data from ELM format to MOAB format + do g = begg, endg_owned + idx = moab_gcell%elm2moab(g) + do j = 1, data_moab%num_comp + data_moab%values(j, idx) = elm_data(g, j) + end do + end do + + ! perform halo exchange + call do_haloexchange_twoD_real_data_for_moab(data_moab) + + ! convert data from MOAB format to ELM format + do idx = 1, moab_gcell%num_ghosted + if (.not.moab_gcell%is_owned(idx)) then + g = moab_gcell%moab2elm(idx) + do j = 1, data_moab%num_comp + elm_data(g, j) = data_moab%values(j, idx) + end do + end if + end do + + end subroutine GridLevelRealDataHaloExchange + #else !----------------------------------------------------------------------- @@ -610,4 +899,6 @@ end subroutine domainlateral_init #endif +#endif + end module domainLateralMod diff --git a/components/gcam/bld/build-namelist b/components/gcam/bld/build-namelist index 6fec202d0178..3d15ddf85e0d 100755 --- a/components/gcam/bld/build-namelist +++ b/components/gcam/bld/build-namelist @@ -441,10 +441,13 @@ add_default($nl, 'base_co2_aircraft_file'); add_default($nl, 'base_npp_file'); add_default($nl, 'base_hr_file'); add_default($nl, 'base_pft_file'); +add_default($nl, 'base_hdd_file'); +add_default($nl, 'base_cdd_file'); add_default($nl, 'gcam2elm_co2_mapping_file'); add_default($nl, 'gcam2elm_luc_mapping_file'); add_default($nl, 'gcam2elm_woodharvest_mapping_file'); add_default($nl, 'gcam2elm_cdensity_mapping_file'); +add_default($nl, 'gcam2elm_degdays_mapping_file'); # grid mapping and initialization, relative to inputdata add_default($nl, 'gcam_gridfile'); @@ -478,9 +481,12 @@ add_default($nl, 'fdyndat_ehc'); add_default($nl, 'read_scalars'); add_default($nl, 'scalar_source_dir'); add_default($nl, 'write_scalars'); +add_default($nl, 'read_hdd_cdd'); +add_default($nl, 'write_hdd_cdd'); add_default($nl, 'write_co2'); add_default($nl, 'elm_ehc_agyield_scaling'); add_default($nl, 'elm_ehc_carbon_scaling'); +add_default($nl, 'elm_ehc_hdd_cdd'); add_default($nl, 'ehc_eam_co2_emissions'); add_default($nl, 'gcam_spinup'); add_default($nl, 'run_gcam'); @@ -871,4 +877,4 @@ sub quote_string { $str = "\'$str\'"; } return $str; -} \ No newline at end of file +} diff --git a/components/gcam/bld/namelist_files/namelist_defaults_gcam.xml b/components/gcam/bld/namelist_files/namelist_defaults_gcam.xml index df3ab6b04c8b..dc17c14aa62e 100644 --- a/components/gcam/bld/namelist_files/namelist_defaults_gcam.xml +++ b/components/gcam/bld/namelist_files/namelist_defaults_gcam.xml @@ -62,22 +62,33 @@ for the iac data in the e3sm distribution iac/giac/gcam/gcam_6_0/data/base_r0125_set_this_in_namelist_annAvgMonthly_2010-2014_npp.csv iac/giac/gcam/gcam_6_0/data/base_r025_set_this_in_namelist_annAvgMonthly_2010-2014_npp.csv iac/giac/gcam/gcam_6_0/data/base_r05_set_this_in_namelist_annAvgMonthly_2010-2014_npp.csv -iac/giac/gcam/gcam_6_0/data/base_20260303_I20TREAMELMCNPRDCTCBCPHSBGC_ne30pg2_f09_oEC60to30v3_PerAvg_2010-2014_npp.csv +iac/giac/gcam/gcam_6_0/data/base_20260624_I20TREAMELMCNPRDCTCBCPHSBGC_f09_PerAvg_2010-2014_npp.csv iac/giac/gcam/gcam_6_0/data/base_f19_set_this_in_namelist_annAvgMonthly_2010-2014_npp.csv iac/giac/gcam/gcam_6_0/data/base_r0125_set_this_in_namelist_annAvgMonthly_2010-2014_hr.csv iac/giac/gcam/gcam_6_0/data/base_r025_set_this_in_namelist_annAvgMonthly_2010-2014_hr.csv iac/giac/gcam/gcam_6_0/data/base_r05_set_this_in_namelist_annAvgMonthly_2010-2014_hr.csv -iac/giac/gcam/gcam_6_0/data/base_20260303_I20TREAMELMCNPRDCTCBCPHSBGC_ne30pg2_f09_oEC60to30v3_PerAvg_2010-2014_hr.csv +iac/giac/gcam/gcam_6_0/data/base_20260624_I20TREAMELMCNPRDCTCBCPHSBGC_f09_PerAvg_2010-2014_hr.csv iac/giac/gcam/gcam_6_0/data/base_f19_set_this_in_namelist_annAvgMonthly_2010-2014_hr.csv iac/giac/gcam/gcam_6_0/data/base_r0125_set_this_in_namelist_annAvgMonthly_2010-2014_pft_wt.csv iac/giac/gcam/gcam_6_0/data/base_r025_set_this_in_namelist_annAvgMonthly_2010-2014_pft_wt.csv iac/giac/gcam/gcam_6_0/data/base_r05_set_this_in_namelist_annAvgMonthly_2010-2014_pft_wt.csv -iac/giac/gcam/gcam_6_0/data/base_20260303_I20TREAMELMCNPRDCTCBCPHSBGC_ne30pg2_f09_oEC60to30v3_PerAvg_2010-2014_pft_wt.csv +iac/giac/gcam/gcam_6_0/data/base_20260624_I20TREAMELMCNPRDCTCBCPHSBGC_f09_PerAvg_2010-2014_pft_wt.csv iac/giac/gcam/gcam_6_0/data/base_f19_set_this_in_namelist_annAvgMonthly_2010-2014_pft_wt.csv +iac/giac/gcam/gcam_6_0/data/base_r0125_set_this_in_namelist_annAvg_2010-2014_hdd.csv +iac/giac/gcam/gcam_6_0/data/base_r025_set_this_in_namelist_annAvg_2010-2014_hdd.csv +iac/giac/gcam/gcam_6_0/data/base_r05_set_this_in_namelist_annAvg_2010-2014_hdd.csv +iac/giac/gcam/gcam_6_0/data/base_20260624_I20TREAMELMCNPRDCTCBCPHSBGC_f09_annAvg_2010-2014_hdd.csv +iac/giac/gcam/gcam_6_0/data/base_f19_set_this_in_namelist_annAvg_2010-2014_hdd.csv +iac/giac/gcam/gcam_6_0/data/base_r0125_set_this_in_namelist_annAvg_2010-2014_cdd.csv +iac/giac/gcam/gcam_6_0/data/base_r025_set_this_in_namelist_annAvg_2010-2014_cdd.csv +iac/giac/gcam/gcam_6_0/data/base_r05_set_this_in_namelist_annAvg_2010-2014_cdd.csv +iac/giac/gcam/gcam_6_0/data/base_20260624_I20TREAMELMCNPRDCTCBCPHSBGC_f09_annAvg_2010-2014_cdd.csv +iac/giac/gcam/gcam_6_0/data/base_f19_set_this_in_namelist_annAvg_2010-2014_cdd.csv iac/giac/gcam/gcam_6_0/mappings/co2_regional.xml iac/giac/gcam/gcam_6_0/mappings/luc.xml iac/giac/gcam/gcam_6_0/mappings/woodharvest.xml iac/giac/gcam/gcam_6_0/mappings/cdensity.xml +iac/giac/gcam/gcam_6_0/mappings/degree_days.xml iac/giac/glm2iac/landuse.timeseries_0.125x0.125_HIST_simyr2015_c241205.nc @@ -124,9 +135,12 @@ for the iac data in the e3sm distribution .false. .true. +.false. +.true. .false. .true. .true. +.true. .true. .true. .true. @@ -328,4 +342,4 @@ for the iac data in the e3sm distribution surfdata_iESM.log surfdata_iESM_dyn.nc - \ No newline at end of file + diff --git a/components/gcam/bld/namelist_files/namelist_definition_gcam.xml b/components/gcam/bld/namelist_files/namelist_definition_gcam.xml index 9928f66bc0a8..227c74f739d7 100644 --- a/components/gcam/bld/namelist_files/namelist_definition_gcam.xml +++ b/components/gcam/bld/namelist_files/namelist_definition_gcam.xml @@ -178,6 +178,24 @@ Initial hr values file for scalars Initial pft weights values file for scalars + +Baseline hdd values file for scaling + + + +Baseline cdd values file for scaling + + + +Mapping file for heating/cooling degree days + + + +If FALSE, hdd/cdd are calculated from t_ref2m + + + +Write hdd cdd to file + + + +Heating degree days and Cooling degree days from elm to gcam + + - \ No newline at end of file + diff --git a/components/gcam/src b/components/gcam/src index dccf4b26f889..4e4860c9c21c 160000 --- a/components/gcam/src +++ b/components/gcam/src @@ -1 +1 @@ -Subproject commit dccf4b26f889d2308a6c7b7d01f82e6633ba6f89 +Subproject commit 4e4860c9c21cdcd40e55c8e1c69a553217dca4f0 diff --git a/components/gcam/tools/create_e3sm_gcam_land_scalar_baseline_local.r b/components/gcam/tools/create_e3sm_gcam_land_scalar_baseline_local.r deleted file mode 100644 index 1388ac924acc..000000000000 --- a/components/gcam/tools/create_e3sm_gcam_land_scalar_baseline_local.r +++ /dev/null @@ -1,519 +0,0 @@ -# create_e3sm_gcam_land_scalar_baseline_local.r - -# This function reads monthly elm history files to obtain pft-level values for baseline of: -# veg_cf%npp, col_cf%hr, veg_pp%wtgcell for the veg landunit only, and total cell area -# this is currently the h2 file in our current run script - -# the period averaged data in E3SM-GCAM include zero values (and not uninitialized or large missing data values) -# because only active pfts are updated to be set to non-zero values and sent through the coupler -# the coupler does not distinguish which values to include in the total average -# the passed pft weights are also zero when the pft is not active, -# which reduces the influence of averages that include a pft that is active for only part of a period -# so need to include missing data in the period average as zero values - -# It is necessary to use the total period average because the max monthly values do not line up across variables, -# and when matched to npp max the hr is far from a max - -# the current E3SM-GCAM period is 5-years, so use this for the baseline calculation -# even if the GCAM period decrdeases to 1, we may still want a 5 year average for a baseline -# the calculation period is (year_end - year_start + 1) - -# Note that the output wtgcell here is the weight of the full grid cell, -# while veg_pp%wtgcell in the h1 file is the weight of pft on land that may not cover the entire cell -# because elm assumes that land covers the entire cell - -# The monthly values for npp (gC/m^2/s) and hr (gC/m^2/s) and pft weight (fraction) are averages of 30-minute elm calcs - -# The default outputs of this script are the average values over the period calculated as: -# the month-weighted average (calculated via days) of the monthly averages across years -# this just splits the period average into two steps, reducing the number of weights -# and possibly reducing the precision error as the weights for 12 months are larger than the weights for all months -# this matches how it is calculated in E3SM, wich is a total average over the period, based on 30-minute increments -# note that any missing monthly input data is included in the calculations as zero values - - -# Don't use these, they have been deprecated, but the code is still here for diagnostic purposes: -# Alternatively, the output can be values corresponding to the month of max monthly npp -# Alternatively, the max monthly value of each can be output for comparison - -# The full grid cell area is also output (km^2), only for land cells; -# the elm h1 file has missing values for non-land cells - -# if using a sequence of multiple years, the monthly average across the years will be calculated first, -# then the annual average will be calculated or the maximum monthly value will be selected - -# arguments - -# indir: path to history files (requires final "/") -# case_name: simulation case name that is the main part of the history file name including ".elm.h#."; "yyyy-mm.nc" will be appended as appropriate -# year_start: start year of history files to read; will also be included in output file names -# year_end: end year of history files to read; will also be included in output file names -# outdir: path to output files (requires final "/") -# out_base_name: the beginning of the output file names; needs to include E3SM resolution and active component cofig info -# avg_period: Default=5; number of years in averaging period - -# outputs - -# __-_hr.csv -# __-_npp.csv -# __-_pft_wt.csv -# _cell_area.csv - -# output file format - -# csv with one header file, separator = "," -# four columns for pft data (npp, weight, hr), in column order: pft, lon, lat, value -# three columns for area data, in column order: lon, lat, value -# data are in numerical order sorted according to: -# lon varies fastest -# lat varies second fastest -# pft varies third fastest (if present) -# all pftXlonXlat combinations are needed in the output files - -# NOTES - -# about 1.5 hours to finish on desktop - -# resolution will be determined from the history file -# recall that longitude varies fastest -# this sript is agnostic regarding resolution; indices are determined based on input file order - -# for f09 and f19: -# elm grid cell edges start at lon=- (zero center) and lat=-90; pole cells are half-lat-res (pole full-cell center) - -# for r05 and r025 and r0125: -# elm grid cell edges start at lon = -180 lat = -90, with equal spacing between cell centers - -# lon-lat values are indices starting at one, increasing positively from the origin defined above -# not all pftXlonXlat combinations are included in the h1 1d arrays - -# there is only one time record in each file -# all twelve months must be present for each year - -# HR is for the whole column, so all pfts in the veg land unit column have the same HR -# vegetated land unit is in the first index, with id = 1 - -# NPP can be negative! or missing value/NA or zero -# HR and pft weight are all positive or zero or missing value/NA (pft area only has no missing values) -# cell area is always positive and non-zero, na/missing value, in the h1 files - -# Currently this assumes that there is only one topounit per grid cell and that only veg landunit pfts are mapped -# this mapping in E3SM does not include topounits or non-veg land units - -# this updated calculation, which better matches what is now calculated in E3SM-GCAM gives slightly different results from the previous calculation -# for one example month in one year: -# npp values have 238 instances of absolute differences greater than 10% (out of 81159 valid values) -# pft values have max perecent absolute difference of 0.48% (out of 81159 valid values) -# hr values have 272 instances of absolute differences greater than 1.8% (out of 15349 valid values) - -# this code operates only on the veg landunit - -# there should be 17 pfts for each grid cellXtopounit -# pfts in order are: -# 0 - bare -# 1 - needle leaf evergreen temperate tree -# 2 - needle leaf evergreen boreal tree -# 3 - needle leaf deciduous boreal tree -# 4 - broad leaf evergreen tropical tree -# 5 - broad leaf evergreen temperate tree -# 6 - broad leaf deciduaous tropical tree -# 7 - broad leaf deciduaous temperate tree -# 8 - broad leaf deciduaous boreal tree -# 9 - broad leaf evergreen temperate shrub -# 10 - broad leaf deciduaous temperate shrub -# 11 - broad leaf deciduaous boreal shrub -# 12 - C3 arctic grass -# 13 - C3 non-arctic grass -# 14 - C4 grass -# 15 - crop -# 16 - NA - - -library(ncdf4) - -create_e3sm_gcam_land_scalar_baseline_local <- function( indir = "./", - case_name, - year_start = 2010, - year_end = 2014, - outdir = "./", - out_base_name, - avg_period = 5) { - - cat("Start create_e3sm_gcam_land_scalar_baseline_local.r at", date(), "\n") - - veg_lunit_id = 1 - num_years = year_end - year_start + 1 - - # stop if start/end years do not match the averaging period - if(num_years != avg_period) { - cat("Number of input years (num_years = year_end - year_start + 1) do not match the averaging period (avg_period): ", num_years, "!=", avg_period, "\n") - stop() - } - - avg_tag = "_PerAvg" - - # error tolerance for comparing weighted avering methods for precision - # this tolerance shows now differences in all three variables for the test case - # note that precision calc errors are larger for pft weight than npp and hr - err_tol = 1e-14 - - # no leap years in the model - days_in_months = c(31,28,31,30,31,30,31,31,30,31,30,31) - days_in_year = sum(days_in_months) - all_weights = array(dim=c(avg_period, 12)) - - # open one file to get the constant info - - if(year_start < 10){ - ystr = paste0("000", year_start) - } else if(year_start < 100) { - ystr = paste0("00", year_start) - } else if(year_start < 1000) { - ystr = paste0("0", year_start) - } else { - ystr = paste0(year_start) - } - - nid = nc_open(paste0(indir, case_name, ystr, "-01.nc")) - num_lon = nid$dim$lon$len - num_lat = nid$dim$lat$len - num_cells = num_lon * num_lat - num_col_ind = nid$dim$column$len - num_pft_ind = nid$dim$pft$len - num_pft = nid$dim$natpft$len - num_lunit = nid$dim$ltype$len - num_gridcell = nid$dim$gridcell$len - num_topounit = nid$dim$topounit$len - lon = ncvar_get(nid,varid="lon",start=c(1), count=c(num_lon)) - lat = ncvar_get(nid,varid="lat",start=c(1), count=c(num_lat)) - # these are lon, lat variables - area = ncvar_get(nid,varid="area") - landfrac = ncvar_get(nid,varid="landfrac") # this should match landmask - pftmask = ncvar_get(nid,varid="pftmask") # not all landfrac > 0 have valid pfts - # these are column index variables - col_lunit = ncvar_get(nid,varid="cols1d_itype_lunit") - col_lon_ind = ncvar_get(nid,varid="cols1d_ixy") - col_lat_ind = ncvar_get(nid,varid="cols1d_jxy") - col_topounit = ncvar_get(nid,varid="cols1d_topounit") # this is topounit index - # these are pft index variables - pft_pft = ncvar_get(nid,varid="pfts1d_itype_veg") - pft_lon_ind = ncvar_get(nid,varid="pfts1d_ixy") - pft_lat_ind = ncvar_get(nid,varid="pfts1d_jxy") - pft_topounit_ind = ncvar_get(nid,varid="pfts1d_topounit") # this is topounit index - pft_lunit = ncvar_get(nid,varid="pfts1d_itype_lunit") - pft_active = ncvar_get(nid,varid="pfts1d_active") # this includes pfts on other land units and pft wts > 0 on veg land unit - nc_close(nid) - - # check to make sure that there is only one topounit per grid cell - if (num_gridcell != num_topounit) { - stop("Error: This code does not currently support mapping for multiple topounits within a grid cell\n") - } - - # set up some arrays - avg_npp = array(dim=c(num_pft_ind)) - avg_hr = array(dim=c(num_col_ind)) - avg_pft_weight = array(dim=c(num_pft_ind)) - pft_cell_weight = array(dim=c(num_pft_ind)) - hr_monthly_avg = array(dim=c(12, num_col_ind)) - npp_monthly_avg = array(dim=c(12, num_pft_ind)) - pft_weight_monthly_avg = array(dim=c(12, num_pft_ind)) - hr_monthly_cnt = array(dim=c(12, num_col_ind)) - npp_monthly_cnt = array(dim=c(12, num_pft_ind)) - pft_weight_monthly_cnt = array(dim=c(12, num_pft_ind)) - - # these are for period average - npp_month_values = array(dim=c(num_years, 12, num_pft_ind)) - pft_weight_month_values = array(dim=c(num_years, 12, num_pft_ind)) - hr_month_values = array(dim=c(num_years, 12, num_col_ind)) - npp_month_values[,,] = 0.0 - pft_weight_month_values[,,] = 0.0 - hr_month_values[,,] = 0.0 - - avg_npp[] = 0 - avg_hr[] = 0 - avg_pft_weight[] = 0 - - hr_monthly_avg[,] = 0 - npp_monthly_avg[,] = 0 - pft_weight_monthly_avg[,] = 0 - hr_monthly_cnt[,] = 0 - npp_monthly_cnt[,] = 0 - pft_weight_monthly_cnt[,] = 0 - - # loop over 12 months - for (m in 1:12) { - if (m < 10) {mtag = paste0(0,m) - } else { mtag = paste0(m)} - - # loop over years to sum the monthly values - # assume that all values are valid here, unless pft weight is zero - for (y in year_start:year_end) { - cat("Processing month", m, "year", y, "\n") - yind = y - year_start + 1 - - if(y < 10){ - ystr = paste0("000", y) - } else if(y < 100) { - ystr = paste0("00", y) - } else if(y < 1000) { - ystr = paste0("0", y) - } else { - ystr = paste0(y) - } - - fname = paste0(indir, case_name, ystr, "-", mtag, ".nc") - nid = nc_open(fname) - - # NPP and pft weight are by pft index - pft_land_weight = ncvar_get(nid,varid="pfts1d_wtgcell") - npp = ncvar_get(nid,varid="NPP") - - # HR is by column index (and with multiple landunits) - hr = ncvar_get(nid,varid="HR") - - # convert h1 pft frac of land to frac of grid cell - pft_cell_weight[] = 0 - for (p in 1: num_pft_ind) { - pft_cell_weight[p] = pft_land_weight[p] * landfrac[pft_lon_ind[p], pft_lat_ind[p]] * pftmask[pft_lon_ind[p], pft_lat_ind[p]] - } - - # since zero-weight pfts are used to determine whether a cell has valid values for processing, - # zeroing the pft weights for undesired records effectively filters them out - - # zero out the non-veg landunit pft weights and npp values to be able to match with hr uniquely - nvlu_inds = which(pft_lunit != veg_lunit_id) - pft_cell_weight[nvlu_inds] = 0 - npp[nvlu_inds] = 0 - - # if pft weight is not zero, add it into the sum and count it - # need to figure out the hr index - but at the column level; so include if any pft is non-zero in a cell - # this also removes any NA/missing pft weight values (of which there are none) - pft_nonzero_inds = which(pft_cell_weight > 0) - - # store the sum for period avg - pft_weight_monthly_avg[m, pft_nonzero_inds] = pft_weight_monthly_avg[m, pft_nonzero_inds] + pft_cell_weight[pft_nonzero_inds] - pft_weight_monthly_cnt[m, pft_nonzero_inds] = pft_weight_monthly_cnt[m, pft_nonzero_inds] + 1 - - # do not count missing values - na_inds = which(is.na(npp)) - avg_inds = setdiff(pft_nonzero_inds, na_inds) - - # store the sum for period avg - npp_monthly_avg[m, avg_inds] = npp_monthly_avg[m, avg_inds] + npp[avg_inds] - npp_monthly_cnt[m, avg_inds] = npp_monthly_cnt[m, avg_inds] + 1 - - # find the hr columns that are in cells with at least one non-zero pft - # only process veg land unit - # use data frames to only do a cell once - - # get unique valid cells/topounits - unique essentially selects one pft record for each cell/topounit - valid_pft_cell = data.frame(valid_pft_lon = pft_lon_ind[pft_nonzero_inds], - valid_pft_lat = pft_lat_ind[pft_nonzero_inds], - valid_pft_topounit = pft_topounit_ind[pft_nonzero_inds], - valid_pft_lunit = pft_lunit[pft_nonzero_inds]) - valid_pft_cell = unique(valid_pft_cell) - # get unique hr cells for veg land unit - hr_veglu = data.frame(hr_lon_ind = col_lon_ind, hr_lat_ind = col_lat_ind, hr_topounit = col_topounit, - hr_lu = col_lunit, hr_value = hr, hr_col_ind = c(1:num_col_ind)) - hr_veglu = hr_veglu[hr_veglu$hr_lu == veg_lunit_id,] - # this unique call shouldn't be necessary, but do it anyway - hr_veglu = hr_veglu[!duplicated(hr_veglu[,c(1:4)]),] - # merge these on the cell indices, topounit, and landunit - valid_pft_hr = merge(valid_pft_cell, hr_veglu, by.x = c("valid_pft_lon", "valid_pft_lat", "valid_pft_topounit", "valid_pft_lunit"), - by.y = c("hr_lon_ind", "hr_lat_ind", "hr_topounit", "hr_lu"), all.x = TRUE) - # drop any NA/missing values - valid_pft_hr = valid_pft_hr[!is.na(valid_pft_hr$hr_value),] - - # store the sum for period avg - hr_monthly_avg[m, valid_pft_hr$hr_col_ind] = hr_monthly_avg[m, valid_pft_hr$hr_col_ind] + valid_pft_hr$hr_value - hr_monthly_cnt[m, valid_pft_hr$hr_col_ind] = hr_monthly_cnt[m, valid_pft_hr$hr_col_ind] + 1 - - nc_close(nid) - - # store all the valid values for calculating average differently - pft_weight_month_values[yind, m, pft_nonzero_inds] = pft_cell_weight[pft_nonzero_inds] - npp_month_values[yind, m, avg_inds] = npp[avg_inds] - hr_month_values[yind, m, valid_pft_hr$hr_col_ind] = valid_pft_hr$hr_value - all_weights[yind, m] = days_in_months[m] / (avg_period * sum(days_in_months)) - - } # end y loop over year - - # now calc the average monthly values; equal weights cuz month length is the same across years - # doing this first makes the weighted calc easier - # use the avg_period instead of the count so that the zero values are included in the averages - pft_weight_monthly_avg[m,] = pft_weight_monthly_avg[m,] / avg_period - npp_monthly_avg[m,] = npp_monthly_avg[m,] / avg_period - hr_monthly_avg[m,] = hr_monthly_avg[m,] / avg_period - - } # end m loop over month - - cat("Finish time loop, starting average calcs, at", date(), "\n") - - # calc period average - # in E3SM coupler values are summed and averaged without error checking, and zeros are passed for missing/non-active pft data - # in E3SM outlier npp/hr values are removed before scalar calculation (outliers are determined without zero and nan values) - # use weights based on days in month - - # notify of missing monthly averages - # but just sum them all up because this includes all pfts in all cells, many of which do not exist - pft_monthly_zero_inds = NULL - npp_monthly_zero_inds = NULL - hr_monthly_zero_inds = NULL - - # npp and pft weight - for(v in 1: num_pft_ind) { - # pft weight - # log the non-zero and zero monthly values - cnt_mask_pft = pft_weight_monthly_cnt[,v] - pft_monthly_zero_inds = c(pft_monthly_zero_inds, which(cnt_mask_pft == 0)) - # use all weights for all months to include the zero values in the averages - day_weights_pft = (days_in_months) / sum(days_in_months) - avg_pft_weight[v] = sum(pft_weight_monthly_avg[,v] * day_weights_pft) - - # npp - # log the non-zero and zero monthly values - cnt_mask_npp = npp_monthly_cnt[,v] - npp_monthly_zero_inds = c(npp_monthly_zero_inds, which(cnt_mask_npp == 0)) - # use all weights for all months to include the zero values in the averages - day_weights_npp = (days_in_months) / sum(days_in_months) - avg_npp[v] = sum(npp_monthly_avg[,v] * day_weights_npp) - } - # hr - for(v in 1: num_col_ind) { - # log the non-zero and zero monthly values - cnt_mask_hr = hr_monthly_cnt[,v] - hr_monthly_zero_inds = c(hr_monthly_zero_inds, which(cnt_mask_hr == 0)) - # use all weights for all months to include the zero values in the averages - day_weights_hr = (days_in_months) / sum(days_in_months) - avg_hr[v] = sum(hr_monthly_avg[,v] * day_weights_hr) - } - - cat("These missing indices include where the pfts do not exist\n") - if(length(pft_monthly_zero_inds) > 0){ - cat("pft index has", length(pft_monthly_zero_inds), "missing monthly averages\n") - } - if(length(npp_monthly_zero_inds) > 0){ - cat("npp index has", length(npp_monthly_zero_inds), "missing monthly averages\n") - } - if(length(hr_monthly_zero_inds) > 0){ - cat("hr index has", length(hr_monthly_zero_inds), "missing monthly averages\n") - } - - # output the npp, hr, and pft weight data - # use a data frame to organize the data, as they are all output by pft, lon, lat - out_pft_cell = data.frame(pft_id = pft_pft, lon_ind = pft_lon_ind, lat_ind = pft_lat_ind, topounit_ind = pft_topounit_ind, landunit_id = pft_lunit, - npp_gC_per_m2_per_s = avg_npp, pft_wt_cell_frac = avg_pft_weight) - # select only the veg landunit - out_pft_cell = out_pft_cell[out_pft_cell$landunit_id == veg_lunit_id,] - # get unique hr cells for veg land unit - out_hr_veglu = data.frame(lon_ind = col_lon_ind, lat_ind = col_lat_ind, topounit_ind = col_topounit, landunit_id = col_lunit, hr_gC_per_m2_per_s = avg_hr) - out_hr_veglu = out_hr_veglu[out_hr_veglu$landunit_id == veg_lunit_id,] - # again, this shouldn't be necessary - out_hr_veglu = out_hr_veglu[!duplicated(out_hr_veglu[,c(1:4)]),] - # merge these on the cell indices - out_pft_hr = merge(out_pft_cell, out_hr_veglu, by = c("lon_ind", "lat_ind", "topounit_ind", "landunit_id"), all.x = TRUE) - - # check for bad values and change them to zero - # there may not be any bad values cuz the na/NaN values are not included in the weighted average - # any missing data would have zero values - - # npp - npp_inf_inds = which((out_pft_hr$npp_gC_per_m2_per_s == Inf) == TRUE) - npp_neg_inf_inds = which((out_pft_hr$npp_gC_per_m2_per_s == -Inf) == TRUE) - npp_na_inds = which(is.na(out_pft_hr$npp_gC_per_m2_per_s) == TRUE) # this includes NaN - npp_bad_inds = c(npp_inf_inds, npp_neg_inf_inds, npp_na_inds) - if (length(npp_bad_inds) > 0) { - cat("Warning: some bad values have been set to zero\n") - cat("NPP # of bad inds is", length(npp_bad_inds), "\n") - out_pft_hr$npp_gC_per_m2_per_s[npp_bad_inds] = 0 - } - - # also check for negative npp - # negative values are passed cuz they can contribute to positive averages later when aggregated to gcam regions/types - # the final calc scalar code filters out negative npp base values after aggregation because GCAM has only positive yields/accumulation - npp_neg_inds = which((out_pft_hr$npp_gC_per_m2_per_s < 0) == TRUE) - if (length(npp_neg_inds) > 0) { - cat("NPP # of negative inds is", length(npp_neg_inds), "\n") - } - - # pft weight - pftwt_inf_inds = which((out_pft_hr$pft_wt_cell_frac == Inf) == TRUE) - pftwt_neg_inf_inds = which((out_pft_hr$pft_wt_cell_frac == -Inf) == TRUE) - pftwt_na_inds = which(is.na(out_pft_hr$pft_wt_cell_frac) == TRUE) # this includes NaN - pftwt_bad_inds = c(pftwt_inf_inds, pftwt_neg_inf_inds, pftwt_na_inds) - if (length(pftwt_bad_inds) > 0) { - cat("Warning: some bad values have been set to zero\n") - cat("pft wt # of bad inds is", length(pftwt_bad_inds), "\n") - out_pft_hr$pft_wt_cell_frac[pftwt_bad_inds] = 0 - } - - # hr - hr_inf_inds = which((out_pft_hr$hr_gC_per_m2_per_s == Inf) == TRUE) - hr_neg_inf_inds = which((out_pft_hr$hr_gC_per_m2_per_s == -Inf) == TRUE) - hr_na_inds = which(is.na(out_pft_hr$hr_gC_per_m2_per_s) == TRUE) # this includes NaN - hr_bad_inds = c(hr_inf_inds, hr_neg_inf_inds, hr_na_inds) - if (length(hr_bad_inds) > 0) { - cat("Warning: some bad values have been set to zero\n") - cat("HR # of bad inds is", length(hr_bad_inds), "\n") - out_pft_hr$hr_gC_per_m2_per_s[hr_bad_inds] = 0 - } - - out_pft_hr = out_pft_hr[order(out_pft_hr$pft_id, out_pft_hr$lat_ind, out_pft_hr$lon_ind, out_pft_hr$topounit_ind, out_pft_hr$landunit_id),] - - # make complete pftXlonXlat df for writing output files - - # this sets up the full record list - pft_id_out = NULL - lon_ind_out = NULL - lat_ind_out = NULL - area_out = NULL - for (p in 0:(num_pft-1)) { - for (t in 1:num_lat) { - - pft_id_out = c(pft_id_out, rep(p,num_lon)) - lon_ind_out = c(lon_ind_out, 1:num_lon) - lat_ind_out = c(lat_ind_out, rep(t,num_lon)) - # just fill each pft grid here for completeness - area_out = c(area_out, area[,t]) - - } # end for t loop over lat - } # end for loop over pft - - # now merge the data into the complete df and set all NA values to zero - out_df = data.frame(pft_id = pft_id_out, lon_ind = lon_ind_out, lat_ind = lat_ind_out, cell_area_km2 = area_out) - out_df = merge(out_df, out_pft_hr, by = c("pft_id", "lon_ind","lat_ind"), all.x = TRUE) - out_df[is.na(out_df)] = 0 - - # now write the files - - # the output file col names should be, in order, "pft_id", "lon_ind", "lat_ind", "value" - # with lon varying fastest, and pft varying slowest - - npp_out_name = paste0(outdir, out_base_name, avg_tag, "_", year_start, "-", year_end, "_npp.csv") - pft_wt_out_name = paste0(outdir, out_base_name, avg_tag, "_", year_start, "-", year_end, "_pft_wt.csv") - hr_out_name = paste0(outdir, out_base_name, avg_tag, "_", year_start, "-", year_end, "_hr.csv") - area_out_name = paste0(outdir, out_base_name, "_cell_area.csv") - - # npp - write_df = out_df[,c("pft_id", "lon_ind", "lat_ind", "npp_gC_per_m2_per_s")] - write_df = write_df[order(write_df$pft_id, write_df$lat_ind, write_df$lon_ind),] - write.csv(write_df, file = npp_out_name, row.names=FALSE) - - # pft weight - write_df = out_df[,c("pft_id", "lon_ind", "lat_ind", "pft_wt_cell_frac")] - write_df = write_df[order(write_df$pft_id, write_df$lat_ind, write_df$lon_ind),] - write.csv(write_df, file = pft_wt_out_name, row.names=FALSE) - - # hr - write_df = out_df[,c("pft_id", "lon_ind", "lat_ind", "hr_gC_per_m2_per_s")] - write_df = write_df[order(write_df$pft_id, write_df$lat_ind, write_df$lon_ind),] - write.csv(write_df, file = hr_out_name, row.names=FALSE) - - # cell area - write_df = out_df[out_df$pft_id == 0,c("lon_ind", "lat_ind", "cell_area_km2")] - write_df = write_df[order(write_df$lat_ind, write_df$lon_ind),] - write.csv(write_df, file = area_out_name, row.names=FALSE) - - cat("Finish create_e3sm_gcam_land_scalar_baseline_local.r at", date(), "\n") - -} diff --git a/components/gcam/tools/generate_initial_co2_files.sh b/components/gcam/tools/generate_initial_co2_files.sh deleted file mode 100755 index af3af6c13cb5..000000000000 --- a/components/gcam/tools/generate_initial_co2_files.sh +++ /dev/null @@ -1,358 +0,0 @@ -#!/bin/bash - -# Create baseline (2014) gridded CO2 emission files for E3SM-GCAM -# One input argument determines resolution (see below) -# Three main output csv files: aircraft, shipment, surface -# Also generates ssociated netcdf files and some intermediate files - -# this script starts with the 0.5x0.5 CMIP6 CEDS files for historical CO2 emissions -# the final 12 months of 2014 are extracted and processed -# the output is three text files that contain: -# surface co2 emissions (excluding international shipping) at the surface -# aircraft co2 emissions for the defined aggregate levels (currently two) -# international shipping co2 emissions at the surface -# the text files are csv tables with date,lon,lat,value -# values are in co2_kg/m^2/s -# lat,lon are degrees, cell center -# the aircraft text file date includes a level label (yyyymmll) -# this starts at zero for the lowest level - -# the 25 input aircraft levels are in order starting from the surface -# their defined heights are in the file -# define the aggregation of levels here so that changing it for the model simply means updating this file -# the ehc code accepts up to two levels - so make that the default -# the ehc can then sum the two levels if only one level is passed to eam - -# the defined aircraft output levels are: -# lo: 15 bottom levels including 8.845km -# hi: 10 upper levels from 9.455 to 14.945 -# note that GCAM outputs data up to ~11km, ?which is distributed to these 25 levels by ceds? -# 11.285 is the highest large emission level -# 9.455 is where long-range emissions become noticable in the plots - -# there are eight sectors in the other co2 file -# international shipping is id 7 -# sector:ids = "0: Agriculture; 1: Energy; 2: Industrial; 3: Transportation; 4: Residential, Commercial, Other; 5: Solvents production and application; 6: Waste; 7: International Shipping" - -# the final netcdf files that the text is extracted from are retained - -# five output resolutions currently available: -# 0.9x1.25 (aka f09) -# 1.9x2.5 (aka f19) -# 0.5x0.5 (aka r05) -# 0.25x0.25 (aka r025) -# 0.125x0.125 (aka r0125) -# note that no grid remapping is needed for 0.5x0.5 output - -# The desired output resolution is selected by a single argument: -# f09 = 0.9x1.25 -# f19 = 1.9x2.5 -# r05 = 0.5x0.5 -# r025 = 0.25x0.25 -# r0125 = 0.125x0.125 - -if [ "$#" != 1 ]; then - echo "Usage: $0 " - echo "Currently supported resolutions are: f09, f19, r05, r025, and r0215" - exit -fi - -RES=$1 - -date - -# needed modules -#module load intel -#module load nco - -# this gets what is needed also -source /share/apps/E3SM/conda_envs/load_latest_e3sm_unified_compy.sh - -# some useful bash functions - -# ncdmnsz $dmn_nm $fl_nm : What is dimension size? -function ncdmnsz { ncks --trd -m -M ${2} | grep -E -i ": ${1}, size =" | cut -f 7 -d ' ' | uniq ; } -# ncmax $var_nm $fl_nm : What is maximum of variable? -function ncmax { ncap2 -O -C -v -s "foo=${1}.max();print(foo)" ${2} ~/foo.nc | cut -f 3- -d ' ' ; } -# ncmin $var_nm $fl_nm : What is minimum of variable? -function ncmin { ncap2 -O -C -v -s "foo=${1}.min();print(foo)" ${2} ~/foo.nc | cut -f 3- -d ' ' ; } - -proc_dir='/compyfs/inputdata/iac/giac/gcam/gcam_6_0/data/emission_processing/' - -# the original data are at 0.5x0.5 res with origin at -180,-90; and corners aligned with these limits -# this is also how the nomask scrip file is defined for 0.5x0.5 - -# can add more resolutions here - -if [ $RES == 'f09' ]; then - - # f09 - - # grid mapping - map_file="/compyfs/inputdata/lnd/clm2/mappingdata/maps/0.9x1.25/map_0.5x0.5_nomask_to_0.9x1.25_nomask_aave_da_c121019.nc" - - # output nc file names - make sure they match the map file out resolution - sfc_file_out=${proc_dir}'CO2-em-SFC-anthro_0.9x1.25_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_2014.nc' - air_file_out=${proc_dir}'CO2-em-AIR-2lvl-anthro_0.9x1.25_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_2014.nc' - ship_file_out=${proc_dir}'CO2-em-SHIP-anthro_0.9x1.25_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_2014.nc' - - # final output text file names - sfc_text_out=${proc_dir}'CO2-em-SFC-anthro_0.9x1.25_input4MIPs_2014.csv' - air_text_out=${proc_dir}'CO2-em-AIR-2lvl-anthro_0.9x1.25_input4MIPs_2014.csv' - ship_text_out=${proc_dir}'CO2-em-SHIP-anthro_0.9x1.25_input4MIPs_2014.csv' - -elif [ $RES == 'f19' ]; then - - # f19 - - # grid mapping - map_file="/compyfs/inputdata/lnd/clm2/mappingdata/maps/1.9x2.5/map_0.5x0.5_nomask_to_1.9x2.5_nomask_aave_da_c120709.nc" - - # output nc file names - make sure they match the map file out resolution - sfc_file_out=${proc_dir}'CO2-em-SFC-anthro_1.9x2.5_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_2014.nc' - air_file_out=${proc_dir}'CO2-em-AIR-2lvl-anthro_1.9x2.5_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_2014.nc' - ship_file_out=${proc_dir}'CO2-em-SHIP-anthro_1.9x2.5_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_2014.nc' - - # final output text file names - sfc_text_out=${proc_dir}'CO2-em-SFC-anthro_1.9x2.5_input4MIPs_2014.csv' - air_text_out=${proc_dir}'CO2-em-AIR-2lvl-anthro_1.9x2.5_input4MIPs_2014.csv' - ship_text_out=${proc_dir}'CO2-em-SHIP-anthro_1.9x2.5_input4MIPs_2014.csv' - -elif [ $RES == 'r0125' ]; then - - # r0125 - - # grid mapping - map_file="/compyfs/inputdata/lnd/clm2/mappingdata/maps/0.125x0.125/map_0.5x0.5_nomask_to_0.125x0.125_nomask_aave_da_c241205.nc" - - # output nc file names - make sure they match the map file out resolution - sfc_file_out=${proc_dir}'CO2-em-SFC-anthro_0.125x0.125_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_2014.nc' - air_file_out=${proc_dir}'CO2-em-AIR-2lvl-anthro_0.125x0.125_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_2014.nc' - ship_file_out=${proc_dir}'CO2-em-SHIP-anthro_0.125x0.125_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_2014.nc' - - # final output text file names - sfc_text_out=${proc_dir}'CO2-em-SFC-anthro_0.125x0.125_input4MIPs_2014.csv' - air_text_out=${proc_dir}'CO2-em-AIR-2lvl-anthro_0.125x0.125_input4MIPs_2014.csv' - ship_text_out=${proc_dir}'CO2-em-SHIP-anthro_0.125x0.125_input4MIPs_2014.csv' - -elif [ $RES == 'r05' ]; then - - # r05 - - # no grid mapping - map_file="" - - # output nc file names - make sure they match the map file out resolution - sfc_file_out=${proc_dir}'CO2-em-SFC-anthro_0.5x0.5_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_2014.nc' - air_file_out=${proc_dir}'CO2-em-AIR-2lvl-anthro_0.5x0.5_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_2014.nc' - ship_file_out=${proc_dir}'CO2-em-SHIP-anthro_0.5x0.5_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_2014.nc' - - # final output text file names - sfc_text_out=${proc_dir}'CO2-em-SFC-anthro_0.5x0.5_input4MIPs_2014.csv' - air_text_out=${proc_dir}'CO2-em-AIR-2lvl-anthro_0.5x0.5_input4MIPs_2014.csv' - ship_text_out=${proc_dir}'CO2-em-SHIP-anthro_0.5x0.5_input4MIPs_2014.csv' - -elif [ $RES == 'r025' ]; then - - # r025 - - # grid mapping - map_file="/compyfs/inputdata/lnd/clm2/mappingdata/maps/0.25x0.25/map_0.5x0.5_nomask_to_0.25x0.25_nomask_aave_da_c250313.nc" - - # output nc file names - make sure they match the map file out resolution - sfc_file_out=${proc_dir}'CO2-em-SFC-anthro_0.25x0.25_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_2014.nc' - air_file_out=${proc_dir}'CO2-em-AIR-2lvl-anthro_0.25x0.25_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_2014.nc' - ship_file_out=${proc_dir}'CO2-em-SHIP-anthro_0.25x0.25_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_2014.nc' - - # final output text file names - sfc_text_out=${proc_dir}'CO2-em-SFC-anthro_0.25x0.25_input4MIPs_2014.csv' - air_text_out=${proc_dir}'CO2-em-AIR-2lvl-anthro_0.25x0.25_input4MIPs_2014.csv' - ship_text_out=${proc_dir}'CO2-em-SHIP-anthro_0.25x0.25_input4MIPs_2014.csv' - -else - echo "$RES is not supported" - echo "f09, f19, r0125, r025, and r05 are the currently supported output resolutions" - exit -fi - -##### - -# source files -air_file="/compyfs/inputdata/atm/cam/ggas/CO2-em-AIR-anthro_input4MIPs_emissions_CMIP_CEDS-2017-08-30_gn_200001-201412.nc" -other_file="/compyfs/inputdata/atm/cam/ggas/CO2-em-anthro_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_200001-201412.nc" - -# intermediate files -air_file_1y=${proc_dir}'CO2-em-AIR-anthro_input4MIPs_emissions_CMIP_CEDS-2017-08-30_gn_2014.nc' -air_file_1y_agg=${proc_dir}'CO2-em-AIR-2lvl-anthro_input4MIPs_emissions_CMIP_CEDS-2017-08-30_gn_2014.nc' -other_file_1y=${proc_dir}'CO2-em-anthro_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_2014.nc' -ship_file_1y=${proc_dir}'CO2-em-SHIP-anthro_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_2014.nc' -sfc_file_1y=${proc_dir}'CO2-em-SFC-anthro_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_2014.nc' -air_file_lo=${proc_dir}'CO2-em-AIR-LO-anthro_input4MIPs_emissions_CMIP_CEDS-2017-08-30_gn_2014.nc' -air_file_lo_level=${proc_dir}'CO2-em-AIR-LO-level-temp.nc' -air_file_hi=${proc_dir}'CO2-em-AIR-HI-anthro_input4MIPs_emissions_CMIP_CEDS-2017-08-30_gn_2014.nc' -air_file_hi_level=${proc_dir}'CO2-em-AIR-HI-level-temp.nc' - - -# extract year 2014 from the original files, the last 12 months (nces?) -ncea -O -F -d time,169,180 ${air_file} ${air_file_1y} -ncea -O -F -d time,169,180 ${other_file} ${other_file_1y} - -# aircraft level processing - -# aggregate the aircraft data to two levels -ncwa -O -N -b -v CO2_em_AIR_anthro -a level -d level,0,14 ${air_file_1y} ${air_file_lo} -ncwa -O -N -b -v CO2_em_AIR_anthro -a level -d level,15,24 ${air_file_1y} ${air_file_hi} -# Make level record dimension - must put record dimension first for ncrcat to work properly -ncks -O --fix_rec_dmn time ${air_file_lo} ${air_file_lo_level} -ncks -O --mk_rec_dmn level ${air_file_lo_level} ${air_file_lo_level} -ncpdq -O -a level,time,lat,lon ${air_file_lo_level} ${air_file_lo_level} -# Make level record dimension -ncks -O --fix_rec_dmn time ${air_file_hi} ${air_file_hi_level} -ncks -O --mk_rec_dmn level ${air_file_hi_level} ${air_file_hi_level} -ncpdq -O -a level,time,lat,lon ${air_file_hi_level} ${air_file_hi_level} -# concatenate along level -ncrcat -O ${air_file_lo_level} ${air_file_hi_level} ${air_file_1y_agg} -# revert time to record dimension -ncks -O --fix_rec_dmn level ${air_file_1y_agg} ${air_file_1y_agg} -ncks -O --mk_rec_dmn time ${air_file_1y_agg} ${air_file_1y_agg} -ncpdq -O -a time,level,lat,lon ${air_file_1y_agg} ${air_file_1y_agg} - -rm ${air_file_lo} -rm ${air_file_hi} -rm ${air_file_lo_level} -rm ${air_file_hi_level} - -# extract surface shipping data -ncea -O -d sector,7,7 ${other_file_1y} ${ship_file_1y} - -# extract non-shipping surface data summed across remaining sectors -ncwa -O -N -v CO2_em_anthro -a sector -d sector,0,6 ${other_file_1y} ${sfc_file_1y} - -# remap the data to the desired grid -# not needed for 0.5x0.5 -if [ $RES != 'r05' ]; then - ncremap -m ${map_file} ${sfc_file_1y} ${sfc_file_out} - ncremap -m ${map_file} ${air_file_1y_agg} ${air_file_out} - ncremap -m ${map_file} ${ship_file_1y} ${ship_file_out} -elif [ $RES == 'r05' ]; then - cp ${sfc_file_1y} ${sfc_file_out} - cp ${air_file_1y_agg} ${air_file_out} - cp ${ship_file_1y} ${ship_file_out} -fi - -# convert time variable from days since 1750-01-01 to month in yyyymm format -# the time value is the day in the middle of the month, so the simple math below works -# only need to do this once -ncap2 -O -s "time=int(trunc(float(time/365.0)+1750.0)*100 + trunc(((float(time/365.0)+1750.0)-trunc(float(time/365.0)+1750.0))*365.0/30.0+1.0))" ${sfc_file_out} ${sfc_file_out} - -# output text files -# need nested loop for this to work properly - -num_time=$(ncdmnsz time ${sfc_file_out}) -num_lat=$(ncdmnsz lat ${sfc_file_out}) -num_lon=$(ncdmnsz lon ${sfc_file_out}) - -# write separate files then paste them together -# they need to be line by line and have the same length -# so still need the time-lat loops to create the full length time, lat, and lon files -# remove the blank lines from files -# the netcdf time, lat, and lon are the same for all files - -tin=`ncks -C -H -v time -s "%i " ${sfc_file_out}` -ain=`ncks -C -H -v lat -s "%f " ${sfc_file_out}` -ncks -O -C -H -v lon -s "%f\n" ${sfc_file_out} > ${proc_dir}'temp1_lon.txt' -tr -s '\n' < ${proc_dir}'temp1_lon.txt' > ${proc_dir}'temp_lon.txt' -rm ${proc_dir}'temp1_lon.txt' - -# the values are floats but we need full double precision in text to get the values (17 decimals for double) - -ncks -O -C -H -v CO2_em_anthro -s "%.17f\n" ${sfc_file_out} > ${proc_dir}'temp_value.txt' -tr -s '\n' < ${proc_dir}'temp_value.txt' > ${proc_dir}'sfc_value.txt' - -ncks -O -C -H -v CO2_em_anthro -s "%.17f\n" ${ship_file_out} > ${proc_dir}'temp_value.txt' -tr -s '\n' < ${proc_dir}'temp_value.txt' > ${proc_dir}'ship_value.txt' - -# lo air level -ncks -O -d level,0,0 -C -H -v CO2_em_AIR_anthro -s "%.17f\n" ${air_file_out} > ${proc_dir}'temp_value.txt' -tr -s '\n' < ${proc_dir}'temp_value.txt' > ${proc_dir}'air_lo_value.txt' - -# hi air level -ncks -O -d level,1,1 -C -H -v CO2_em_AIR_anthro -s "%.17f\n" ${air_file_out} > ${proc_dir}'temp_value.txt' -tr -s '\n' < ${proc_dir}'temp_value.txt' > ${proc_dir}'air_hi_value.txt' - -# this is the spacer for the time and lat variables -IFS=" " -tz=( $tin ) -az=( $ain ) - -# make sure that these are new, clean files -> time.txt -> lat.txt -> lon.txt - -for ((t=0 ; t<$num_time ; t++)); -do - - for ((a=0 ; a<$num_lat ; a++)); - do - - # write repeated appropriate time and lat and lon - yes ${tz[$t]} | head -n ${num_lon} >> ${proc_dir}'time.txt' - # the -- tells that the next argument is an input, not an option, so that negative numbers can be used - yes -- ${az[$a]} | head -n ${num_lon} >> ${proc_dir}'lat.txt' - dd if=${proc_dir}'temp_lon.txt' bs=1M status=none >> ${proc_dir}'lon.txt' - - done - -done - -wc -l ${proc_dir}'sfc_value.txt' -wc -l ${proc_dir}'ship_value.txt' -wc -l ${proc_dir}'air_lo_value.txt' -wc -l ${proc_dir}'air_hi_value.txt' -wc -l ${proc_dir}'time.txt' -wc -l ${proc_dir}'lat.txt' -wc -l ${proc_dir}'lon.txt' - -# surface csv file - -# no quotes enters the text exactly -echo yyyymm,lon_deg,lat_deg,co2_kg/m2/s > ${sfc_text_out} -# - represents piped input -paste -d "," ${proc_dir}'time.txt' ${proc_dir}'lon.txt' | paste -d "," - ${proc_dir}'lat.txt' | paste -d "," - ${proc_dir}'sfc_value.txt' >> ${sfc_text_out} -$(echo wc -l ${sfc_text_out}) - - -# ship csv file - -echo yyyymm,lon_deg,lat_deg,co2_kg/m2/s > ${ship_text_out} -paste -d "," ${proc_dir}'time.txt' ${proc_dir}'lon.txt' | paste -d "," - ${proc_dir}'lat.txt' | paste -d "," - ${proc_dir}'ship_value.txt' >> ${ship_text_out} -$(echo wc -l ${ship_text_out}) - - -# aircraft csv file - -# write the low level, then the high level -# add a level tag to the first column for lo (00) and hi (01) -echo yyyymmll,lon_deg,lat_deg,co2_kg/m2/s > ${air_text_out} - -sed "s/$/00/" ${proc_dir}'time.txt' > ${proc_dir}'temp_time.txt' -paste -d "," ${proc_dir}'temp_time.txt' ${proc_dir}'lon.txt' | paste -d "," - ${proc_dir}'lat.txt' | paste -d "," - ${proc_dir}'air_lo_value.txt' >> ${air_text_out} - -sed "s/$/01/" ${proc_dir}'time.txt' > ${proc_dir}'temp_time.txt' -paste -d "," ${proc_dir}'temp_time.txt' ${proc_dir}'lon.txt' | paste -d "," - ${proc_dir}'lat.txt' | paste -d "," - ${proc_dir}'air_hi_value.txt' >> ${air_text_out} - -$(echo wc -l ${air_text_out}) - -rm ${proc_dir}'time.txt' -rm ${proc_dir}'temp_time.txt' -rm ${proc_dir}'lat.txt' -rm ${proc_dir}'lon.txt' -rm ${proc_dir}'temp_lon.txt' -rm ${proc_dir}'sfc_value.txt' -rm ${proc_dir}'ship_value.txt' -rm ${proc_dir}'air_lo_value.txt' -rm ${proc_dir}'air_hi_value.txt' -rm ${proc_dir}'temp_value.txt' - -date diff --git a/components/gcam/tools/regrid_popden_files.sh b/components/gcam/tools/regrid_popden_files.sh deleted file mode 100644 index 45820802c1f7..000000000000 --- a/components/gcam/tools/regrid_popden_files.sh +++ /dev/null @@ -1,153 +0,0 @@ -#!/bin/bash - -# Create multiple population density files for E3SM-GCAM at different resolutions -# One input argument determines resolution (see below) -# These are all the same format as the original 0.5x0.5 files, but regridded to the desired resolution -# This format is what is used for the population density ELM files for the fire model, which are specified by namelist - -# this script starts with the 0.5x0.5 ssp popluation density netcdf files -# all 5 ssp files are processed in the same way, using ncremap, but with different output names - -# five output resolutions currently available: -# 0.9x1.25 (aka f09) -# 1.9x2.5 (aka f19) -# 0.5x0.5 (aka r05) -# 0.25x0.25 (aka r025) -# 0.125x0.125 (aka r0125) -# note that no grid remapping is needed for 0.5x0.5 output - -# The desired output resolution is selected by a single argument: -# f09 = 0.9x1.25 -# f19 = 1.9x2.5 -# r05 = 0.5x0.5 -# r025 = 0.25x0.25 -# r0125 = 0.125x0.125 - -if [ "$#" != 1 ]; then - echo "Usage: $0 " - echo "Currently supported resolutions are: f09, f19, r05, r025, and r0215" - exit -fi - -RES=$1 - -date - -# needed modules -#module load intel -#module load nco - -# this gets what is needed also -source /share/apps/E3SM/conda_envs/load_latest_e3sm_unified_compy.sh - -proc_dir='/compyfs/inputdata/iac/giac/iac2gcam/' - -# the original data are at 0.5x0.5 res with origin at -180,-90; and corners aligned with these limits -# this is also how the nomask scrip file is defined for 0.5x0.5 - -# can add more resolutions here - -if [ $RES == 'f09' ]; then - - # f09 - - # grid mapping - map_file="/compyfs/inputdata/lnd/clm2/mappingdata/maps/0.9x1.25/map_0.5x0.5_nomask_to_0.9x1.25_nomask_aave_da_c121019.nc" - - # output nc file names - make sure they match the map file out resolution - ssp1_file_out=${proc_dir}'elmforc.ssp1_hdm_0.9x1.25_simyr1850-2101_c'$(date +%Y%m%d)'.nc' - ssp2_file_out=${proc_dir}'elmforc.ssp2_hdm_0.9x1.25_simyr1850-2101_c'$(date +%Y%m%d)'.nc' - ssp3_file_out=${proc_dir}'elmforc.ssp3_hdm_0.9x1.25_simyr1850-2101_c'$(date +%Y%m%d)'.nc' - ssp4_file_out=${proc_dir}'elmforc.ssp4_hdm_0.9x1.25_simyr1850-2101_c'$(date +%Y%m%d)'.nc' - ssp5_file_out=${proc_dir}'elmforc.ssp5_hdm_0.9x1.25_simyr1850-2101_c'$(date +%Y%m%d)'.nc' - -elif [ $RES == 'f19' ]; then - - # f19 - - # grid mapping - map_file="/compyfs/inputdata/lnd/clm2/mappingdata/maps/1.9x2.5/map_0.5x0.5_nomask_to_1.9x2.5_nomask_aave_da_c120709.nc" - - # output nc file names - make sure they match the map file out resolution - ssp1_file_out=${proc_dir}'elmforc.ssp1_hdm_1.9x2.5_simyr1850-2101_c'$(date +%Y%m%d)'.nc' - ssp2_file_out=${proc_dir}'elmforc.ssp2_hdm_1.9x2.5_simyr1850-2101_c'$(date +%Y%m%d)'.nc' - ssp3_file_out=${proc_dir}'elmforc.ssp3_hdm_1.9x2.5_simyr1850-2101_c'$(date +%Y%m%d)'.nc' - ssp4_file_out=${proc_dir}'elmforc.ssp4_hdm_1.9x2.5_simyr1850-2101_c'$(date +%Y%m%d)'.nc' - ssp5_file_out=${proc_dir}'elmforc.ssp5_hdm_1.9x2.5_simyr1850-2101_c'$(date +%Y%m%d)'.nc' - -elif [ $RES == 'r0125' ]; then - - # r0125 - - # grid mapping - map_file="/compyfs/inputdata/lnd/clm2/mappingdata/maps/0.125x0.125/map_0.5x0.5_nomask_to_0.125x0.125_nomask_aave_da_c241205.nc" - - # output nc file names - make sure they match the map file out resolution - ssp1_file_out=${proc_dir}'elmforc.ssp1_hdm_0.125x0.125_simyr1850-2101_c'$(date +%Y%m%d)'.nc' - ssp2_file_out=${proc_dir}'elmforc.ssp2_hdm_0.125x0.125_simyr1850-2101_c'$(date +%Y%m%d)'.nc' - ssp3_file_out=${proc_dir}'elmforc.ssp3_hdm_0.125x0.125_simyr1850-2101_c'$(date +%Y%m%d)'.nc' - ssp4_file_out=${proc_dir}'elmforc.ssp4_hdm_0.125x0.125_simyr1850-2101_c'$(date +%Y%m%d)'.nc' - ssp5_file_out=${proc_dir}'elmforc.ssp5_hdm_0.125x0.125_simyr1850-2101_c'$(date +%Y%m%d)'.nc' - -elif [ $RES == 'r05' ]; then - - # r05 - - # no grid mapping - map_file="" - - # output nc file names - make sure they match the map file out resolution - ssp1_file_out=${proc_dir}'elmforc.ssp1_hdm_0.5x0.5_simyr1850-2101_c'$(date +%Y%m%d)'.nc' - ssp2_file_out=${proc_dir}'elmforc.ssp2_hdm_0.5x0.5_simyr1850-2101_c'$(date +%Y%m%d)'.nc' - ssp3_file_out=${proc_dir}'elmforc.ssp3_hdm_0.5x0.5_simyr1850-2101_c'$(date +%Y%m%d)'.nc' - ssp4_file_out=${proc_dir}'elmforc.ssp4_hdm_0.5x0.5_simyr1850-2101_c'$(date +%Y%m%d)'.nc' - ssp5_file_out=${proc_dir}'elmforc.ssp5_hdm_0.5x0.5_simyr1850-2101_c'$(date +%Y%m%d)'.nc' - -elif [ $RES == 'r025' ]; then - - # r025 - - # grid mapping - map_file="/compyfs/inputdata/lnd/clm2/mappingdata/maps/0.25x0.25/map_0.5x0.5_nomask_to_0.25x0.25_nomask_aave_da_c250313.nc" - - # output nc file names - make sure they match the map file out resolution - ssp1_file_out=${proc_dir}'elmforc.ssp1_hdm_0.25x0.25_simyr1850-2101_c'$(date +%Y%m%d)'.nc' - ssp2_file_out=${proc_dir}'elmforc.ssp2_hdm_0.25x0.25_simyr1850-2101_c'$(date +%Y%m%d)'.nc' - ssp3_file_out=${proc_dir}'elmforc.ssp3_hdm_0.25x0.25_simyr1850-2101_c'$(date +%Y%m%d)'.nc' - ssp4_file_out=${proc_dir}'elmforc.ssp4_hdm_0.25x0.25_simyr1850-2101_c'$(date +%Y%m%d)'.nc' - ssp5_file_out=${proc_dir}'elmforc.ssp5_hdm_0.25x0.25_simyr1850-2101_c'$(date +%Y%m%d)'.nc' - -else - echo "$RES is not supported" - echo "f09, f19, r0125, r025, and r05 are the currently supported output resolutions" - exit -fi - -##### - -# source files -ssp1_source_file="/compyfs/inputdata/lnd/clm2/firedata/elmforc.ssp1_hdm_0.5x0.5_simyr1850-2101_c20200624.nc" -ssp2_source_file="/compyfs/inputdata/lnd/clm2/firedata/elmforc.ssp2_hdm_0.5x0.5_simyr1850-2101_c20200623.nc" -ssp3_source_file="/compyfs/inputdata/lnd/clm2/firedata/elmforc.ssp3_hdm_0.5x0.5_simyr1850-2101_c20200624.nc" -ssp4_source_file="/compyfs/inputdata/lnd/clm2/firedata/elmforc.ssp4_hdm_0.5x0.5_simyr1850-2101_c20200624.nc" -ssp5_source_file="/compyfs/inputdata/lnd/clm2/firedata/elmforc.ssp5_hdm_0.5x0.5_simyr1850-2100_c190109.nc" - - -# remap the data to the desired grid -# not needed for 0.5x0.5 -if [ $RES != 'r05' ]; then - ncremap -m ${map_file} ${ssp1_source_file} ${ssp1_file_out} - ncremap -m ${map_file} ${ssp2_source_file} ${ssp2_file_out} - ncremap -m ${map_file} ${ssp3_source_file} ${ssp3_file_out} - ncremap -m ${map_file} ${ssp4_source_file} ${ssp4_file_out} - ncremap -m ${map_file} ${ssp5_source_file} ${ssp5_file_out} -elif [ $RES == 'r05' ]; then - cp ${ssp1_source_file} ${ssp1_file_out} - cp ${ssp2_source_file} ${ssp2_file_out} - cp ${ssp3_source_file} ${ssp3_file_out} - cp ${ssp4_source_file} ${ssp4_file_out} - cp ${ssp5_source_file} ${ssp5_file_out} -fi - -date - diff --git a/components/gcam/tools/run_v3_SSP245_ZATM_BGC_ne30pg2_f09_oEC60to30v3.sh b/components/gcam/tools/run_v3_SSP245_ZATM_BGC_ne30pg2_f09_oEC60to30v3.sh index 1c1d868d55b8..f7459e95d73d 100755 --- a/components/gcam/tools/run_v3_SSP245_ZATM_BGC_ne30pg2_f09_oEC60to30v3.sh +++ b/components/gcam/tools/run_v3_SSP245_ZATM_BGC_ne30pg2_f09_oEC60to30v3.sh @@ -58,7 +58,10 @@ readonly MYDATE=$(date '+%Y%m%d%H') # use current date if MYDATE is not set to a # export COMPSET=SSP245_EAM%CMIP6_ELM%TOPCNPRDCTCBCPHS_MPASSI%PRES_DOCN%DOM_SROF_SGLC_SWAV_GCAM_BGC%LNDATM readonly COMPSET="SSP245_ZATM_BGC" # see long name above readonly RESOLUTION="ne30pg2_f09_oEC60to30v3" -readonly CASE_NAME="${COMPSET}_${RESOLUTION}_${MYDATE}" +readonly RESABBREV="n30p2_f09_EC30" +readonly CONFIG="TER_DEG_FDBK" # CONTROL, TER_FDBK, TER_DEG_FDBK, TER_DEG_FDBK_READ +readonly REFINIYEAR=2015 +readonly CASE_NAME="${COMPSET}_${CONFIG}_INI${REFINIYEAR}_${MYDATE}_${RESABBREV}" # readonly CASE_GROUP="E3SM_GCAM" # set the machine inputdata directory, scratch directory, and queue names @@ -67,18 +70,21 @@ if [ "${MACHINE}" == "chrysalis" ]; then readonly CASE_ROOT="/lcrc/group/e3sm/$USER/e3sm_scratch/${CASE_NAME}" readonly MACH_QUEUE='compute' readonly MACH_QUEUE_DEBUG='debug' + readonly WALLTIME_DEBUG='4:00:00' fi if [ "${MACHINE}" == "compy" ]; then readonly din_loc_root=/compyfs/inputdata readonly CASE_ROOT="/compyfs/${USER}/e3sm_scratch/${CASE_NAME}" readonly MACH_QUEUE='slurm' readonly MACH_QUEUE_DEBUG='short' + readonly WALLTIME_DEBUG='0:30:00' fi if [ "${MACHINE}" == "pm-cpu" ]; then din_loc_root=/global/cfs/cdirs/e3sm/inputdata readonly CASE_ROOT="${SCRATCH}/e3sm_scratch/${CASE_NAME}" readonly MACH_QUEUE='regular' readonly MACH_QUEUE_DEBUG='debug' + readonly WALLTIME_DEBUG='0:30:00' fi # Sub-directories @@ -94,7 +100,7 @@ readonly START_DATE="2015-01-01" # Additional options for 'branch' and 'hybrid' readonly GET_REFCASE=TRUE -readonly RUN_REFCASE="20260303_I20TREAMELMCNPRDCTCBCPHSBGC_${RESOLUTION}" +readonly RUN_REFCASE="20260624_I20TREAMELMCNPRDCTCBCPHSBGC_${RESOLUTION}" readonly RUN_REFDATE="2015-01-01" readonly RUN_REFDIR="$din_loc_root/e3sm_init/${RUN_REFCASE}/${RUN_REFDATE}-00000" readonly MPASSI_CONFIG_START=2015-01-01_0 @@ -128,8 +134,8 @@ if [ "${run}" != "production" ]; then readonly REST_N=${STOP_N} readonly RESUBMIT=${resubmit} readonly DO_SHORT_TERM_ARCHIVING=false - - readonly WALLTIME="4:00:00" + + readonly WALLTIME=${WALLTIME_DEBUG:-0:30:00} readonly RUN_QUEUE=${MACH_QUEUE_DEBUG} else @@ -200,7 +206,6 @@ cat << EOF >> user_nl_eam co2_print_diags_timestep = .true. co2_print_diags_monthly = .true. co2_print_diags_total = .true. - cflx_cpl_opt=1 ncdata = '${ncd_string}' EOF @@ -218,6 +223,7 @@ EOF cat << EOF >> user_nl_gcam !read_scalars = .true. !scalar_source_dir = '' +!read_hdd_cdd = .true. EOF cat << EOF >> user_nl_cpl @@ -446,8 +452,8 @@ modify_pe_layout() { ./xmlchange NTASKS_ATM=$(($ppn * $nnodes)) ./xmlchange NTASKS_CPL=$(($ppn * $nnodes)) ./xmlchange NTASKS_LND=$(($ppn * $nnodes)) - ./xmlchange NTASKS_ICE=256 - ./xmlchange NTASKS_OCN=256 + ./xmlchange NTASKS_ICE=$(($ppn * $nnodes)) + ./xmlchange NTASKS_OCN=$(($ppn * $nnodes)) ./xmlchange NTASKS_ROF=1 else echo 'ERROR: $PELAYOUT = '${PELAYOUT}' but no layout exists for this setting.' diff --git a/components/homme/cmake/machineFiles/aurora-aot.cmake b/components/homme/cmake/machineFiles/aurora-aot.cmake index 28cced899c4d..fb594020e8e5 100644 --- a/components/homme/cmake/machineFiles/aurora-aot.cmake +++ b/components/homme/cmake/machineFiles/aurora-aot.cmake @@ -37,21 +37,19 @@ SET(Kokkos_ENABLE_SYCL ON CACHE BOOL "") SET(Kokkos_ENABLE_DEBUG OFF CACHE BOOL "") SET(Kokkos_ENABLE_DEBUG_BOUNDS_CHECK OFF CACHE BOOL "") -SET(CMAKE_CXX_STANDARD 17) - SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") #AOT flags -#SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda -Xclang -fsycl-allow-virtual-functions") -SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") +#SET(SYCL_COMPILE_FLAGS "-fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda -Xclang -fsycl-allow-virtual-functions") +SET(SYCL_COMPILE_FLAGS "-fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") SET(SYCL_LINK_FLAGS "-Wl,--no-relax -flink-huge-device-code -fsycl-max-parallel-link-jobs=32 -fsycl -fsycl-device-code-split=per_kernel -fsycl-targets=intel_gpu_pvc") SET(ADD_Fortran_FLAGS "-fc=ifx -fpscomp logicals -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "") SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "") -SET(ADD_CXX_FLAGS "-std=c++17 -fp-model=precise -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "") +SET(ADD_CXX_FLAGS "-fp-model=precise -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "") SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "") set (ENABLE_OPENMP OFF CACHE BOOL "") diff --git a/components/homme/cmake/machineFiles/aurora-jit.cmake b/components/homme/cmake/machineFiles/aurora-jit.cmake index c8658cde323a..1cc8c9ad6e71 100644 --- a/components/homme/cmake/machineFiles/aurora-jit.cmake +++ b/components/homme/cmake/machineFiles/aurora-jit.cmake @@ -27,20 +27,18 @@ SET(USE_TRILINOS OFF CACHE BOOL "") SET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "") -SET(CMAKE_CXX_STANDARD 17) - SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") #AOT flags -SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") +SET(SYCL_COMPILE_FLAGS "-fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") SET(SYCL_LINK_FLAGS "-fsycl-max-parallel-link-jobs=32 -fsycl") SET(ADD_Fortran_FLAGS "-fc=ifx -fpscomp logicals -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "") SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "") -SET(ADD_CXX_FLAGS "-std=c++17 -fp-model=precise -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "") +SET(ADD_CXX_FLAGS "-fp-model=precise -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "") SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "") set (ENABLE_OPENMP OFF CACHE BOOL "") diff --git a/components/homme/cmake/machineFiles/chrysalis-bfb.cmake b/components/homme/cmake/machineFiles/chrysalis-bfb.cmake index d439346f9c81..cdccd6706d1e 100644 --- a/components/homme/cmake/machineFiles/chrysalis-bfb.cmake +++ b/components/homme/cmake/machineFiles/chrysalis-bfb.cmake @@ -76,3 +76,6 @@ ELSE() SET (HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "") ENDIF() +#FIXME: Remove when updating to Kokkos 5, then +# homme will inherit c++20 from Kokkos. +set(CMAKE_CXX_STANDARD 20) diff --git a/components/homme/cmake/machineFiles/chrysalis.cmake b/components/homme/cmake/machineFiles/chrysalis.cmake index e8e818d955b2..48d06eabaec3 100644 --- a/components/homme/cmake/machineFiles/chrysalis.cmake +++ b/components/homme/cmake/machineFiles/chrysalis.cmake @@ -75,3 +75,6 @@ ELSE() SET (HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "") ENDIF() +#FIXME: Remove when updating to Kokkos 5, then +# homme will inherit c++20 from Kokkos. +set(CMAKE_CXX_STANDARD 20) diff --git a/components/homme/cmake/machineFiles/pm-cpu-bfb.cmake b/components/homme/cmake/machineFiles/pm-cpu-bfb.cmake index 9d7a9dd21f2c..bc57989d44dd 100644 --- a/components/homme/cmake/machineFiles/pm-cpu-bfb.cmake +++ b/components/homme/cmake/machineFiles/pm-cpu-bfb.cmake @@ -139,3 +139,7 @@ SET(USE_NUM_PROCS 24 CACHE STRING "") # only default SET(USE_MPIEXEC "srun" CACHE STRING "") SET(USE_MPI_OPTIONS "-K --cpu-bind=cores" CACHE STRING "") SET(HOMME_TESTING_TIMELIMIT 1800 CACHE STRING "") + +#FIXME: Remove when updating to Kokkos 5, then +# homme will inherit c++20 from Kokkos. +set(CMAKE_CXX_STANDARD 20) diff --git a/components/homme/cmake/machineFiles/pm-cpu.cmake b/components/homme/cmake/machineFiles/pm-cpu.cmake index 84cb0875e7b9..88c398633d2e 100644 --- a/components/homme/cmake/machineFiles/pm-cpu.cmake +++ b/components/homme/cmake/machineFiles/pm-cpu.cmake @@ -85,3 +85,7 @@ SET(USE_NUM_PROCS 24 CACHE STRING "") # only default SET(USE_MPIEXEC "srun" CACHE STRING "") SET(USE_MPI_OPTIONS "-K --cpu-bind=cores" CACHE STRING "") SET(HOMME_TESTING_TIMELIMIT 1800 CACHE STRING "") + +#FIXME: Remove when updating to Kokkos 5, then +# homme will inherit c++20 from Kokkos. +set(CMAKE_CXX_STANDARD 20) diff --git a/components/homme/cmake/machineFiles/spot-aot-AB2.cmake b/components/homme/cmake/machineFiles/spot-aot-AB2.cmake index 2ab993b3c80e..500fc4435375 100644 --- a/components/homme/cmake/machineFiles/spot-aot-AB2.cmake +++ b/components/homme/cmake/machineFiles/spot-aot-AB2.cmake @@ -32,21 +32,18 @@ SET(USE_TRILINOS OFF CACHE BOOL "") SET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "") -#SET(CMAKE_CXX_STANDARD 17) -SET(CMAKE_CXX_STANDARD 17 CACHE STRING "CXX Standard") - SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") -SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") +SET(SYCL_COMPILE_FLAGS "-fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") SET(SYCL_LINK_FLAGS "-fsycl-max-parallel-link-jobs=32 -fsycl-link-huge-device-code -fsycl -fsycl-device-code-split=per_kernel -fsycl-targets=spir64_gen -Xsycl-target-backend \"-device 12.60.7\"") #-fpscomp does not actually solve the issue with bools in here,another suggestion was -fp-model=precise, not working either SET(ADD_Fortran_FLAGS " -fc=ifx -fpscomp logicals -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "") SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "") -SET(ADD_CXX_FLAGS " -std=c++17 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "") +SET(ADD_CXX_FLAGS " -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "") SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "") set (ENABLE_OPENMP OFF CACHE BOOL "") diff --git a/components/homme/src/preqx/share/viscosity_preqx_base.F90 b/components/homme/src/preqx/share/viscosity_preqx_base.F90 index 0fe3f54bc83d..f226eb382e99 100644 --- a/components/homme/src/preqx/share/viscosity_preqx_base.F90 +++ b/components/homme/src/preqx/share/viscosity_preqx_base.F90 @@ -58,12 +58,9 @@ subroutine biharmonic_wk_dp3d(elem,dptens,ptens,vtens,deriv,edge3,hybrid,nt,nets real (kind=real_kind), dimension(np,np) :: tmp2 real (kind=real_kind), dimension(np,np,2) :: v real (kind=real_kind) :: nu_ratio1, nu_ratio2 -logical var_coef1 !if tensor hyperviscosity with tensor V is used, then biharmonic operator is (\grad\cdot V\grad) (\grad \cdot \grad) !so tensor is only used on second call to laplace_sphere_wk - var_coef1 = .true. - if(hypervis_scaling > 0) var_coef1 = .false. ! note: there is a scaling bug in the treatment of nu_div ! nu_ratio is applied twice, once in each laplace operator @@ -92,11 +89,11 @@ subroutine biharmonic_wk_dp3d(elem,dptens,ptens,vtens,deriv,edge3,hybrid,nt,nets #endif do k=1,nlev tmp=elem(ie)%state%T(:,:,k,nt) - ptens(:,:,k,ie)=laplace_sphere_wk(tmp,deriv,elem(ie),var_coef=var_coef1) + ptens(:,:,k,ie)=laplace_sphere_wk(tmp,deriv,elem(ie),var_coef=.false.) tmp=elem(ie)%state%dp3d(:,:,k,nt) - dptens(:,:,k,ie)=laplace_sphere_wk(tmp,deriv,elem(ie),var_coef=var_coef1) + dptens(:,:,k,ie)=laplace_sphere_wk(tmp,deriv,elem(ie),var_coef=.false.) vtens(:,:,:,k,ie)=vlaplace_sphere_wk(elem(ie)%state%v(:,:,:,k,nt),deriv,elem(ie),& - var_coef=var_coef1,nu_ratio=nu_ratio1) + var_coef=.false.,nu_ratio=nu_ratio1) enddo kptr=0 call edgeVpack_nlyr(edge3, elem(ie)%desc, ptens(1,1,1,ie),nlev,kptr,4*nlev) @@ -127,13 +124,13 @@ subroutine biharmonic_wk_dp3d(elem,dptens,ptens,vtens,deriv,edge3,hybrid,nt,nets #endif do k=1,nlev tmp(:,:)=rspheremv(:,:)*ptens(:,:,k,ie) - ptens(:,:,k,ie)=laplace_sphere_wk(tmp,deriv,elem(ie),var_coef=.true.) + ptens(:,:,k,ie)=laplace_sphere_wk(tmp,deriv,elem(ie),var_coef=(hypervis_scaling>0)) tmp2(:,:)=rspheremv(:,:)*dptens(:,:,k,ie) - dptens(:,:,k,ie)=laplace_sphere_wk(tmp2,deriv,elem(ie),var_coef=.true.) + dptens(:,:,k,ie)=laplace_sphere_wk(tmp2,deriv,elem(ie),var_coef=(hypervis_scaling>0)) v(:,:,1)=rspheremv(:,:)*vtens(:,:,1,k,ie) v(:,:,2)=rspheremv(:,:)*vtens(:,:,2,k,ie) vtens(:,:,:,k,ie)=vlaplace_sphere_wk(v(:,:,:),deriv,elem(ie),& - var_coef=.true.,nu_ratio=nu_ratio2) + var_coef=(hypervis_scaling>0),nu_ratio=nu_ratio2) enddo enddo diff --git a/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp b/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp index b433a48c2abc..349ccffaadcf 100644 --- a/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp +++ b/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp @@ -239,8 +239,7 @@ void init_elements_c (const int& num_elems) Elements& e = c.create (); const SimulationParams& params = c.get(); - const bool consthv = (params.hypervis_scaling==0.0); - e.init (num_elems, consthv, /* alloc_gradphis = */ false, + e.init (num_elems, /* alloc_gradphis = */ false, params.scale_factor, params.laplacian_rigid_factor, /* alloc_sphere_coords = */ false); @@ -309,13 +308,17 @@ void init_functors_c () void init_elements_2d_c (const int& ie, CF90Ptr& D, CF90Ptr& Dinv, CF90Ptr& fcor, CF90Ptr& spheremp, CF90Ptr& rspheremp, CF90Ptr& metdet, CF90Ptr& metinv, CF90Ptr& phis, - CF90Ptr &tensorvisc, CF90Ptr &vec_sph2cart) + CF90Ptr &tensorvisc, CF90Ptr &vec_sph2cart, + CF90Ptr &tensorvisc2) { Elements& e = Context::singleton().get (); - const SimulationParams& params = Context::singleton().get(); - const bool consthv = (params.hypervis_scaling==0.0); - e.m_geometry.set_elem_data(ie,D,Dinv,fcor,spheremp,rspheremp,metdet,metinv,tensorvisc,vec_sph2cart,consthv); + // preqx_kokkos never uses tensorVisc_2 (the sponge-layer tensor + // coefficient, only used by theta-l_kokkos), but it is passed through + // here (and copied into m_tensorvisc2) so that ElementsGeometry's + // initialization is uniform across dycores. + e.m_geometry.set_elem_data(ie,D,Dinv,fcor,spheremp,rspheremp,metdet,metinv,tensorvisc, + vec_sph2cart,nullptr,nullptr,tensorvisc2); e.m_geometry.set_phis(ie,phis); } diff --git a/components/homme/src/preqx_kokkos/preqx_f2c_mod.F90 b/components/homme/src/preqx_kokkos/preqx_f2c_mod.F90 index 4d743a09954d..0f9a209a4bab 100644 --- a/components/homme/src/preqx_kokkos/preqx_f2c_mod.F90 +++ b/components/homme/src/preqx_kokkos/preqx_f2c_mod.F90 @@ -60,7 +60,8 @@ subroutine init_elements_2d_c (ie, D_ptr, Dinv_ptr, elem_fcor_ptr, & elem_spheremp_ptr, elem_rspheremp_ptr, & elem_metdet_ptr, elem_metinv_ptr, & phis_ptr, & - tensorvisc_ptr, vec_sph2cart_ptr) bind(c) + tensorvisc_ptr, vec_sph2cart_ptr, & + tensorvisc2_ptr) bind(c) use iso_c_binding, only: c_int, c_ptr ! ! Inputs @@ -70,6 +71,7 @@ subroutine init_elements_2d_c (ie, D_ptr, Dinv_ptr, elem_fcor_ptr, & type (c_ptr) , intent(in) :: elem_spheremp_ptr, elem_rspheremp_ptr type (c_ptr) , intent(in) :: elem_metdet_ptr, elem_metinv_ptr, phis_ptr type (c_ptr) , intent(in) :: tensorvisc_ptr, vec_sph2cart_ptr + type (c_ptr) , intent(in) :: tensorvisc2_ptr end subroutine init_elements_2d_c ! Initializes C++ diagnostics arrays with ptrs provided from f90 diff --git a/components/homme/src/preqx_kokkos/prim_driver_mod.F90 b/components/homme/src/preqx_kokkos/prim_driver_mod.F90 index 85748eefc939..4ab82290fec5 100644 --- a/components/homme/src/preqx_kokkos/prim_driver_mod.F90 +++ b/components/homme/src/preqx_kokkos/prim_driver_mod.F90 @@ -130,11 +130,13 @@ subroutine prim_init_elements_views (elem) type (c_ptr) :: elem_state_q_ptr, elem_state_Qdp_ptr type (c_ptr) :: elem_accum_qvar_ptr, elem_accum_qmass_ptr, elem_accum_q1mass_ptr type (c_ptr) :: elem_accum_iener_ptr, elem_accum_iener_wet_ptr, elem_accum_kener_ptr, elem_accum_pener_ptr + type (c_ptr) :: elem_tensorvisc2_ptr real (kind=real_kind), target, dimension(np,np,2,2) :: elem_D, elem_Dinv, elem_metinv, elem_tensorvisc + real (kind=real_kind), target, dimension(np,np,2,2) :: elem_tensorvisc2 real (kind=real_kind), target, dimension(np,np) :: elem_spheremp, elem_rspheremp, elem_metdet real (kind=real_kind), target, dimension(np,np) :: elem_state_phis, elem_fcor - real (kind=real_kind), target, dimension(np,np,3,2) :: elem_vec_sph2cart + real (kind=real_kind), target, dimension(np,np,3,3) :: elem_vec_sph2cart integer :: ie ! Initialize the 2d element arrays in C++ @@ -148,6 +150,7 @@ subroutine prim_init_elements_views (elem) elem_tensorvisc_ptr = c_loc(elem_tensorvisc) elem_vec_sph2cart_ptr = c_loc(elem_vec_sph2cart) elem_state_phis_ptr = c_loc(elem_state_phis) + elem_tensorvisc2_ptr = c_loc(elem_tensorvisc2) do ie=1,nelemd elem_D = elem(ie)%D @@ -160,12 +163,14 @@ subroutine prim_init_elements_views (elem) elem_state_phis = elem(ie)%state%phis elem_tensorvisc = elem(ie)%tensorVisc elem_vec_sph2cart = elem(ie)%vec_sphere2cart + elem_tensorvisc2 = elem(ie)%tensorVisc_2 call init_elements_2d_c (ie-1, & elem_D_ptr, elem_Dinv_ptr, elem_fcor_ptr, & elem_spheremp_ptr, elem_rspheremp_ptr, & elem_metdet_ptr, elem_metinv_ptr, & elem_state_phis_ptr, & - elem_tensorvisc_ptr, elem_vec_sph2cart_ptr) + elem_tensorvisc_ptr, elem_vec_sph2cart_ptr, & + elem_tensorvisc2_ptr) enddo ! Initialize the 3d element arrays in C++ diff --git a/components/homme/src/share/control_mod.F90 b/components/homme/src/share/control_mod.F90 index 48ddf10e31b2..0db924ad2403 100644 --- a/components/homme/src/share/control_mod.F90 +++ b/components/homme/src/share/control_mod.F90 @@ -178,6 +178,7 @@ module control_mod integer, public :: hypervis_order=0 ! laplace**hypervis_order. 0=not used 1=regular viscosity, 2=grad**4 real (kind=real_kind), public :: hypervis_scaling=0 ! use tensor hyperviscosity + real (kind=real_kind), public :: laplace_scaling=0 ! scaling for tensor viscosity !three types of hyper viscosity are supported right now: ! (1) const hv: nu * del^2 del^2 diff --git a/components/homme/src/share/cube_mod.F90 b/components/homme/src/share/cube_mod.F90 index 6cc51aad2fc0..9b7d7a4d16fe 100644 --- a/components/homme/src/share/cube_mod.F90 +++ b/components/homme/src/share/cube_mod.F90 @@ -20,7 +20,7 @@ module cube_mod change_coordinates use physical_constants, only : dd_pi, rearth - use control_mod, only : hypervis_scaling, cubed_sphere_map + use control_mod, only : hypervis_scaling, laplace_scaling, cubed_sphere_map use parallel_mod, only : abortmp use dimensions_mod, only : np,ne @@ -177,6 +177,10 @@ subroutine coordinates_atomic(elem,gll_points) elem%vec_sphere2cart(:,:,1,2) = -SIN(elem%spherep(:,:)%lat)*COS(elem%spherep(:,:)%lon) elem%vec_sphere2cart(:,:,2,2) = -SIN(elem%spherep(:,:)%lat)*SIN(elem%spherep(:,:)%lon) elem%vec_sphere2cart(:,:,3,2) = COS(elem%spherep(:,:)%lat) + ! Radial (vertical) direction + elem%vec_sphere2cart(:,:,1,3) = COS(elem%spherep(:,:)%lat) * COS(elem%spherep(:,:)%lon) + elem%vec_sphere2cart(:,:,2,3) = COS(elem%spherep(:,:)%lat) * SIN(elem%spherep(:,:)%lon) + elem%vec_sphere2cart(:,:,3,3) = SIN(elem%spherep(:,:)%lat) end subroutine coordinates_atomic @@ -423,6 +427,20 @@ subroutine metric_atomic(elem,gll_points,alpha) elem%tensorVisc(i,j,:,:)=V(:,:) + +! +! Tensor with scalings=2 for regular laplace operator (spnge layer) + lamStar1=1/(eig(1)**(laplace_scaling/2.0d0))*(rearth**2.0d0) + lamStar2=1/(eig(2)**(laplace_scaling/2.0d0))*(rearth**2.0d0) + DEL(1:2,1) = lamStar1 *eig(1)*DE(1:2,1) + DEL(1:2,2) = lamStar2 *eig(2)*DE(1:2,2) + V(1,1)=sum(DEL(1,:)*DE(1,:)) + V(1,2)=sum(DEL(1,:)*DE(2,:)) + V(2,1)=sum(DEL(2,:)*DE(1,:)) + V(2,2)=sum(DEL(2,:)*DE(2,:)) + + elem%tensorVisc_2(i,j,:,:)=V(:,:) + end do end do diff --git a/components/homme/src/share/cxx/ComposeTransportImpl.hpp b/components/homme/src/share/cxx/ComposeTransportImpl.hpp index 78b7459684e5..f7ebfe20222a 100644 --- a/components/homme/src/share/cxx/ComposeTransportImpl.hpp +++ b/components/homme/src/share/cxx/ComposeTransportImpl.hpp @@ -312,7 +312,7 @@ struct ComposeTransportImpl { KOKKOS_FUNCTION static void ugradv_sphere ( const SphereOperators& sphere_ops, const KernelVariables& kv, - const typename ViewConst >::type& vec_sphere2cart, + const typename ViewConst >::type& vec_sphere2cart, // velocity, latlon const typename ViewConst >::type& u, const typename ViewConst >::type& v, diff --git a/components/homme/src/share/cxx/ComposeTransportImplEnhancedTrajectoryImpl.hpp b/components/homme/src/share/cxx/ComposeTransportImplEnhancedTrajectoryImpl.hpp index 0a7ace37841a..5708eb54f445 100644 --- a/components/homme/src/share/cxx/ComposeTransportImplEnhancedTrajectoryImpl.hpp +++ b/components/homme/src/share/cxx/ComposeTransportImplEnhancedTrajectoryImpl.hpp @@ -780,7 +780,7 @@ KOKKOS_FUNCTION void calc_eta_dot_ref ( // velocity estimates at midpoint nodes. KOKKOS_INLINE_FUNCTION void calc_vel_horiz_formula_node_ref_mid ( const KernelVariables& kv, const SphereOperators& sphere_ops, - const CSNV& hyetam, const ExecViewUnmanaged& vec_sph2cart, + const CSNV& hyetam, const ExecViewUnmanaged& vec_sph2cart, // Velocities are at midpoints. Final eta_dot entry is ignored. const Real dtsub, const CS2elNlev vsph[2], const CSelNlevp eta_dot[2], const SelNlevp& wrk1, const S2elNlevp& vwrk1, const S2elNlevp& vwrk2, diff --git a/components/homme/src/share/cxx/Elements.cpp b/components/homme/src/share/cxx/Elements.cpp index fee99c5e4f22..6096dc23517c 100644 --- a/components/homme/src/share/cxx/Elements.cpp +++ b/components/homme/src/share/cxx/Elements.cpp @@ -12,7 +12,7 @@ namespace Homme { -void Elements::init(const int num_elems, const bool consthv, const bool alloc_gradphis, +void Elements::init(const int num_elems, const bool alloc_gradphis, const Real scale_factor, const Real laplacian_rigid_factor, const bool alloc_sphere_coords) { // Sanity check @@ -20,7 +20,7 @@ void Elements::init(const int num_elems, const bool consthv, const bool alloc_gr m_num_elems = num_elems; - m_geometry.init(num_elems,consthv,alloc_gradphis, + m_geometry.init(num_elems,alloc_gradphis, scale_factor, laplacian_rigid_factor < 0 ? 1/scale_factor : laplacian_rigid_factor, alloc_sphere_coords); diff --git a/components/homme/src/share/cxx/Elements.hpp b/components/homme/src/share/cxx/Elements.hpp index 89669ba57460..752b473c6552 100644 --- a/components/homme/src/share/cxx/Elements.hpp +++ b/components/homme/src/share/cxx/Elements.hpp @@ -41,7 +41,7 @@ class Elements { int num_elems () const { return m_num_elems; } - void init (const int num_elems, const bool consthv, const bool alloc_gradphis, + void init (const int num_elems, const bool alloc_gradphis, // See ElementsGeometry::init for details about these arguments. const Real scale_factor, const Real laplacian_rigid_factor=-1, const bool alloc_sphere_coords=false); diff --git a/components/homme/src/share/cxx/ElementsDerivedState.cpp b/components/homme/src/share/cxx/ElementsDerivedState.cpp index dda873abb8cd..3868b92d3341 100644 --- a/components/homme/src/share/cxx/ElementsDerivedState.cpp +++ b/components/homme/src/share/cxx/ElementsDerivedState.cpp @@ -35,7 +35,7 @@ void ElementsDerivedState::init(const int num_elems) { m_dpdiss_biharmonic = ExecViewManaged("derived_dpdiss_biharmonic", m_num_elems); m_dpdiss_ave = ExecViewManaged("derived_dpdiss_ave", m_num_elems); - // allocate SGS diffusivity fields + // allocate SGS turbulence related fields m_turb_diff_mom = ExecViewManaged("turb_diff_mom", m_num_elems); m_turb_diff_heat = ExecViewManaged("turb_diff_heat", m_num_elems); diff --git a/components/homme/src/share/cxx/ElementsGeometry.cpp b/components/homme/src/share/cxx/ElementsGeometry.cpp index ccc84ba50573..62b9f80e9bf9 100644 --- a/components/homme/src/share/cxx/ElementsGeometry.cpp +++ b/components/homme/src/share/cxx/ElementsGeometry.cpp @@ -16,14 +16,13 @@ namespace Homme { -void ElementsGeometry::init(const int num_elems, const bool consthv, const bool alloc_gradphis, +void ElementsGeometry::init(const int num_elems, const bool alloc_gradphis, const Real scale_factor, const Real laplacian_rigid_factor, const bool alloc_sphere_coords) { // Sanity check assert (num_elems>0); m_num_elems = num_elems; - m_consthv = consthv; assert(scale_factor > 0); m_scale_factor = scale_factor; @@ -40,10 +39,15 @@ void ElementsGeometry::init(const int num_elems, const bool consthv, const bool m_metinv = ExecViewManaged("METINV", m_num_elems); m_metdet = ExecViewManaged("METDET", m_num_elems); - if(!consthv){ - m_tensorvisc = ExecViewManaged("TENSORVISC", m_num_elems); - } - m_vec_sph2cart = ExecViewManaged("VEC_SPH2CART", m_num_elems); + // tensorVisc/tensorVisc2 are always allocated and copied (the Fortran + // side always computes valid values for them, in metric_atomic(), even + // when hypervis_scaling/laplace_scaling are 0). Whether the tensor or + // the constant-coefficient operator is actually used at run time is + // decided in the timestepping code (see HyperviscosityFunctorImpl's + // consthv/constsponge). + m_tensorvisc = ExecViewManaged("TENSORVISC", m_num_elems); + m_tensorvisc2 = ExecViewManaged("TENSORVISC2", m_num_elems); + m_vec_sph2cart = ExecViewManaged("VEC_SPH2CART", m_num_elems); m_phis = ExecViewManaged("PHIS", m_num_elems); @@ -66,21 +70,30 @@ set_elem_data (const int ie, CF90Ptr& D, CF90Ptr& Dinv, CF90Ptr& fcor, CF90Ptr& spheremp, CF90Ptr& rspheremp, CF90Ptr& metdet, CF90Ptr& metinv, - CF90Ptr& tensorvisc, CF90Ptr& vec_sph2cart, const bool consthv, - const Real* sphere_cart, const Real* sphere_latlon) { + CF90Ptr& tensorvisc, CF90Ptr& vec_sph2cart, + const Real* sphere_cart, const Real* sphere_latlon, + CF90Ptr& tensorvisc2) { // Check geometry was inited assert (m_num_elems>0); // Check input assert (ie>=0 && ie; using TensorView = ExecViewUnmanaged; - using Tensor23View = ExecViewUnmanaged; + using Tensor33View = ExecViewUnmanaged; using ScalarViewF90 = HostViewUnmanaged; using TensorViewF90 = HostViewUnmanaged; - using Tensor23ViewF90 = HostViewUnmanaged; + using Tensor33ViewF90 = HostViewUnmanaged; ScalarView::host_mirror_type h_fcor = Kokkos::create_mirror_view(Homme::subview(m_fcor,ie)); ScalarView::host_mirror_type h_metdet = Kokkos::create_mirror_view(Homme::subview(m_metdet,ie)); @@ -90,11 +103,7 @@ set_elem_data (const int ie, TensorView::host_mirror_type h_d = Kokkos::create_mirror_view(Homme::subview(m_d,ie)); TensorView::host_mirror_type h_dinv = Kokkos::create_mirror_view(Homme::subview(m_dinv,ie)); - TensorView::host_mirror_type h_tensorvisc; - Tensor23View::host_mirror_type h_vec_sph2cart; - if( !consthv ){ - h_tensorvisc = Kokkos::create_mirror_view(Homme::subview(m_tensorvisc,ie)); - } + Tensor33View::host_mirror_type h_vec_sph2cart; h_vec_sph2cart = Kokkos::create_mirror_view(Homme::subview(m_vec_sph2cart,ie)); ScalarViewF90 h_fcor_f90 (fcor); @@ -104,8 +113,7 @@ set_elem_data (const int ie, TensorViewF90 h_metinv_f90 (metinv); TensorViewF90 h_d_f90 (D); TensorViewF90 h_dinv_f90 (Dinv); - TensorViewF90 h_tensorvisc_f90 (tensorvisc); - Tensor23ViewF90 h_vec_sph2cart_f90 (vec_sph2cart); + Tensor33ViewF90 h_vec_sph2cart_f90 (vec_sph2cart); // 2d scalars for (int igp = 0; igp < NP; ++igp) { @@ -130,18 +138,7 @@ set_elem_data (const int ie, } } - if(!consthv) { - for (int idim = 0; idim < 2; ++idim) { - for (int jdim = 0; jdim < 2; ++jdim) { - for (int igp = 0; igp < NP; ++igp) { - for (int jgp = 0; jgp < NP; ++jgp) { - h_tensorvisc (idim,jdim,igp,jgp) = h_tensorvisc_f90 (idim,jdim,igp,jgp); - } - } - } - } - }//end if consthv - for (int idim = 0; idim < 2; ++idim) { + for (int idim = 0; idim < 3; ++idim) { for (int jdim = 0; jdim < 3; ++jdim) { for (int igp = 0; igp < NP; ++igp) { for (int jgp = 0; jgp < NP; ++jgp) { @@ -158,9 +155,6 @@ set_elem_data (const int ie, Kokkos::deep_copy(Homme::subview(m_rspheremp,ie), h_rspheremp); Kokkos::deep_copy(Homme::subview(m_d,ie), h_d); Kokkos::deep_copy(Homme::subview(m_dinv,ie), h_dinv); - if( !consthv ) { - Kokkos::deep_copy(Homme::subview(m_tensorvisc,ie), h_tensorvisc); - } Kokkos::deep_copy(Homme::subview(m_vec_sph2cart,ie), h_vec_sph2cart); if (sphere_cart && m_sphere_cart.size() != 0) { @@ -173,6 +167,62 @@ set_elem_data (const int ie, } } +void ElementsGeometry:: +set_tensorvisc (const int ie, CF90Ptr& tensorvisc) { + // Check geometry was inited + assert (m_num_elems>0); + + // Check input + assert (ie>=0 && ie; + using TensorViewF90 = HostViewUnmanaged; + + TensorView::host_mirror_type h_tensorvisc = + Kokkos::create_mirror_view(Homme::subview(m_tensorvisc,ie)); + TensorViewF90 h_tensorvisc_f90 (tensorvisc); + + for (int idim = 0; idim < 2; ++idim) { + for (int jdim = 0; jdim < 2; ++jdim) { + for (int igp = 0; igp < NP; ++igp) { + for (int jgp = 0; jgp < NP; ++jgp) { + h_tensorvisc (idim,jdim,igp,jgp) = h_tensorvisc_f90 (idim,jdim,igp,jgp); + } + } + } + } + + Kokkos::deep_copy(Homme::subview(m_tensorvisc,ie), h_tensorvisc); +} + +void ElementsGeometry:: +set_tensorvisc2 (const int ie, CF90Ptr& tensorvisc2) { + // Check geometry was inited + assert (m_num_elems>0); + + // Check input + assert (ie>=0 && ie; + using TensorViewF90 = HostViewUnmanaged; + + TensorView::host_mirror_type h_tensorvisc2 = + Kokkos::create_mirror_view(Homme::subview(m_tensorvisc2,ie)); + TensorViewF90 h_tensorvisc2_f90 (tensorvisc2); + + for (int idim = 0; idim < 2; ++idim) { + for (int jdim = 0; jdim < 2; ++jdim) { + for (int igp = 0; igp < NP; ++igp) { + for (int jgp = 0; jgp < NP; ++jgp) { + h_tensorvisc2 (idim,jdim,igp,jgp) = h_tensorvisc2_f90 (idim,jdim,igp,jgp); + } + } + } + } + + Kokkos::deep_copy(Homme::subview(m_tensorvisc2,ie), h_tensorvisc2); +} + void ElementsGeometry:: set_phis (const int ie, CF90Ptr& phis) { // Check geometry was inited @@ -209,6 +259,7 @@ void ElementsGeometry::randomize(const int seed) { genRandArray(m_spheremp, engine, random_dist); genRandArray(m_tensorvisc, engine, random_dist); + genRandArray(m_tensorvisc2, engine, random_dist); genRandArray(m_vec_sph2cart, engine, random_dist); genRandArray(m_phis, engine, random_dist); diff --git a/components/homme/src/share/cxx/ElementsGeometry.hpp b/components/homme/src/share/cxx/ElementsGeometry.hpp index bd427d47acd6..d8c011bc61ad 100644 --- a/components/homme/src/share/cxx/ElementsGeometry.hpp +++ b/components/homme/src/share/cxx/ElementsGeometry.hpp @@ -31,7 +31,10 @@ class ElementsGeometry { ExecViewManaged m_metinv; ExecViewManaged m_metdet; ExecViewManaged m_tensorvisc; - ExecViewManaged m_vec_sph2cart; + // Second tensor viscosity matrix, used by the sponge layer (theta-l_kokkos + // only), controlled by laplace_scaling. Always allocated (see init()). + ExecViewManaged m_tensorvisc2; + ExecViewManaged m_vec_sph2cart; // Prescribed surface geopotential height at eta = 1 ExecViewManaged m_phis; @@ -50,7 +53,7 @@ class ElementsGeometry { Real m_scale_factor, m_laplacian_rigid_factor; - void init (const int num_elems, const bool consthv, const bool alloc_gradphis, + void init (const int num_elems, const bool alloc_gradphis, // Usually, scale_factor is rearth for sphere-Earth problems and 1 // for planar problems. Usually, laplacian_rigid_factor is 1/rearth // for the sphere and 0 for the plane. If laplacian_rigid_factor is @@ -70,13 +73,26 @@ class ElementsGeometry { CF90Ptr& spheremp, CF90Ptr& rspheremp, CF90Ptr& metdet, CF90Ptr& metinv, CF90Ptr& tensorvisc, - CF90Ptr& vec_sph2cart, const bool consthv, - const Real* sphere_cart = nullptr, const Real* sphere_latlon = nullptr); + CF90Ptr& vec_sph2cart, + const Real* sphere_cart = nullptr, const Real* sphere_latlon = nullptr, + CF90Ptr& tensorvisc2 = nullptr); + + // Fill (or refresh) just the tensorVisc view for one element. This is + // separate from set_elem_data() because tensorVisc is the only field in + // prim_init_grid_views's payload that is computed AFTER dss_hvtensor runs; + // all other geometry fields (D, Dinv, fcor, spheremp, rspheremp, metdet, + // metinv, vec_sph2cart, sphere_cart/latlon) are constant and are copied + // once, early, by set_elem_data(). + void set_tensorvisc (const int ie, CF90Ptr& tensorvisc); + + // Same as set_tensorvisc(), but for tensorVisc_2 (the sponge-layer tensor + // coefficient), which is likewise recomputed by dss_hvtensor after the + // initial (constant-field) set_elem_data() copy. + void set_tensorvisc2 (const int ie, CF90Ptr& tensorvisc2); void set_phis (const int ie, CF90Ptr& phis); private: - bool m_consthv; int m_num_elems; }; diff --git a/components/homme/src/share/cxx/GllFvRemap.cpp b/components/homme/src/share/cxx/GllFvRemap.cpp index 5c9319ada122..876bc21279d8 100644 --- a/components/homme/src/share/cxx/GllFvRemap.cpp +++ b/components/homme/src/share/cxx/GllFvRemap.cpp @@ -7,11 +7,9 @@ #include "GllFvRemap.hpp" #include "GllFvRemapImpl.hpp" #include "Context.hpp" -#include "ErrorDefs.hpp" #include "profiling.hpp" #include -#include namespace Homme { @@ -61,14 +59,18 @@ ::init_data (const int nf, const int nf_max, const bool theta_hydrostatic_mode, void GllFvRemap ::run_dyn_to_fv_phys (const int time_idx, const Phys1T& ps, const Phys1T& phis, - const Phys2T& T, const Phys2T& omega, const Phys3T& uv, - const Phys3T& q, const Phys2T* dp) { - m_impl->run_dyn_to_fv_phys(time_idx, ps, phis, T, omega, uv, q, dp); + const Phys2T& T, const Phys2T& omega, + const CPhys3T* strain3d_components_gll, + const Phys3T* strain3d_components_fv, + const Phys3T& uv, const Phys3T& q, const Phys2T* dp) { + m_impl->run_dyn_to_fv_phys(time_idx, ps, phis, T, omega, + strain3d_components_gll, strain3d_components_fv, + uv, q, dp); } void GllFvRemap ::run_fv_phys_to_dyn (const int time_idx, const CPhys2T& T, const CPhys3T& uv, - const CPhys3T& q, const CPhys2T& Km, const CPhys2T& Kh) { + const CPhys3T& q, const CPhys2T* Km, const CPhys2T* Kh) { m_impl->run_fv_phys_to_dyn(time_idx, T, uv, q, Km, Kh); } @@ -81,4 +83,3 @@ ::remap_tracer_dyn_to_fv_phys (const int time_idx, const int nq, } } // Namespace Homme - diff --git a/components/homme/src/share/cxx/GllFvRemap.hpp b/components/homme/src/share/cxx/GllFvRemap.hpp index a6222ebb7f56..e115d77a0a68 100644 --- a/components/homme/src/share/cxx/GllFvRemap.hpp +++ b/components/homme/src/share/cxx/GllFvRemap.hpp @@ -52,6 +52,8 @@ class GllFvRemap { const Phys1T& ps, const Phys1T& phis, // T,omega(ie,col,lev) const Phys2T& T, const Phys2T& omega, + const CPhys3T* strain3d_components_gll, + const Phys3T* strain3d_components_fv, // uv(ie, col, 0 or 1, lev) const Phys3T& uv, // q(ie,col,idx,lev) @@ -60,7 +62,8 @@ class GllFvRemap { const Phys2T* dp = nullptr); // Remap physics state and tendencies to dynamics state and tendencies. void run_fv_phys_to_dyn(const int time_idx, const CPhys2T& T, const CPhys3T& uv, - const CPhys3T& q, const CPhys2T& Km, const CPhys2T& Kh); + const CPhys3T& q, const CPhys2T* Km = nullptr, + const CPhys2T* Kh = nullptr); // DSS the remapped dynamics tendencies and state. Call this after // run_fv_phys_to_dyn if the dynamics-physics coupler does not already // provide it. diff --git a/components/homme/src/share/cxx/GllFvRemapImpl.cpp b/components/homme/src/share/cxx/GllFvRemapImpl.cpp index a920695ba9f9..21bbc146b6a7 100644 --- a/components/homme/src/share/cxx/GllFvRemapImpl.cpp +++ b/components/homme/src/share/cxx/GllFvRemapImpl.cpp @@ -339,7 +339,8 @@ f2g_scalar_dp (const KernelVariables& kv, const int nf2, const int np2, const in void GllFvRemapImpl ::run_dyn_to_fv_phys (const int timeidx, const Phys1T& ps, const Phys1T& phis, const Phys2T& Ts, - const Phys2T& omegas, const Phys3T& uvs, const Phys3T& qs, + const Phys2T& omegas, const CPhys3T* shear_strain3d_components_gll_ptr, + const Phys3T* shear_strain3d_components_fv_ptr, const Phys3T& uvs, const Phys3T& qs, const Phys2T* dp_fv_out_ptr) { // Impl only for theta-l until ElementOps is provided in preqx_kokkos. #ifdef MODEL_THETA_L @@ -354,6 +355,8 @@ ::run_dyn_to_fv_phys (const int timeidx, const Phys1T& ps, const Phys1T& phis, c const auto buf10 = m_data.buf1[0]; const auto buf11 = m_data.buf1[1]; const auto buf20 = m_data.buf2[0]; + const bool remap_strain = shear_strain3d_components_gll_ptr != nullptr && + shear_strain3d_components_fv_ptr != nullptr; #ifndef NDEBUG const auto nelemd = m_data.nelemd; @@ -362,6 +365,14 @@ ::run_dyn_to_fv_phys (const int timeidx, const Phys1T& ps, const Phys1T& phis, c assert(Ts.extent_int(0) >= nelemd && Ts.extent_int(1) >= nf2 && Ts.extent_int(2) % packn == 0); assert(omegas.extent_int(0) >= nelemd && omegas.extent_int(1) >= nf2 && omegas.extent_int(2) % packn == 0); + if (remap_strain) { + const auto& shear_strain3d_components_gll = *shear_strain3d_components_gll_ptr; + const auto& shear_strain3d_components_fv = *shear_strain3d_components_fv_ptr; + assert(shear_strain3d_components_gll.extent_int(0) >= nelemd && shear_strain3d_components_gll.extent_int(1) >= np2 && + shear_strain3d_components_gll.extent_int(2) == 6 && shear_strain3d_components_gll.extent_int(3) % packn == 0); + assert(shear_strain3d_components_fv.extent_int(0) >= nelemd && shear_strain3d_components_fv.extent_int(1) >= nf2 && + shear_strain3d_components_fv.extent_int(2) == 6 && shear_strain3d_components_fv.extent_int(3) % packn == 0); + } assert(uvs.extent_int(0) >= nelemd && uvs.extent_int(1) >= nf2 && uvs.extent_int(2) == 2 && uvs.extent_int(3) % packn == 0); assert(qs.extent_int(0) >= nelemd && qs.extent_int(1) >= nf2 && qs.extent_int(2) >= qsize && @@ -377,6 +388,18 @@ ::run_dyn_to_fv_phys (const int timeidx, const Phys1T& ps, const Phys1T& phis, c uvs.extent_int(3)/packn), q(real2pack(qs), qs.extent_int(0), qs.extent_int(1), qs.extent_int(2), qs.extent_int(3)/packn); + CVPhys3T strain_components_gll; + VPhys3T strain_components_fv; + if (remap_strain) { + const auto& shear_strain3d_components_gll = *shear_strain3d_components_gll_ptr; + const auto& shear_strain3d_components_fv = *shear_strain3d_components_fv_ptr; + strain_components_gll = CVPhys3T(creal2pack(shear_strain3d_components_gll), shear_strain3d_components_gll.extent_int(0), + shear_strain3d_components_gll.extent_int(1), shear_strain3d_components_gll.extent_int(2), + shear_strain3d_components_gll.extent_int(3)/packn); + strain_components_fv = VPhys3T(real2pack(shear_strain3d_components_fv), shear_strain3d_components_fv.extent_int(0), + shear_strain3d_components_fv.extent_int(1), shear_strain3d_components_fv.extent_int(2), + shear_strain3d_components_fv.extent_int(3)/packn); + } const auto dp3d = m_state.m_dp3d; const auto vthdp = m_state.m_vtheta_dp; @@ -523,11 +546,37 @@ ::run_dyn_to_fv_phys (const int timeidx, const Phys1T& ps, const Phys1T& phis, c evur3(&Dinv(ie,0,0,0), np2, 2, 2), w_ff, evur3(&D_f(ie,0,0,0), nf2, 2, 2), evucs_2_np2_nlev(&v(ie,timeidx,0,0,0,0)), evus_2_np2_nlev(r2w.data()), evus3(&uv(ie,0,0,0), uv.extent_int(1), uv.extent_int(2), uv.extent_int(3))); + kv.team_barrier(); // r2w scratch is reused below // omega remapd(team, nf2, np2, nlevpk, g2f_remapd, gll_metdet_ie, w_ff, fv_metdet_ie, evucs_np2_nlev(&omega_g(ie,0,0,0)), evus_np2_nlev(rw1.data()), evus2(&omega(ie,0,0), nf2, nlevpk)); + kv.team_barrier(); // rw1 scratch is reused below + + if (remap_strain) { + // shear-strain tensor components + const auto ttrg = Kokkos::TeamThreadRange(kv.team, np2); + const EVU comp_g(rw2.data()); + const EVU comp_f(&r2w(0,0,0,0), nf2); + for (int icomp = 0; icomp < 6; ++icomp) { + parallel_for(ttrg, [&] (const int ij) { + const int i = ij / NP; + const int j = ij % NP; + parallel_for(tvr, [&] (const int k) { + comp_g(i,j,k) = strain_components_gll(ie,ij,icomp,k); + }); + }); + kv.team_barrier(); + remapd(team, nf2, np2, nlevpk, g2f_remapd, gll_metdet_ie, w_ff, fv_metdet_ie, + evucs_np2_nlev(comp_g.data()), evus_np2_nlev(rw1.data()), + evus2(comp_f.data(), nf2, nlevpk)); + kv.team_barrier(); + loop_ik(ttrf, tvr, [&] (int i, int k) { strain_components_fv(ie,i,icomp,k) = comp_f(i,k); }); + kv.team_barrier(); + } + } + }; Kokkos::fence(); Kokkos::parallel_for(m_tp_ne, fe); @@ -560,7 +609,7 @@ ::run_dyn_to_fv_phys (const int timeidx, const Phys1T& ps, const Phys1T& phis, c void GllFvRemapImpl:: run_fv_phys_to_dyn (const int timeidx, const CPhys2T& Ts, const CPhys3T& uvs, - const CPhys3T& qs, const CPhys2T& Kms, const CPhys2T& Khs) { + const CPhys3T& qs, const CPhys2T* Kms, const CPhys2T* Khs) { #ifdef MODEL_THETA_L using Kokkos::parallel_for; @@ -570,6 +619,8 @@ run_fv_phys_to_dyn (const int timeidx, const CPhys2T& Ts, const CPhys3T& uvs, const auto nf2 = m_data.nf2; const auto qsize = m_data.qsize; const auto uv_ndim = uvs.extent_int(2); + const bool remap_turb_diff = Kms != nullptr; + assert((Kms == nullptr) == (Khs == nullptr)); const auto buf10 = m_data.buf1[0]; const auto buf11 = m_data.buf1[1]; @@ -582,12 +633,18 @@ run_fv_phys_to_dyn (const int timeidx, const CPhys2T& Ts, const CPhys3T& uvs, (uv_ndim == 2 || uv_ndim == 3) && uvs.extent_int(3) % packn == 0); assert(qs.extent_int(0) >= nelemd && qs.extent_int(1) >= nf2 && qs.extent_int(2) >= qsize && qs.extent_int(3) % packn == 0); + if (remap_turb_diff) { + assert(Kms->extent_int(0) >= nelemd && Kms->extent_int(1) >= nf2 && Kms->extent_int(2) % packn == 0); + assert(Khs->extent_int(0) >= nelemd && Khs->extent_int(1) >= nf2 && Khs->extent_int(2) % packn == 0); + } #endif - CVPhys2T - T(creal2pack(Ts), Ts.extent_int(0), Ts.extent_int(1), Ts.extent_int(2)/packn), - Km(creal2pack(Kms), Kms.extent_int(0), Kms.extent_int(1), Kms.extent_int(2)/packn), - Kh(creal2pack(Khs), Khs.extent_int(0), Khs.extent_int(1), Khs.extent_int(2)/packn); + CVPhys2T T(creal2pack(Ts), Ts.extent_int(0), Ts.extent_int(1), Ts.extent_int(2)/packn); + CVPhys2T Km, Kh; + if (remap_turb_diff) { + Km = CVPhys2T(creal2pack(*Kms), Kms->extent_int(0), Kms->extent_int(1), Kms->extent_int(2)/packn); + Kh = CVPhys2T(creal2pack(*Khs), Khs->extent_int(0), Khs->extent_int(1), Khs->extent_int(2)/packn); + } CVPhys3T uv(creal2pack(uvs), uvs.extent_int(0), uvs.extent_int(1), uvs.extent_int(2), uvs.extent_int(3)/packn), @@ -642,7 +699,7 @@ run_fv_phys_to_dyn (const int timeidx, const CPhys2T& Ts, const CPhys3T& uvs, kv.team_barrier(); } - { + if (remap_turb_diff) { using Homme::Scalar; const evucs2 Km_f_ie(&Km(ie,0,0), nf2, nlevpk); diff --git a/components/homme/src/share/cxx/GllFvRemapImpl.hpp b/components/homme/src/share/cxx/GllFvRemapImpl.hpp index b690f4ac6d5c..c815eec26083 100644 --- a/components/homme/src/share/cxx/GllFvRemapImpl.hpp +++ b/components/homme/src/share/cxx/GllFvRemapImpl.hpp @@ -111,10 +111,12 @@ struct GllFvRemapImpl { const Real* f2g_remapd_r, const Real* D_f_r, const Real* Dinv_f_r); void run_dyn_to_fv_phys(const int time_idx, const Phys1T& ps, const Phys1T& phis, - const Phys2T& T, const Phys2T& omega, const Phys3T& uv, - const Phys3T& q, const Phys2T* dp); + const Phys2T& T, const Phys2T& omega, + const CPhys3T* strain3d_components_gll, + const Phys3T* strain3d_components_fv, + const Phys3T& uv, const Phys3T& q, const Phys2T* dp); void run_fv_phys_to_dyn(const int time_idx, const CPhys2T& T, const CPhys3T& uv, - const CPhys3T& q, const CPhys2T& Km, const CPhys2T& Kh); + const CPhys3T& q, const CPhys2T* Km, const CPhys2T* Kh); void run_fv_phys_to_dyn_dss(); void remap_tracer_dyn_to_fv_phys(const int time_idx, const int nq, diff --git a/components/homme/src/share/cxx/SimulationParams.hpp b/components/homme/src/share/cxx/SimulationParams.hpp index bfa9ee94d664..3f239053a064 100644 --- a/components/homme/src/share/cxx/SimulationParams.hpp +++ b/components/homme/src/share/cxx/SimulationParams.hpp @@ -57,6 +57,11 @@ struct SimulationParams int hypervis_subcycle; int hypervis_subcycle_tom; double hypervis_scaling; + // Scaling exponent for the sponge-layer (nu_top) tensor viscosity, mirroring + // Fortran's laplace_scaling (theta-l_kokkos only; defaults to 0, i.e. constant + // coefficient sponge-layer diffusion, for preqx_kokkos and any case that + // doesn't set it explicitly). + double laplace_scaling = 0.0; double nu_ratio1, nu_ratio2; // control balance between div and vort components in vector laplace int nsplit = 0; int nsplit_iteration; @@ -102,6 +107,7 @@ inline void SimulationParams::print (std::ostream& out) { out << " hypervis_subcycle: " << hypervis_subcycle << "\n"; out << " hypervis_subcycle_tom: " << hypervis_subcycle_tom << "\n"; out << " hypervis_scaling: " << hypervis_scaling << "\n"; + out << " laplace_scaling: " << laplace_scaling << "\n"; out << " nu_ratio1: " << nu_ratio1 << "\n"; out << " nu_ratio2: " << nu_ratio2 << "\n"; out << " use_cpstar: " << (use_cpstar ? "yes" : "no") << "\n"; diff --git a/components/homme/src/share/cxx/SphereOperators.hpp b/components/homme/src/share/cxx/SphereOperators.hpp index c227d97ea708..9ab3e73c9234 100644 --- a/components/homme/src/share/cxx/SphereOperators.hpp +++ b/components/homme/src/share/cxx/SphereOperators.hpp @@ -812,24 +812,25 @@ class SphereOperators laplace_simple(kv, field, laplace, NUM_LEV_REQUEST); }//end of laplace_simple - template + template KOKKOS_INLINE_FUNCTION void laplace_tensor(const KernelVariables &kv, const ExecViewUnmanaged& tensorVisc, const typename ViewConst>::type& field, // input - const ExecViewUnmanaged& laplace) const + const ExecViewUnmanaged& laplace, + const int NUM_LEV_REQUEST) const { - static_assert(NUM_LEV_REQUEST>=0, "Error! Invalid value for NUM_LEV_REQUEST.\n"); - static_assert(NUM_LEV_REQUEST<=NUM_LEV_IN, "Error! Input view does not have enough levels.\n"); - static_assert(NUM_LEV_REQUEST<=NUM_LEV_OUT, "Error! Output view does not have enough levels.\n"); + assert(NUM_LEV_REQUEST>=0); + assert(NUM_LEV_REQUEST<=NUM_LEV_IN); + assert(NUM_LEV_REQUEST<=NUM_LEV_OUT); // Make sure the buffers have been created assert (vector_buf_ml.size()>0); - vector_buf grad_s(Homme::subview(vector_buf_ml, kv.team_idx, 1).data()); - vector_buf sphere_buf(Homme::subview(vector_buf_ml, kv.team_idx, 2).data()); + vector_buf grad_s(Homme::subview(vector_buf_ml, kv.team_idx, 1).data()); + vector_buf sphere_buf(Homme::subview(vector_buf_ml, kv.team_idx, 2).data()); - gradient_sphere(kv, field, grad_s); + gradient_sphere(kv, field, grad_s, NUM_LEV_REQUEST); //now multiply tensorVisc(:,:,i,j)*grad_s(i,j) (matrix*vector, independent of i,j ) //but it requires a temp var to store a result. the result is then placed to grad_s, //or should it be an extra temp var instead of an extra loop? @@ -847,7 +848,21 @@ class SphereOperators }); kv.team_barrier(); - divergence_sphere_wk(kv, sphere_buf, laplace); + divergence_sphere_wk(kv, sphere_buf, laplace, NUM_LEV_REQUEST); + }//end of laplace_tensor + + template + KOKKOS_INLINE_FUNCTION void + laplace_tensor(const KernelVariables &kv, + const ExecViewUnmanaged& tensorVisc, + const typename ViewConst>::type& field, // input + const ExecViewUnmanaged& laplace) const + { + static_assert(NUM_LEV_REQUEST>=0, "Error! Invalid value for NUM_LEV_REQUEST.\n"); + static_assert(NUM_LEV_REQUEST<=NUM_LEV_IN, "Error! Input view does not have enough levels.\n"); + static_assert(NUM_LEV_REQUEST<=NUM_LEV_OUT, "Error! Output view does not have enough levels.\n"); + + laplace_tensor(kv, tensorVisc, field, laplace, NUM_LEV_REQUEST); }//end of laplace_tensor template @@ -1015,25 +1030,26 @@ class SphereOperators grad_sphere_wk_testcov(kv, scalar, grads, NUM_LEV_REQUEST); } - template + template KOKKOS_INLINE_FUNCTION void vlaplace_sphere_wk_cartesian (const KernelVariables &kv, const ExecViewUnmanaged& tensorVisc, - const ExecViewUnmanaged& vec_sph2cart, + const ExecViewUnmanaged& vec_sph2cart, const typename ViewConst>::type& vector, - const ExecViewUnmanaged& laplace) const + const ExecViewUnmanaged& laplace, + const int NUM_LEV_REQUEST) const { - static_assert(NUM_LEV_REQUEST>=0, "Error! Invalid value for NUM_LEV_REQUEST.\n"); - static_assert(NUM_LEV_REQUEST<=NUM_LEV_IN, "Error! Input view does not have enough levels.\n"); - static_assert(NUM_LEV_REQUEST<=NUM_LEV_OUT, "Error! Output view does not have enough levels.\n"); + assert(NUM_LEV_REQUEST>=0); + assert(NUM_LEV_REQUEST<=NUM_LEV_IN); + assert(NUM_LEV_REQUEST<=NUM_LEV_OUT); // Make sure the buffers have been created assert (vector_buf_ml.size()>0); const auto& spheremp = Homme::subview(m_spheremp, kv.ie); - scalar_buf laplace0(Homme::subview(scalar_buf_ml,kv.team_idx,0).data()); - scalar_buf laplace1(Homme::subview(scalar_buf_ml,kv.team_idx,1).data()); - scalar_buf laplace2(Homme::subview(scalar_buf_ml,kv.team_idx,2).data()); + scalar_buf laplace0(Homme::subview(scalar_buf_ml,kv.team_idx,0).data()); + scalar_buf laplace1(Homme::subview(scalar_buf_ml,kv.team_idx,1).data()); + scalar_buf laplace2(Homme::subview(scalar_buf_ml,kv.team_idx,2).data()); constexpr int np_squared = NP * NP; Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, np_squared), [&](const int loop_idx) { @@ -1050,9 +1066,9 @@ class SphereOperators kv.team_barrier(); // Use laplace* as input, and then overwrite it with the output (saves temporaries) - laplace_tensor(kv,tensorVisc,laplace0,laplace0); - laplace_tensor(kv,tensorVisc,laplace1,laplace1); - laplace_tensor(kv,tensorVisc,laplace2,laplace2); + laplace_tensor(kv,tensorVisc,laplace0,laplace0,NUM_LEV_REQUEST); + laplace_tensor(kv,tensorVisc,laplace1,laplace1,NUM_LEV_REQUEST); + laplace_tensor(kv,tensorVisc,laplace2,laplace2,NUM_LEV_REQUEST); Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, np_squared), [&](const int loop_idx) { @@ -1085,6 +1101,21 @@ class SphereOperators kv.team_barrier(); } // end of vlaplace_sphere_wk_cartesian + template + KOKKOS_INLINE_FUNCTION void + vlaplace_sphere_wk_cartesian (const KernelVariables &kv, + const ExecViewUnmanaged& tensorVisc, + const ExecViewUnmanaged& vec_sph2cart, + const typename ViewConst>::type& vector, + const ExecViewUnmanaged& laplace) const + { + static_assert(NUM_LEV_REQUEST>=0, "Error! Invalid value for NUM_LEV_REQUEST.\n"); + static_assert(NUM_LEV_REQUEST<=NUM_LEV_IN, "Error! Input view does not have enough levels.\n"); + static_assert(NUM_LEV_REQUEST<=NUM_LEV_OUT, "Error! Output view does not have enough levels.\n"); + + vlaplace_sphere_wk_cartesian(kv, tensorVisc, vec_sph2cart, vector, laplace, NUM_LEV_REQUEST); + } // end of vlaplace_sphere_wk_cartesian + template KOKKOS_INLINE_FUNCTION void vlaplace_sphere_wk_contra (const KernelVariables &kv, const Real nu_ratio, diff --git a/components/homme/src/share/derivative_mod.F90 b/components/homme/src/share/derivative_mod.F90 index 5daf74b67166..ee74190f0c8f 100644 --- a/components/homme/src/share/derivative_mod.F90 +++ b/components/homme/src/share/derivative_mod.F90 @@ -10,7 +10,7 @@ module derivative_mod use quadrature_mod, only : quadrature_t, gauss, gausslobatto,legendre, jacobi use parallel_mod, only : abortmp use element_mod, only : element_t - use control_mod, only : hypervis_scaling + use control_mod, only : hypervis_scaling, laplace_scaling use physical_constants, only : scale_factor_inv, laplacian_rigid_factor implicit none @@ -77,7 +77,7 @@ module derivative_mod public :: curl_sphere_wk_testcov ! public :: curl_sphere_wk_testcontra ! not coded public :: divergence_sphere_wk - public :: laplace_sphere_wk + public :: laplace_sphere_wk ! laplace with hypervis_scaling tensor option public :: vlaplace_sphere_wk public :: vlaplace_sphere_wk_contra public :: vlaplace_sphere_wk_cartesian @@ -704,7 +704,7 @@ function ugradv_sphere(u,v,deriv,elem) result(ugradv) ! (This is just a faster way of doing a dot product for each grid point, ! since reindexing the inputs to use the intrinsic effectively would be ! just asking for trouble.) - dum_cart(:,:,component)=sum( elem%vec_sphere2cart(:,:,component,:)*v(:,:,:) ,3) + dum_cart(:,:,component)=sum( elem%vec_sphere2cart(:,:,component,1:2)*v(:,:,:) ,3) ! dum_cart(:,:,component)= elem%vec_sphere2cart(:,:,component,1)*v(:,:,1) + & ! elem%vec_sphere2cart(:,:,component,2)*v(:,:,2) end do @@ -1069,7 +1069,7 @@ end function divergence_sphere !DIR$ ATTRIBUTES FORCEINLINE :: laplace_sphere_wk - function laplace_sphere_wk(s,deriv,elem,var_coef) result(laplace) + function laplace_sphere_wk(s,deriv,elem,var_coef,tensor) result(laplace) ! ! input: s = scalar ! ouput: -< grad(PHI), grad(s) > = weak divergence of grad(s) @@ -1079,6 +1079,8 @@ function laplace_sphere_wk(s,deriv,elem,var_coef) result(laplace) logical, intent(in) :: var_coef type (derivative_t), intent(in) :: deriv type (element_t), intent(in) :: elem + real(kind=real_kind), optional, intent(in) :: tensor(np,np,2,2) + real(kind=real_kind) :: laplace(np,np) integer :: i,j @@ -1088,9 +1090,17 @@ function laplace_sphere_wk(s,deriv,elem,var_coef) result(laplace) grads=gradient_sphere(s,deriv,elem%Dinv) if (var_coef) then - if (hypervis_scaling /=0 ) then - ! tensor hv, (3) - oldgrads=grads + oldgrads=grads + if (present(tensor)) then + do j=1,np + do i=1,np + grads(i,j,1) = oldgrads(i,j,1)*tensor(i,j,1,1) + & + oldgrads(i,j,2)*tensor(i,j,1,2) + grads(i,j,2) = oldgrads(i,j,1)*tensor(i,j,2,1) + & + oldgrads(i,j,2)*tensor(i,j,2,2) + end do + end do + else do j=1,np do i=1,np grads(i,j,1) = oldgrads(i,j,1)*elem%tensorVisc(i,j,1,1) + & @@ -1099,8 +1109,6 @@ function laplace_sphere_wk(s,deriv,elem,var_coef) result(laplace) oldgrads(i,j,2)*elem%tensorVisc(i,j,2,2) end do end do - else - ! do nothing: constant coefficient viscsoity endif endif @@ -1111,7 +1119,7 @@ function laplace_sphere_wk(s,deriv,elem,var_coef) result(laplace) end function laplace_sphere_wk !DIR$ ATTRIBUTES FORCEINLINE :: vlaplace_sphere_wk - function vlaplace_sphere_wk(v,deriv,elem,var_coef,nu_ratio) result(laplace) + function vlaplace_sphere_wk(v,deriv,elem,var_coef,nu_ratio,tensor) result(laplace) ! ! input: v = vector in lat-lon coordinates ! ouput: weak laplacian of v, in lat-lon coordinates @@ -1127,27 +1135,33 @@ function vlaplace_sphere_wk(v,deriv,elem,var_coef,nu_ratio) result(laplace) type (derivative_t), intent(in) :: deriv type (element_t), intent(in) :: elem real(kind=real_kind), optional :: nu_ratio + real(kind=real_kind), optional :: tensor(np,np,2,2) + real(kind=real_kind) :: laplace(np,np,2) - if (hypervis_scaling/=0 .and. var_coef) then + if (var_coef) then ! tensorHV is turned on - requires cartesian formulation if (present(nu_ratio)) then if (nu_ratio /= 1) then call abortmp('ERROR: tensorHV can not be used with nu_div/=nu') endif endif - laplace=vlaplace_sphere_wk_cartesian(v,deriv,elem,var_coef) + if (present(tensor)) then + laplace=vlaplace_sphere_wk_cartesian(v,deriv,elem,var_coef,tensor) + else + laplace=vlaplace_sphere_wk_cartesian(v,deriv,elem,var_coef,elem%tensorVisc) + endif else ! all other cases, use contra formulation: - laplace=vlaplace_sphere_wk_contra(v,deriv,elem,var_coef,nu_ratio) + laplace=vlaplace_sphere_wk_contra(v,deriv,elem,nu_ratio) endif end function vlaplace_sphere_wk - function vlaplace_sphere_wk_cartesian(v,deriv,elem,var_coef) result(laplace) + function vlaplace_sphere_wk_cartesian(v,deriv,elem,var_coef,tensor) result(laplace) ! ! input: v = vector in lat-lon coordinates ! ouput: weak laplacian of v, in lat-lon coordinates @@ -1156,6 +1170,7 @@ function vlaplace_sphere_wk_cartesian(v,deriv,elem,var_coef) result(laplace) logical :: var_coef type (derivative_t), intent(in) :: deriv type (element_t), intent(in) :: elem + real(kind=real_kind) :: tensor(np,np,2,2) real(kind=real_kind) :: laplace(np,np,2) ! Local @@ -1165,14 +1180,14 @@ function vlaplace_sphere_wk_cartesian(v,deriv,elem,var_coef) result(laplace) ! latlon -> cartesian do component=1,3 -!JMD dum_cart(:,:,component)=sum( elem%vec_sphere2cart(:,:,component,:)*v(:,:,:) ,3) +!JMD dum_cart(:,:,component)=sum( elem%vec_sphere2cart(:,:,component,1:2)*v(:,:,:) ,3) dum_cart(:,:,component) = elem%vec_sphere2cart(:,:,component,1)*v(:,:,1) + & elem%vec_sphere2cart(:,:,component,2)*v(:,:,2) end do ! Do laplace on cartesian comps do component=1,3 - dum_cart(:,:,component) = laplace_sphere_wk(dum_cart(:,:,component),deriv,elem,var_coef) + dum_cart(:,:,component) = laplace_sphere_wk(dum_cart(:,:,component),deriv,elem,var_coef,tensor) enddo ! cartesian -> latlon @@ -1194,7 +1209,7 @@ end function vlaplace_sphere_wk_cartesian - function vlaplace_sphere_wk_contra(v,deriv,elem,var_coef,nu_ratio) result(laplace) + function vlaplace_sphere_wk_contra(v,deriv,elem,nu_ratio) result(laplace) ! ! input: v = vector in lat-lon coordinates ! ouput: weak laplacian of v, in lat-lon coordinates @@ -1205,7 +1220,6 @@ function vlaplace_sphere_wk_contra(v,deriv,elem,var_coef,nu_ratio) result(laplac ! = grad_wk(div) - curl_wk(vor) ! real(kind=real_kind), intent(in) :: v(np,np,2) - logical, intent(in) :: var_coef type (derivative_t), intent(in) :: deriv type (element_t), intent(in) :: elem real(kind=real_kind) :: laplace(np,np,2) diff --git a/components/homme/src/share/element_mod.F90 b/components/homme/src/share/element_mod.F90 index 390c8c54fe78..83e25deef0d9 100644 --- a/components/homme/src/share/element_mod.F90 +++ b/components/homme/src/share/element_mod.F90 @@ -58,7 +58,8 @@ module element_mod real (kind=real_kind) :: variable_hyperviscosity(np,np) ! hyperviscosity based on above real (kind=real_kind) :: hv_courant ! hyperviscosity courant number - real (kind=real_kind) :: tensorVisc(np,np,2,2) !og, matrix V for tensor viscosity + real (kind=real_kind) :: tensorVisc(np,np,2,2) ! matrix V for tensor hyperviscosity + real (kind=real_kind) :: tensorVisc_2(np,np,2,2) ! matrix V for tensor viscosity type (GridVertex_t) :: vertex ! element grid vertex information type (EdgeDescriptor_t) :: desc @@ -116,7 +117,7 @@ module element_mod ! The transpose of this operation is its pseudoinverse. ! This is just "identity" for plane - real (kind=real_kind) :: vec_sphere2cart(np,np,3,2) + real (kind=real_kind) :: vec_sphere2cart(np,np,3,3) ! Mass matrix terms for an element on a cube or reference face real (kind=real_kind) :: mp(np,np) ! mass matrix on v and p grid diff --git a/components/homme/src/share/global_norms_mod.F90 b/components/homme/src/share/global_norms_mod.F90 index 6f160e4610db..dff07decdd12 100644 --- a/components/homme/src/share/global_norms_mod.F90 +++ b/components/homme/src/share/global_norms_mod.F90 @@ -255,7 +255,7 @@ subroutine print_cfl(elem,hybrid,nets,nete) use reduction_mod, only : ParallelMin,ParallelMax use physical_constants, only : scale_factor_inv use control_mod, only : nu, nu_q, nu_div, hypervis_order, nu_top, & - hypervis_scaling, dcmip16_mu,dcmip16_mu_s + hypervis_scaling, laplace_scaling, dcmip16_mu,dcmip16_mu_s use control_mod, only : tstep_type type(element_t) , intent(inout) :: elem(:) @@ -265,7 +265,7 @@ subroutine print_cfl(elem,hybrid,nets,nete) ! Element statisics real (kind=real_kind) :: min_max_dx ! used for normalizing scalar HV real (kind=real_kind) :: max_normDinv ! used for CFL - real (kind=real_kind) :: normDinv_hypervis + real (kind=real_kind) :: normDinv_hypervis, normDinv_laplace real (kind=real_kind) :: lambda_max, lambda_vis, min_gw, lambda, nu_div_actual, nu_top_actual integer :: ie type (quadrature_t) :: gp @@ -338,6 +338,16 @@ subroutine print_cfl(elem,hybrid,nets,nete) ! constant coefficient formula: normDinv_hypervis = (lambda_vis**2) * (scale_factor_inv*max_normDinv)**4 endif + if (laplace_scaling/=0) then + ! tensor laplace. New eigenvalues are the eigenvalues of the tensor V + ! formulas here must match what is in cube_mod.F90 + lambda = max_normDinv**2 + normDinv_laplace = (lambda_vis) * (max_normDinv**2) * & + (lambda**(-laplace_scaling/2) ) + else + ! constant coefficient formula: + normDinv_laplace = (lambda_vis) * (scale_factor_inv*max_normDinv)**2 + endif if (hybrid%masterthread) then write(iulog,'(a,f10.2)') 'CFL estimates in terms of S=time step stability region' @@ -375,12 +385,12 @@ subroutine print_cfl(elem,hybrid,nets,nete) if(nu_top>0) then #ifdef MODEL_THETA_L nu_top_actual=maxval(nu_scale_top)*nu_top - write(iulog,'(a,f10.2,a)') 'scaled nu_top viscosity CFL: dt < S*', & - 1.0d0/(nu_top_actual*((scale_factor_inv*max_normDinv)**2)*lambda_vis),'s' + write(iulog,'(a,f12.4,a)') 'scaled nu_top viscosity CFL: dt < S*', & + 1.0d0/(nu_top_actual*normDinv_laplace),'s' #else nu_top_actual=4*nu_top write(iulog,'(a,f10.2,a)') '4*nu_top viscosity CFL: dt < S*', & - 1.0d0/(nu_top_actual*((scale_factor_inv*max_normDinv)**2)*lambda_vis),'s' + 1.0d0/(nu_top_actual*normDinv_laplace),'s' #endif end if @@ -410,7 +420,7 @@ subroutine dss_hvtensor(elem,hybrid,nets,nete) use dimensions_mod, only : np use quadrature_mod, only : gausslobatto, quadrature_t - use control_mod, only : hypervis_scaling + use control_mod, only : hypervis_scaling, laplace_scaling use edge_mod, only : initedgebuffer, FreeEdgeBuffer, edgeVpack, edgeVunpack use bndry_mod, only : bndry_exchangeV @@ -430,6 +440,12 @@ subroutine dss_hvtensor(elem,hybrid,nets,nete) ! this block of code will DSS it so the tensor if C0 ! and also make it bilinear in each element. ! Oksana Guba + ! + ! The tensorVisc_2() array (used by the sponge-layer/nu_top tensor + ! viscosity, controlled by laplace_scaling) is computed in cube_mod.F90 + ! in the same way as tensorVisc, and is DSS'd/bilinearized below with + ! an independent block, gated on laplace_scaling rather than + ! hypervis_scaling (the two controls are independent of one another). !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! if (hypervis_scaling /= 0) then @@ -480,6 +496,54 @@ subroutine dss_hvtensor(elem,hybrid,nets,nete) deallocate(gp%weights) endif + if (laplace_scaling /= 0) then + call initEdgeBuffer(hybrid%par,edgebuf,elem,1) + do rowind=1,2 + do colind=1,2 + do ie=nets,nete + zeta(:,:,ie) = elem(ie)%tensorVisc_2(:,:,rowind,colind)*elem(ie)%spheremp(:,:) + call edgeVpack(edgebuf,zeta(1,1,ie),1,0,ie) + end do + + call bndry_exchangeV(hybrid,edgebuf) + do ie=nets,nete + call edgeVunpack(edgebuf,zeta(1,1,ie),1,0,ie) + elem(ie)%tensorVisc_2(:,:,rowind,colind) = zeta(:,:,ie)*elem(ie)%rspheremp(:,:) + end do + enddo !rowind + enddo !colind + call FreeEdgeBuffer(edgebuf) + + gp=gausslobatto(np) + + !IF BILINEAR MAP OF V NEEDED + do rowind=1,2 + do colind=1,2 + ! replace hypervis w/ bilinear based on continuous corner values + do ie=nets,nete + noreast = elem(ie)%tensorVisc_2(np,np,rowind,colind) + nw = elem(ie)%tensorVisc_2(1,np,rowind,colind) + se = elem(ie)%tensorVisc_2(np,1,rowind,colind) + sw = elem(ie)%tensorVisc_2(1,1,rowind,colind) + do i=1,np + x = gp%points(i) + do j=1,np + y = gp%points(j) + elem(ie)%tensorVisc_2(i,j,rowind,colind) = 0.25d0*( & + (1.0d0-x)*(1.0d0-y)*sw + & + (1.0d0-x)*(y+1.0d0)*nw + & + (x+1.0d0)*(1.0d0-y)*se + & + (x+1.0d0)*(y+1.0d0)*noreast) + end do + end do + end do + enddo !rowind + enddo !colind + + deallocate(gp%points) + deallocate(gp%weights) + endif + end subroutine dss_hvtensor ! ================================ diff --git a/components/homme/src/share/hybvcoord_mod.F90 b/components/homme/src/share/hybvcoord_mod.F90 index d9cf71dba072..77a2738b0181 100644 --- a/components/homme/src/share/hybvcoord_mod.F90 +++ b/components/homme/src/share/hybvcoord_mod.F90 @@ -148,6 +148,12 @@ function hvcoord_init(hvfile_mid, hvfile_int, lprint, masterproc, ierr) result(h end if endif + ! Mark error if the B coefficient at model top is non-zero. + if (hvcoord%hybi(1) .ne. 0._r8) then + write(iulog,*) 'error: hvcoord%hybi(1) is non-zero' + ierr = 99 + end if + #if (defined HORIZ_OPENMP) !$OMP END CRITICAL #endif diff --git a/components/homme/src/share/namelist_mod.F90 b/components/homme/src/share/namelist_mod.F90 index 4c8dedfe280a..e59c7b892584 100644 --- a/components/homme/src/share/namelist_mod.F90 +++ b/components/homme/src/share/namelist_mod.F90 @@ -76,6 +76,7 @@ module namelist_mod dcmip16_pbl_type,& interp_lon0, & hypervis_scaling, & ! use tensor HV instead of scalar coefficient + laplace_scaling, & ! use tensor laplace instead of scalar coefficient disable_diagnostics, & ! use to disable diagnostics for timing reasons hypervis_order, & hypervis_subcycle, & @@ -316,6 +317,7 @@ subroutine readnl(par) hypervis_subcycle_tom, & hypervis_subcycle_q, & hypervis_scaling, & + laplace_scaling, & smooth_phis_numcycle, & smooth_phis_p2filt, & smooth_phis_nudt, & @@ -815,6 +817,7 @@ subroutine readnl(par) call MPI_bcast(disable_diagnostics,1,MPIlogical_t,par%root,par%comm,ierr) call MPI_bcast(hypervis_order,1,MPIinteger_t ,par%root,par%comm,ierr) call MPI_bcast(hypervis_scaling,1,MPIreal_t ,par%root,par%comm,ierr) + call MPI_bcast(laplace_scaling,1,MPIreal_t ,par%root,par%comm,ierr) call MPI_bcast(hypervis_subcycle,1,MPIinteger_t ,par%root,par%comm,ierr) call MPI_bcast(hypervis_subcycle_tom,1,MPIinteger_t ,par%root,par%comm,ierr) call MPI_bcast(hypervis_subcycle_q,1,MPIinteger_t ,par%root,par%comm,ierr) @@ -1223,9 +1226,14 @@ subroutine readnl(par) write(iulog,*)"readnl: internal_diagnostics_level = ",internal_diagnostics_level if(hypervis_scaling /=0)then - write(iulog,*)"Tensor hyperviscosity: hypervis_scaling=",hypervis_scaling + write(iulog,*)"Tensor hyperviscosity: hypervis_scaling=",hypervis_scaling else - write(iulog,*)"Constant (hyper)viscosity used." + write(iulog,*)"Constant (hyper)viscosity. hypervis_scaling=",hypervis_scaling + endif + if(laplace_scaling /=0)then + write(iulog,*)"Sponge layer viscosity: laplace_scaling=",laplace_scaling + else + write(iulog,*)"Sponge layer Constant viscosity. laplace_scaling=",laplace_scaling endif write(iulog,*)"hypervis_subcycle = ",hypervis_subcycle diff --git a/components/homme/src/share/planar_mod.F90 b/components/homme/src/share/planar_mod.F90 index 2af18ddcece3..1ed7201e49f4 100644 --- a/components/homme/src/share/planar_mod.F90 +++ b/components/homme/src/share/planar_mod.F90 @@ -306,6 +306,10 @@ subroutine coordinates_atomic(elem,gll_points) elem%vec_sphere2cart(:,:,1,2) = 0.0_real_kind elem%vec_sphere2cart(:,:,2,2) = 1.0_real_kind elem%vec_sphere2cart(:,:,3,2) = 0.0_real_kind + ! z direction = vertical direction + elem%vec_sphere2cart(:,:,1,3) = 0.0_real_kind + elem%vec_sphere2cart(:,:,2,3) = 0.0_real_kind + elem%vec_sphere2cart(:,:,3,3) = 1.0_real_kind end subroutine coordinates_atomic diff --git a/components/homme/src/share/sl_advection.F90 b/components/homme/src/share/sl_advection.F90 index 44a794c58922..ec17b2e79711 100644 --- a/components/homme/src/share/sl_advection.F90 +++ b/components/homme/src/share/sl_advection.F90 @@ -710,7 +710,7 @@ subroutine ALE_departure_from_gll(acart, ndim, vstar, elem, dt, normalize) ! (This is just a faster way of doing a dot product for each grid point, ! since reindexing the inputs to use the intrinsic effectively would be ! just asking for trouble.) - uxyz(:,:,i)=sum( elem%vec_sphere2cart(:,:,i,:)*vstar(:,:,:) ,3) + uxyz(:,:,i)=sum( elem%vec_sphere2cart(:,:,i,1:2)*vstar(:,:,:) ,3) end do ! compute departure point ! crude, 1st order accurate approximation. to be improved @@ -910,8 +910,6 @@ subroutine biharmonic_wk_scalar(elem,qtens,deriv,edgeq,hybrid,nets,nete,nq) !if tensor hyperviscosity with tensor V is used, then biharmonic operator is (\grad\cdot V\grad) (\grad \cdot \grad) !so tensor is only used on second call to laplace_sphere_wk - var_coef1 = .true. - if(hypervis_scaling > 0) var_coef1 = .false. do ie=nets,nete #if (defined COLUMN_OPENMP) @@ -921,7 +919,7 @@ subroutine biharmonic_wk_scalar(elem,qtens,deriv,edgeq,hybrid,nets,nete,nq) do k=1,nlev ! Potential loop inversion (AAM) lap_p(:,:)=qtens(:,:,k,q,ie) ! Original use of qtens on left and right hand sides caused OpenMP errors (AAM) - qtens(:,:,k,q,ie)=laplace_sphere_wk(lap_p,deriv,elem(ie),var_coef=var_coef1) + qtens(:,:,k,q,ie)=laplace_sphere_wk(lap_p,deriv,elem(ie),var_coef=.false.) enddo call edgeVpack_nlyr(edgeq, elem(ie)%desc, qtens(:,:,:,q,ie),nlev,nlev*(q-1),nq*nlev) enddo @@ -941,7 +939,7 @@ subroutine biharmonic_wk_scalar(elem,qtens,deriv,edgeq,hybrid,nets,nete,nq) call edgeVunpack_nlyr(edgeq,elem(ie)%desc,qtens(:,:,:,q,ie),nlev,nlev*(q-1),nq*nlev) do k=1,nlev ! Potential loop inversion (AAM) lap_p(:,:)=elem(ie)%rspheremp(:,:)*qtens(:,:,k,q,ie) - qtens(:,:,k,q,ie)=laplace_sphere_wk(lap_p,deriv,elem(ie),var_coef=.true.) + qtens(:,:,k,q,ie)=laplace_sphere_wk(lap_p,deriv,elem(ie),(hypervis_scaling>0)) enddo enddo enddo @@ -1648,7 +1646,7 @@ subroutine calc_vel_horiz_formula_node_ref_mid( & vfsph = half*vfsph ! Transform to Cartesian. do d = 1, 3 - vnode(d,:,:,k) = sum(elem%vec_sphere2cart(:,:,d,:)*vfsph, 3) + vnode(d,:,:,k) = sum(elem%vec_sphere2cart(:,:,d,1:2)*vfsph, 3) end do end do end subroutine calc_vel_horiz_formula_node_ref_mid diff --git a/components/homme/src/share/viscosity_base.F90 b/components/homme/src/share/viscosity_base.F90 index ccfff35c959a..672ead84f8e3 100644 --- a/components/homme/src/share/viscosity_base.F90 +++ b/components/homme/src/share/viscosity_base.F90 @@ -17,7 +17,8 @@ module viscosity_base use hybrid_mod, only : hybrid_t, hybrid_create use parallel_mod, only : parallel_t, abortmp use element_mod, only : element_t -use derivative_mod, only : derivative_t, laplace_sphere_wk, vlaplace_sphere_wk, vorticity_sphere, derivinit, divergence_sphere +use derivative_mod, only : derivative_t, laplace_sphere_wk, vlaplace_sphere_wk, vorticity_sphere, derivinit,& + divergence_sphere use edgetype_mod, only : EdgeBuffer_t, EdgeDescriptor_t use edge_mod, only : edgevpack, edgevunpack, edgevunpackmin, & edgevunpackmax, initEdgeBuffer, FreeEdgeBuffer, edgeSunpackmax, edgeSunpackmin,edgeSpack, & @@ -85,13 +86,6 @@ subroutine biharmonic_wk_scalar(elem,qtens,deriv,edgeq,hybrid,nets,nete) ! local integer :: k,kptr,i,j,ie,ic,q real (kind=real_kind), dimension(np,np) :: lap_p -logical var_coef1 - - !if tensor hyperviscosity with tensor V is used, then biharmonic operator is (\grad\cdot V\grad) (\grad \cdot \grad) - !so tensor is only used on second call to laplace_sphere_wk - var_coef1 = .true. - if(hypervis_scaling > 0) var_coef1 = .false. - do ie=nets,nete @@ -102,7 +96,7 @@ subroutine biharmonic_wk_scalar(elem,qtens,deriv,edgeq,hybrid,nets,nete) do k=1,nlev ! Potential loop inversion (AAM) lap_p(:,:)=qtens(:,:,k,q,ie) ! Original use of qtens on left and right hand sides caused OpenMP errors (AAM) - qtens(:,:,k,q,ie)=laplace_sphere_wk(lap_p,deriv,elem(ie),var_coef=var_coef1) + qtens(:,:,k,q,ie)=laplace_sphere_wk(lap_p,deriv,elem(ie),var_coef=.false.) enddo call edgeVpack_nlyr(edgeq, elem(ie)%desc, qtens(:,:,:,q,ie),nlev,nlev*(q-1),qsize*nlev) enddo @@ -122,7 +116,7 @@ subroutine biharmonic_wk_scalar(elem,qtens,deriv,edgeq,hybrid,nets,nete) call edgeVunpack_nlyr(edgeq,elem(ie)%desc,qtens(:,:,:,q,ie),nlev,nlev*(q-1),qsize*nlev) do k=1,nlev ! Potential loop inversion (AAM) lap_p(:,:)=elem(ie)%rspheremp(:,:)*qtens(:,:,k,q,ie) - qtens(:,:,k,q,ie)=laplace_sphere_wk(lap_p,deriv,elem(ie),var_coef=.true.) + qtens(:,:,k,q,ie)=laplace_sphere_wk(lap_p,deriv,elem(ie),var_coef=(hypervis_scaling>0)) enddo enddo enddo @@ -645,7 +639,7 @@ subroutine smooth_phis(phis,elem,hybrid,deriv,nets,nete,minf,numcycle,p2filt,xgl do ie=nets,nete - pstens(:,:,ie)=laplace_sphere_wk(phis(:,:,ie),deriv,elem(ie),var_coef=.true.) + pstens(:,:,ie)=laplace_sphere_wk(phis(:,:,ie),deriv,elem(ie),var_coef=(hypervis_scaling>0)) enddo do ie=nets,nete diff --git a/components/homme/src/sweqx/viscosity_mod.F90 b/components/homme/src/sweqx/viscosity_mod.F90 index 56fab03084af..ab1d736544e7 100644 --- a/components/homme/src/sweqx/viscosity_mod.F90 +++ b/components/homme/src/sweqx/viscosity_mod.F90 @@ -53,8 +53,6 @@ subroutine biharmonic_wk(elem,ptens,vtens,deriv,edge3,hybrid,nt,nets,nete) !if tensor hyperviscosity with tensor V is used, then biharmonic operator is (\grad\cdot V\grad) (\grad \cdot \grad) !so tensor is only used on second call to laplace_sphere_wk - var_coef1 = .true. - if(hypervis_scaling > 0) var_coef1= .false. ! note: there is a scaling bug in the treatment of nu_div ! nu_ratio is applied twice, once in each laplace operator @@ -85,9 +83,9 @@ subroutine biharmonic_wk(elem,ptens,vtens,deriv,edge3,hybrid,nt,nets,nete) enddo enddo - ptens(:,:,k,ie)=laplace_sphere_wk(T(:,:,k),deriv,elem(ie),var_coef=var_coef1) + ptens(:,:,k,ie)=laplace_sphere_wk(T(:,:,k),deriv,elem(ie),var_coef=.false.) vtens(:,:,:,k,ie)=vlaplace_sphere_wk(elem(ie)%state%v(:,:,:,k,nt),deriv,& - elem(ie),var_coef=var_coef1,nu_ratio=nu_ratio1) + elem(ie),var_coef=.false.,nu_ratio=nu_ratio1) enddo kptr=0 @@ -116,8 +114,8 @@ subroutine biharmonic_wk(elem,ptens,vtens,deriv,edge3,hybrid,nt,nets,nete) v(i,j,2)=rspheremv(i,j)*vtens(i,j,2,k,ie) enddo enddo - ptens(:,:,k,ie)=laplace_sphere_wk(T(:,:,k),deriv,elem(ie),var_coef=.true.) - vtens(:,:,:,k,ie)=vlaplace_sphere_wk(v(:,:,:),deriv,elem(ie),var_coef=.true.,& + ptens(:,:,k,ie)=laplace_sphere_wk(T(:,:,k),deriv,elem(ie),var_coef=(hypervis_scaling>0)) + vtens(:,:,:,k,ie)=vlaplace_sphere_wk(v(:,:,:),deriv,elem(ie),var_coef=(hypervis_scaling>0),& nu_ratio=nu_ratio2) enddo enddo diff --git a/components/homme/src/theta-l/share/prim_advance_mod.F90 b/components/homme/src/theta-l/share/prim_advance_mod.F90 index f263676544d9..d80eea1840d8 100644 --- a/components/homme/src/theta-l/share/prim_advance_mod.F90 +++ b/components/homme/src/theta-l/share/prim_advance_mod.F90 @@ -18,7 +18,7 @@ module prim_advance_mod use control_mod, only: dcmip16_mu, dcmip16_mu_s, hypervis_order, hypervis_subcycle,& integration, nu, nu_div, nu_p, nu_s, nu_top, prescribed_wind, qsplit, rsplit, test_case,& theta_hydrostatic_mode, tstep_type, theta_advect_form, hypervis_subcycle_tom, pgrad_correction,& - vtheta_thresh, dp3d_thresh + vtheta_thresh, dp3d_thresh, laplace_scaling use derivative_mod, only: derivative_t, divergence_sphere, gradient_sphere, laplace_sphere_wk,& laplace_z, vorticity_sphere, vlaplace_sphere_wk use derivative_mod, only: subcell_div_fluxes, subcell_dss_fluxes @@ -784,11 +784,11 @@ subroutine advance_hypervis(elem,hvcoord,hybrid,deriv,nt,nets,nete,dt2,eta_ave_w do ie=nets,nete do k=1,nlev_tom ! add regular diffusion near top - lap_s(:,:,1)=laplace_sphere_wk(elem(ie)%state%dp3d (:,:,k,nt),deriv,elem(ie),var_coef=.false.) - lap_s(:,:,2)=laplace_sphere_wk(elem(ie)%state%vtheta_dp(:,:,k,nt),deriv,elem(ie),var_coef=.false.) - lap_s(:,:,3)=laplace_sphere_wk(elem(ie)%state%w_i (:,:,k,nt),deriv,elem(ie),var_coef=.false.) - lap_s(:,:,4)=laplace_sphere_wk(elem(ie)%state%phinh_i (:,:,k,nt),deriv,elem(ie),var_coef=.false.) - lap_v=vlaplace_sphere_wk(elem(ie)%state%v (:,:,:,k,nt),deriv,elem(ie),var_coef=.false.) + lap_s(:,:,1)=laplace_sphere_wk(elem(ie)%state%dp3d (:,:,k,nt),deriv,elem(ie),var_coef=(laplace_scaling>0),tensor=elem(ie)%tensorVisc_2) + lap_s(:,:,2)=laplace_sphere_wk(elem(ie)%state%vtheta_dp(:,:,k,nt),deriv,elem(ie),var_coef=(laplace_scaling>0),tensor=elem(ie)%tensorVisc_2) + lap_s(:,:,3)=laplace_sphere_wk(elem(ie)%state%w_i (:,:,k,nt),deriv,elem(ie),var_coef=(laplace_scaling>0),tensor=elem(ie)%tensorVisc_2) + lap_s(:,:,4)=laplace_sphere_wk(elem(ie)%state%phinh_i (:,:,k,nt),deriv,elem(ie),var_coef=(laplace_scaling>0),tensor=elem(ie)%tensorVisc_2) + lap_v=vlaplace_sphere_wk(elem(ie)%state%v (:,:,:,k,nt),deriv,elem(ie),var_coef=(laplace_scaling>0),tensor=elem(ie)%tensorVisc_2) xfac=dt*nu_scale_top(k)*nu_top diff --git a/components/homme/src/theta-l/share/prim_state_mod.F90 b/components/homme/src/theta-l/share/prim_state_mod.F90 index 31b11b1481f0..dc41172aa34a 100644 --- a/components/homme/src/theta-l/share/prim_state_mod.F90 +++ b/components/homme/src/theta-l/share/prim_state_mod.F90 @@ -813,7 +813,7 @@ subroutine prim_printstate(elem, tl,hybrid,hvcoord,nets,nete) time0=time1 endif - + call flush(iulog) call t_stopf('prim_printstate') end subroutine prim_printstate diff --git a/components/homme/src/theta-l/share/viscosity_theta.F90 b/components/homme/src/theta-l/share/viscosity_theta.F90 index 6e325b13af71..772b7828b948 100644 --- a/components/homme/src/theta-l/share/viscosity_theta.F90 +++ b/components/homme/src/theta-l/share/viscosity_theta.F90 @@ -144,21 +144,21 @@ subroutine biharmonic_wk_theta(elem,stens,vtens,deriv,edgebuf,hybrid,nt,nets,net ! apply inverse mass matrix, then apply laplace again do k=1,nlev tmp(:,:)=rspheremv(:,:)*stens(:,:,k,1,ie) - stens(:,:,k,1,ie)=laplace_sphere_wk(tmp,deriv,elem(ie),var_coef=.true.) + stens(:,:,k,1,ie)=laplace_sphere_wk(tmp,deriv,elem(ie),var_coef=(hypervis_scaling>0)) tmp(:,:)=rspheremv(:,:)*stens(:,:,k,2,ie) - stens(:,:,k,2,ie)=laplace_sphere_wk(tmp,deriv,elem(ie),var_coef=.true.) + stens(:,:,k,2,ie)=laplace_sphere_wk(tmp,deriv,elem(ie),var_coef=(hypervis_scaling>0)) tmp(:,:)=rspheremv(:,:)*stens(:,:,k,3,ie) - stens(:,:,k,3,ie)=laplace_sphere_wk(tmp,deriv,elem(ie),var_coef=.true.) + stens(:,:,k,3,ie)=laplace_sphere_wk(tmp,deriv,elem(ie),var_coef=(hypervis_scaling>0)) tmp(:,:)=rspheremv(:,:)*stens(:,:,k,4,ie) - stens(:,:,k,4,ie)=laplace_sphere_wk(tmp,deriv,elem(ie),var_coef=.true.) + stens(:,:,k,4,ie)=laplace_sphere_wk(tmp,deriv,elem(ie),var_coef=(hypervis_scaling>0)) v(:,:,1)=rspheremv(:,:)*vtens(:,:,1,k,ie) v(:,:,2)=rspheremv(:,:)*vtens(:,:,2,k,ie) vtens(:,:,:,k,ie)=vlaplace_sphere_wk(v(:,:,:),deriv,elem(ie),& - var_coef=.true.,nu_ratio=nu_ratio2) + var_coef=(hypervis_scaling>0),nu_ratio=nu_ratio2) enddo enddo diff --git a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp index 7324cf093667..0737b8b087fd 100644 --- a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp @@ -27,7 +27,7 @@ HyperviscosityFunctorImpl (const SimulationParams& params, , m_data (params.hypervis_subcycle,params.hypervis_subcycle_tom, params.nu_ratio1,params.nu_ratio2,params.nu_top,params.nu, params.nu_p,params.nu_s,params.hypervis_scaling, - params.do_3d_turbulence, params.tom_sponge_start) + params.do_3d_turbulence, params.tom_sponge_start, params.laplace_scaling) , m_state (state) , m_derived (derived) , m_geometry (geometry) @@ -36,7 +36,6 @@ HyperviscosityFunctorImpl (const SimulationParams& params, , m_policy_update_states (Homme::get_default_team_policy(m_num_elems)) , m_policy_first_laplace (Homme::get_default_team_policy(m_num_elems)) , m_policy_pre_exchange (Homme::get_default_team_policy(m_num_elems)) - , m_policy_nutop_laplace (Homme::get_default_team_policy(m_num_elems)) , m_policy_nutop_update_states (Homme::get_default_team_policy(m_num_elems)) , m_policy_sgsturb_laplace (Homme::get_default_team_policy(m_num_elems)) , m_policy_sgsturb_update_states (Homme::get_default_team_policy(m_num_elems)) @@ -54,13 +53,12 @@ HyperviscosityFunctorImpl (const int num_elems, const SimulationParams ¶ms) , m_data (params.hypervis_subcycle,params.hypervis_subcycle_tom, params.nu_ratio1,params.nu_ratio2,params.nu_top,params.nu, params.nu_p,params.nu_s,params.hypervis_scaling, - params.do_3d_turbulence, params.tom_sponge_start) + params.do_3d_turbulence, params.tom_sponge_start, params.laplace_scaling) , m_hvcoord (Context::singleton().get()) , m_policy_update_states (Homme::get_default_team_policy(m_num_elems)) , m_policy_first_laplace (Homme::get_default_team_policy(m_num_elems)) , m_policy_pre_exchange (Homme::get_default_team_policy(m_num_elems)) - , m_policy_nutop_laplace (Homme::get_default_team_policy(m_num_elems)) - , m_policy_nutop_update_states (Homme::get_default_team_policy(m_num_elems)) + , m_policy_nutop_update_states (Homme::get_default_team_policy(m_num_elems)) , m_policy_sgsturb_laplace (Homme::get_default_team_policy(m_num_elems)) , m_policy_sgsturb_update_states (Homme::get_default_team_policy(m_num_elems)) , m_tu(m_policy_update_states) @@ -391,9 +389,16 @@ void HyperviscosityFunctorImpl::run (const int np1, const Real dt, const Real et // sponge layer if (m_data.nu_top > 0) { + const int ne = m_geometry.num_elems(); for (int icycle = 0; icycle < m_data.hypervis_subcycle_tom; ++icycle) { // laplace(fields) --> ttens, etc. - Kokkos::parallel_for(m_policy_nutop_laplace, *this); + if ( m_data.constsponge ) { + auto policy = Homme::get_default_team_policy(ne); + Kokkos::parallel_for(policy, *this); + } else { + auto policy = Homme::get_default_team_policy(ne); + Kokkos::parallel_for(policy, *this); + } Kokkos::fence(); // exchange is done on ttens, dptens, vtens, etc. @@ -435,9 +440,9 @@ void HyperviscosityFunctorImpl::biharmonic_wk_theta() const Kokkos::fence(); } //biharmonic -// Laplace for nu_top +// Laplace for nu_top, constant coefficient KOKKOS_INLINE_FUNCTION -void HyperviscosityFunctorImpl::operator() (const TagNutopLaplace&, const TeamMember& team) const { +void HyperviscosityFunctorImpl::operator() (const TagNutopLaplaceConst&, const TeamMember& team) const { KernelVariables kv(team, m_tu); using MidColumn = decltype(Homme::subview(m_buffers.wtens,0,0,0)); @@ -507,7 +512,88 @@ void HyperviscosityFunctorImpl::operator() (const TagNutopLaplace&, const TeamMe }); // threadvectorrange }); // teamthreadrange -} // TagNutopLaplace +} // TagNutopLaplaceConst + +// Laplace for nu_top, tensor coefficient (laplace_scaling>0), using tensorVisc_2 +KOKKOS_INLINE_FUNCTION +void HyperviscosityFunctorImpl::operator() (const TagNutopLaplaceTensor&, const TeamMember& team) const { + KernelVariables kv(team, m_tu); + + using MidColumn = decltype(Homme::subview(m_buffers.wtens,0,0,0)); + + const auto& tensorvisc2 = Homme::subview(m_geometry.m_tensorvisc2,kv.ie); + + // Laplacian of layer thickness + m_sphere_ops.laplace_tensor(kv, tensorvisc2, + Homme::subview(m_state.m_dp3d,kv.ie,m_data.np1), + Homme::subview(m_buffers.dptens,kv.ie), + m_nu_scale_top_ilev_pack_lim); + // Laplacian of theta + m_sphere_ops.laplace_tensor(kv, tensorvisc2, + Homme::subview(m_state.m_vtheta_dp,kv.ie,m_data.np1), + Homme::subview(m_buffers.ttens,kv.ie), + m_nu_scale_top_ilev_pack_lim); + + if (m_process_nh_vars) { + // Laplacian of vertical velocity (do not compute last interface) + m_sphere_ops.laplace_tensor(kv, tensorvisc2, + Homme::subview(m_state.m_w_i,kv.ie,m_data.np1), + Homme::subview(m_buffers.wtens,kv.ie), + m_nu_scale_top_ilev_pack_lim); + // Laplacian of geopotential (do not compute last interface) + m_sphere_ops.laplace_tensor(kv, tensorvisc2, + Homme::subview(m_state.m_phinh_i,kv.ie,m_data.np1), + Homme::subview(m_buffers.phitens,kv.ie), + m_nu_scale_top_ilev_pack_lim); + } + + // Laplacian of velocity + m_sphere_ops.vlaplace_sphere_wk_cartesian(kv, tensorvisc2, + Homme::subview(m_geometry.m_vec_sph2cart,kv.ie), + Homme::subview(m_state.m_v,kv.ie,m_data.np1), + Homme::subview(m_buffers.vtens,kv.ie), + m_nu_scale_top_ilev_pack_lim); + + kv.team_barrier(); + + Kokkos::parallel_for( + Kokkos::TeamThreadRange(kv.team,NP*NP), + [&] (const int idx) { + const int igp = idx / NP; + const int jgp = idx % NP; + + const auto utens = Homme::subview(m_buffers.vtens,kv.ie,0,igp,jgp); + const auto vtens = Homme::subview(m_buffers.vtens,kv.ie,1,igp,jgp); + const auto ttens = Homme::subview(m_buffers.ttens,kv.ie,igp,jgp); + const auto dptens = Homme::subview(m_buffers.dptens,kv.ie,igp,jgp); + + MidColumn wtens, phitens; + if (m_process_nh_vars) { + wtens = Homme::subview(m_buffers.wtens,kv.ie,igp,jgp); + phitens = Homme::subview(m_buffers.phitens,kv.ie,igp,jgp); + } + + // Note: only the first m_nu_scale_top_ilev_pack_lim packs are scaled and + // used here, exactly as in the constant-coefficient path -- laplace_tensor + // and vlaplace_sphere_wk_cartesian above are now passed the same runtime + // limit, so unused levels are not even computed. + Kokkos::parallel_for( + Kokkos::ThreadVectorRange(kv.team, m_nu_scale_top_ilev_pack_lim), + [&] (const int ilev) { + const auto xf = m_data.dt_hvs_tom * m_nu_scale_top(ilev) * m_data.nu_top; + utens(ilev) *= xf; + vtens(ilev) *= xf; + ttens(ilev) *= xf; + dptens(ilev) *= xf; + + if (m_process_nh_vars) { + wtens(ilev) *= xf; + phitens(ilev) *= xf; + } + + }); // threadvectorrange + }); // teamthreadrange +} // TagNutopLaplaceTensor KOKKOS_INLINE_FUNCTION void HyperviscosityFunctorImpl::operator() (const TagNutopUpdateStates&, const TeamMember& team) const { @@ -572,6 +658,53 @@ void HyperviscosityFunctorImpl::operator() (const TagSGSTurbLaplace&, const Team using MidColumn = decltype(Homme::subview(m_buffers.wtens,0,0,0)); using IntColumn = decltype(Homme::subview(m_state.m_w_i,0,0,0,0)); + Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team,NP*NP), + [&](const int idx) { + const int igp = idx / NP; + const int jgp = idx % NP; + + auto vtheta = Homme::subview(m_state.m_vtheta_dp,kv.ie,m_data.np1,igp,jgp); + auto dp = Homme::subview(m_state.m_dp3d,kv.ie,m_data.np1,igp,jgp); + auto theta_ref = Homme::subview(m_state.m_ref_states.theta_ref,kv.ie,igp,jgp); + auto dp_ref = Homme::subview(m_state.m_ref_states.dp_ref,kv.ie,igp,jgp); + + Kokkos::parallel_for(Kokkos::ThreadVectorRange(kv.team,NUM_LEV), + [&](const int ilev) { + vtheta(ilev) -= theta_ref(ilev); + dp(ilev) -= dp_ref(ilev); + }); + }); + + kv.team_barrier(); + + if (m_process_nh_vars) { + // Diffuse only the perturbational geopotential, not the terrain-following + // reference profile tied to phis. + Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team,NP*NP), + [&](const int idx) { + const int igp = idx / NP; + const int jgp = idx % NP; + + auto phi_i = Homme::subview(m_state.m_phinh_i,kv.ie,m_data.np1,igp,jgp); + auto phi_i_ref = Homme::subview(m_state.m_ref_states.phi_i_ref,kv.ie,igp,jgp); + + Kokkos::parallel_for(Kokkos::ThreadVectorRange(kv.team,NUM_LEV), + [&](const int ilev) { + phi_i(ilev) -= phi_i_ref(ilev); + }); + +#ifndef XX_NONBFB_COMING + if (NUM_LEV!=NUM_LEV_P) { + Kokkos::single(Kokkos::PerThread(kv.team),[&](){ + phi_i(NUM_LEV_P-1) -= phi_i_ref(NUM_LEV_P-1); + }); + } +#endif + }); + + kv.team_barrier(); + } + // Laplacian of layer thickness m_sphere_ops.laplace_simple(kv, Homme::subview(m_state.m_dp3d,kv.ie,m_data.np1), @@ -599,6 +732,51 @@ void HyperviscosityFunctorImpl::operator() (const TagSGSTurbLaplace&, const Team kv.team_barrier(); + if (m_process_nh_vars) { + Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team,NP*NP), + [&](const int idx) { + const int igp = idx / NP; + const int jgp = idx % NP; + + auto phi_i = Homme::subview(m_state.m_phinh_i,kv.ie,m_data.np1,igp,jgp); + auto phi_i_ref = Homme::subview(m_state.m_ref_states.phi_i_ref,kv.ie,igp,jgp); + + Kokkos::parallel_for(Kokkos::ThreadVectorRange(kv.team,NUM_LEV), + [&](const int ilev) { + phi_i(ilev) += phi_i_ref(ilev); + }); + +#ifndef XX_NONBFB_COMING + if (NUM_LEV!=NUM_LEV_P) { + Kokkos::single(Kokkos::PerThread(kv.team),[&](){ + phi_i(NUM_LEV_P-1) += phi_i_ref(NUM_LEV_P-1); + }); + } +#endif + }); + + kv.team_barrier(); + } + + Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team,NP*NP), + [&](const int idx) { + const int igp = idx / NP; + const int jgp = idx % NP; + + auto vtheta = Homme::subview(m_state.m_vtheta_dp,kv.ie,m_data.np1,igp,jgp); + auto dp = Homme::subview(m_state.m_dp3d,kv.ie,m_data.np1,igp,jgp); + auto theta_ref = Homme::subview(m_state.m_ref_states.theta_ref,kv.ie,igp,jgp); + auto dp_ref = Homme::subview(m_state.m_ref_states.dp_ref,kv.ie,igp,jgp); + + Kokkos::parallel_for(Kokkos::ThreadVectorRange(kv.team,NUM_LEV), + [&](const int ilev) { + vtheta(ilev) += theta_ref(ilev); + dp(ilev) += dp_ref(ilev); + }); + }); + + kv.team_barrier(); + Kokkos::parallel_for( Kokkos::TeamThreadRange(kv.team,NP*NP), [&] (const int idx) { @@ -703,6 +881,7 @@ void HyperviscosityFunctorImpl::operator() (const TagSGSTurbUpdateStates&, const w(k) += wtens(k); phi_i(k) += phitens(k); } + }); // threadvectorrange }); // threadteamrange } // tagSGSTurbUpdateStates diff --git a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.hpp b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.hpp index a2eddc5e5290..c394d2a8ff09 100644 --- a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.hpp @@ -41,12 +41,13 @@ class HyperviscosityFunctorImpl const Real nu_ratio1_in, const Real nu_ratio2_in, const Real nu_top_in, const Real nu_in, const Real nu_p_in, const Real nu_s_in, const Real hypervis_scaling_in, bool do_3d_turbulence_in, - const double tom_sponge_start_in) + const double tom_sponge_start_in, const Real laplace_scaling_in = 0.0) : hypervis_subcycle(hypervis_subcycle_in) , hypervis_subcycle_tom(hypervis_subcycle_tom_in) , nu_ratio1(nu_ratio1_in), nu_ratio2(nu_ratio2_in) , nu_top(nu_top_in), nu(nu_in), nu_p(nu_p_in), nu_s(nu_s_in) , consthv(hypervis_scaling_in == 0) + , constsponge(laplace_scaling_in == 0) , do_3d_turbulence(do_3d_turbulence_in) , tom_sponge_start(tom_sponge_start_in) {} @@ -71,6 +72,7 @@ class HyperviscosityFunctorImpl Real eta_ave_w; bool consthv; + bool constsponge; double tom_sponge_start; };//hyperviscosityData @@ -95,7 +97,8 @@ class HyperviscosityFunctorImpl struct TagApplyInvMass {}; struct TagHyperPreExchange {}; struct TagNutopUpdateStates {}; - struct TagNutopLaplace {}; + struct TagNutopLaplaceConst {}; + struct TagNutopLaplaceTensor {}; struct TagSGSTurbUpdateStates {}; struct TagSGSTurbLaplace {}; @@ -198,7 +201,10 @@ class HyperviscosityFunctorImpl // Laplace for nu_top KOKKOS_INLINE_FUNCTION - void operator()(const TagNutopLaplace&, const TeamMember& team) const; + void operator()(const TagNutopLaplaceConst&, const TeamMember& team) const; + + KOKKOS_INLINE_FUNCTION + void operator()(const TagNutopLaplaceTensor&, const TeamMember& team) const; KOKKOS_INLINE_FUNCTION void operator()(const TagNutopUpdateStates&, const TeamMember& team) const; @@ -424,7 +430,6 @@ class HyperviscosityFunctorImpl Kokkos::TeamPolicy m_policy_first_laplace; Kokkos::TeamPolicy m_policy_pre_exchange; - Kokkos::TeamPolicy m_policy_nutop_laplace; Kokkos::TeamPolicy m_policy_nutop_update_states; Kokkos::TeamPolicy m_policy_sgsturb_laplace; diff --git a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp index 0dd0ee02cdc9..2e7adfe4f480 100644 --- a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp @@ -42,7 +42,7 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, const int& time_step_type, const int& qsize, const int& state_frequency, const Real& nu, const Real& nu_p, const Real& nu_q, const Real& nu_s, const Real& nu_div, const Real& nu_top, const int& hypervis_order, const int& hypervis_subcycle, const int& hypervis_subcycle_tom, - const double& hypervis_scaling, const double& dcmip16_mu, + const double& hypervis_scaling, const double& laplace_scaling, const double& dcmip16_mu, const int& ftype, const int& theta_adv_form, const int& prescribed_wind, const int& use_moisture, const int& disable_diagnostics, const int& use_cpstar, const int& transport_alg, const int& theta_hydrostatic_mode, const char** test_case, const int& dt_remap_factor, const int& dt_tracer_factor, @@ -115,6 +115,7 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, params.hypervis_subcycle = hypervis_subcycle; params.hypervis_subcycle_tom = hypervis_subcycle_tom; params.hypervis_scaling = hypervis_scaling; + params.laplace_scaling = laplace_scaling; params.disable_diagnostics = (bool)disable_diagnostics; params.use_moisture = (bool)use_moisture; params.use_cpstar = (bool)use_cpstar; @@ -284,8 +285,7 @@ void init_elements_c (const int& num_elems) Elements& e = c.create (); const SimulationParams& params = c.get(); - const bool consthv = (params.hypervis_scaling==0.0); - e.init (num_elems, consthv, /* alloc_gradphis = */ true, + e.init (num_elems, /* alloc_gradphis = */ true, params.scale_factor, params.laplacian_rigid_factor, /* alloc_sphere_coords = */ params.transport_alg > 0); @@ -362,7 +362,13 @@ void init_functors_c (const int& allocate_buffer) #ifdef HOMME_ENABLE_COMPOSE else c.create_if_not_there(); #endif - auto& hvf = c.create_if_not_there(); + // Pass (num_elems, params) so that, like caar above, this uses the + // lazy-construction path (is_setup=false), forcing the setup_needed()/ + // setup() call below to actually run. That setup() call is what copies + // nu_scale_top/nu_scale_top_ilev_pack_lim from the Fortran-initialized + // ref states (needed when tom_sponge_start>0); the no-args constructor + // sets is_setup=true immediately, silently skipping that copy. + auto& hvf = c.create_if_not_there(elems.num_elems(), params); auto& ff = c.create_if_not_there(); auto& diag = c.create_if_not_there (elems.num_elems(),tracers.num_tracers(), params.theta_hydrostatic_mode); @@ -462,15 +468,37 @@ void init_elements_2d_c (const int& ie, CF90Ptr& spheremp, CF90Ptr& rspheremp, CF90Ptr& metdet, CF90Ptr& metinv, CF90Ptr &tensorvisc, CF90Ptr &vec_sph2cart, - double* sphere_cart_vec, double* sphere_latlon_vec) + double* sphere_cart_vec, double* sphere_latlon_vec, + CF90Ptr &tensorvisc2) { auto& c = Context::singleton(); Elements& e = c.get (); - const SimulationParams& params = c.get(); - const bool consthv = (params.hypervis_scaling==0.0); e.m_geometry.set_elem_data(ie,D,Dinv,fcor,spheremp,rspheremp,metdet,metinv,tensorvisc, - vec_sph2cart,consthv,sphere_cart_vec,sphere_latlon_vec); + vec_sph2cart,sphere_cart_vec,sphere_latlon_vec, + tensorvisc2); +} + +// Copies just tensorVisc from f90 arrays into the C++ view. Separate from +// init_elements_2d_c() so that it can be called again, after dss_hvtensor +// has updated tensorVisc, without re-copying the other (constant) geometry +// fields. +void init_tensorvisc_c (const int& ie, CF90Ptr& tensorvisc) +{ + auto& c = Context::singleton(); + Elements& e = c.get (); + + e.m_geometry.set_tensorvisc(ie,tensorvisc); +} + +// Same as init_tensorvisc_c(), but for tensorVisc_2 (the sponge-layer +// tensor coefficient), which is likewise recomputed by dss_hvtensor. +void init_tensorvisc2_c (const int& ie, CF90Ptr& tensorvisc2) +{ + auto& c = Context::singleton(); + Elements& e = c.get (); + + e.m_geometry.set_tensorvisc2(ie,tensorvisc2); } void init_geopotential_c (const int& ie, diff --git a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 index f844e3e9c6be..657bb8a07b79 100644 --- a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 +++ b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 @@ -22,6 +22,8 @@ module prim_driver_mod public :: prim_run_subcycle public :: prim_init_elements_views public :: prim_init_grid_views + public :: prim_init_tensorvisc + public :: prim_init_tensorvisc2 public :: prim_init_geopotential_views public :: prim_init_state_views public :: prim_init_ref_states_views @@ -62,6 +64,12 @@ subroutine prim_init2(elem, hybrid, nets, nete, tl, hvcoord) ! Init the c data structures call prim_create_c_data_structures(tl,hvcoord,elem(1)%mp) + ! Populate the ref-state views early: HyperviscosityFunctorImpl::setup(), + ! called from prim_init_kokkos_functors below, reads Fortran-computed + ! reference-state scalars (e.g. nu_scale_top_ilev_pack_lim) that are only + ! valid once this has run. + call prim_init_ref_states_views (elem) + !Init the kokkos functors (and their boundary exchanges) call prim_init_kokkos_functors () @@ -85,7 +93,7 @@ subroutine prim_create_c_data_structures (tl, hvcoord, mp) use control_mod, only : limiter_option, rsplit, qsplit, tstep_type, statefreq, & nu, nu_p, nu_q, nu_s, nu_div, nu_top, vert_remap_q_alg, & hypervis_order, hypervis_subcycle, hypervis_subcycle_tom,& - hypervis_scaling, & + hypervis_scaling, laplace_scaling, & ftype, prescribed_wind, use_moisture, disable_diagnostics, & use_cpstar, transport_alg, theta_hydrostatic_mode, & dcmip16_mu, theta_advect_form, test_case, & @@ -128,7 +136,7 @@ subroutine prim_create_c_data_structures (tl, hvcoord, mp) call init_simulation_params_c (vert_remap_q_alg, limiter_option, rsplit, qsplit, tstep_type, & qsize, statefreq, nu, nu_p, nu_q, nu_s, nu_div, nu_top, & hypervis_order, hypervis_subcycle, hypervis_subcycle_tom, & - hypervis_scaling, & + hypervis_scaling, laplace_scaling, & dcmip16_mu, ftype, theta_advect_form, & prescribed_wind, & use_moisture_int, & @@ -174,14 +182,16 @@ subroutine prim_init_grid_views (elem) ! Local(s) ! real (kind=real_kind), target, dimension(np,np,2,2) :: elem_D, elem_Dinv, elem_metinv, elem_tensorvisc + real (kind=real_kind), target, dimension(np,np,2,2) :: elem_tensorvisc2 real (kind=real_kind), target, dimension(np,np) :: elem_fcor, elem_spheremp real (kind=real_kind), target, dimension(np,np) :: elem_rspheremp, elem_metdet - real (kind=real_kind), target, dimension(np,np,3,2) :: elem_vec_sph2cart + real (kind=real_kind), target, dimension(np,np,3,3) :: elem_vec_sph2cart type (c_ptr) :: elem_D_ptr, elem_Dinv_ptr, elem_fcor_ptr type (c_ptr) :: elem_spheremp_ptr, elem_rspheremp_ptr type (c_ptr) :: elem_metdet_ptr, elem_metinv_ptr type (c_ptr) :: elem_tensorvisc_ptr, elem_vec_sph2cart_ptr + type (c_ptr) :: elem_tensorvisc2_ptr type (cartesian3D_t) :: sphere_cart real (kind=real_kind) :: sphere_cart_vec(3,np,np), sphere_latlon_vec(2,np,np) @@ -198,6 +208,7 @@ subroutine prim_init_grid_views (elem) elem_metinv_ptr = c_loc(elem_metinv) elem_tensorvisc_ptr = c_loc(elem_tensorvisc) elem_vec_sph2cart_ptr = c_loc(elem_vec_sph2cart) + elem_tensorvisc2_ptr = c_loc(elem_tensorvisc2) is_sphere = trim(geometry) /= 'plane' @@ -210,6 +221,7 @@ subroutine prim_init_grid_views (elem) elem_metdet = elem(ie)%metdet elem_metinv = elem(ie)%metinv elem_tensorvisc = elem(ie)%tensorVisc + elem_tensorvisc2 = elem(ie)%tensorVisc_2 elem_vec_sph2cart = elem(ie)%vec_sphere2cart do j = 1,np do i = 1,np @@ -232,10 +244,65 @@ subroutine prim_init_grid_views (elem) elem_spheremp_ptr, elem_rspheremp_ptr, & elem_metdet_ptr, elem_metinv_ptr, & elem_tensorvisc_ptr, elem_vec_sph2cart_ptr,& - sphere_cart_vec, sphere_latlon_vec) + sphere_cart_vec, sphere_latlon_vec, & + elem_tensorvisc2_ptr) enddo end subroutine prim_init_grid_views + ! Copies just tensorVisc into the C++ ElementsGeometry. This is used to + ! (re)populate tensorVisc after dss_hvtensor() has updated it, without + ! re-copying the other (constant) geometry fields that prim_init_grid_views + ! already sent to C++ earlier. + subroutine prim_init_tensorvisc (elem) + use iso_c_binding, only : c_ptr, c_loc + use element_mod, only : element_t + use theta_f2c_mod, only : init_tensorvisc_c + ! + ! Input(s) + ! + type (element_t), intent(in) :: elem (:) + ! + ! Local(s) + ! + real (kind=real_kind), target, dimension(np,np,2,2) :: elem_tensorvisc + type (c_ptr) :: elem_tensorvisc_ptr + + integer :: ie + + elem_tensorvisc_ptr = c_loc(elem_tensorvisc) + + do ie=1,nelemd + elem_tensorvisc = elem(ie)%tensorVisc + call init_tensorvisc_c (ie-1, elem_tensorvisc_ptr) + enddo + end subroutine prim_init_tensorvisc + + ! Same as prim_init_tensorvisc, but for tensorVisc_2 (the sponge-layer + ! tensor coefficient), which is likewise recomputed by dss_hvtensor. + subroutine prim_init_tensorvisc2 (elem) + use iso_c_binding, only : c_ptr, c_loc + use element_mod, only : element_t + use theta_f2c_mod, only : init_tensorvisc2_c + ! + ! Input(s) + ! + type (element_t), intent(in) :: elem (:) + ! + ! Local(s) + ! + real (kind=real_kind), target, dimension(np,np,2,2) :: elem_tensorvisc2 + type (c_ptr) :: elem_tensorvisc2_ptr + + integer :: ie + + elem_tensorvisc2_ptr = c_loc(elem_tensorvisc2) + + do ie=1,nelemd + elem_tensorvisc2 = elem(ie)%tensorVisc_2 + call init_tensorvisc2_c (ie-1, elem_tensorvisc2_ptr) + enddo + end subroutine prim_init_tensorvisc2 + subroutine prim_init_geopotential_views (elem) use iso_c_binding, only : c_ptr, c_loc use element_mod, only : element_t @@ -364,9 +431,10 @@ subroutine prim_init_elements_views (elem) ! Initialize the 3d states views in C++ call prim_init_state_views (elem) - - ! Initialize the reference states in C++ - call prim_init_ref_states_views (elem) + + ! Note: reference states are already initialized earlier in prim_init2, + ! before prim_init_kokkos_functors (needed there for + ! HyperviscosityFunctorImpl::setup() to read valid nu_scale_top data). ! Initialize the diagnostics arrays in C++ call prim_init_diags_views (elem) diff --git a/components/homme/src/theta-l_kokkos/theta_f2c_mod.F90 b/components/homme/src/theta-l_kokkos/theta_f2c_mod.F90 index e682ac616f87..76fa87bf36bc 100644 --- a/components/homme/src/theta-l_kokkos/theta_f2c_mod.F90 +++ b/components/homme/src/theta-l_kokkos/theta_f2c_mod.F90 @@ -10,7 +10,7 @@ module theta_f2c_mod subroutine init_simulation_params_c (remap_alg, limiter_option, rsplit, qsplit, time_step_type, & qsize, state_frequency, nu, nu_p, nu_q, nu_s, nu_div, nu_top, & hypervis_order, hypervis_subcycle, hypervis_subcycle_tom, & - hypervis_scaling, & + hypervis_scaling, laplace_scaling, & dcmip16_mu, ftype, theta_adv_form, prescribed_wind, use_moisture, & disable_diagnostics, use_cpstar, transport_alg, & theta_hydrostatic_mode, test_case_name, dt_remap_factor, & @@ -25,7 +25,7 @@ subroutine init_simulation_params_c (remap_alg, limiter_option, rsplit, qsplit, integer(kind=c_int), intent(in) :: remap_alg, limiter_option, rsplit, qsplit, time_step_type, nsplit integer(kind=c_int), intent(in) :: dt_remap_factor, dt_tracer_factor, transport_alg integer(kind=c_int), intent(in) :: state_frequency, qsize, internal_diagnostics_level - real(kind=c_double), intent(in) :: nu, nu_p, nu_q, nu_s, nu_div, nu_top, hypervis_scaling, dcmip16_mu, & + real(kind=c_double), intent(in) :: nu, nu_p, nu_q, nu_s, nu_div, nu_top, hypervis_scaling, laplace_scaling, dcmip16_mu, & scale_factor, laplacian_rigid_factor, dp3d_thresh, vtheta_thresh integer(kind=c_int), intent(in) :: hypervis_order, hypervis_subcycle, hypervis_subcycle_tom integer(kind=c_int), intent(in) :: ftype, theta_adv_form @@ -69,7 +69,8 @@ subroutine init_elements_2d_c (ie, D_ptr, Dinv_ptr, elem_fcor_ptr, & elem_spheremp_ptr, elem_rspheremp_ptr, & elem_metdet_ptr, elem_metinv_ptr, & tensorvisc_ptr, vec_sph2cart_ptr, & - sphere_cart_vec, sphere_latlon_vec) bind(c) + sphere_cart_vec, sphere_latlon_vec, & + tensorvisc2_ptr) bind(c) use iso_c_binding, only: c_int, c_ptr, c_double use dimensions_mod, only : np ! @@ -81,8 +82,32 @@ subroutine init_elements_2d_c (ie, D_ptr, Dinv_ptr, elem_fcor_ptr, & type (c_ptr) , intent(in) :: elem_metdet_ptr, elem_metinv_ptr type (c_ptr) , intent(in) :: tensorvisc_ptr, vec_sph2cart_ptr real (kind=c_double), intent(in) :: sphere_cart_vec(3,np,np), sphere_latlon_vec(2,np,np) + type (c_ptr) , intent(in) :: tensorvisc2_ptr end subroutine init_elements_2d_c + ! Copies just tensorVisc from f90 arrays into the C++ view. Used to + ! (re)populate tensorVisc after dss_hvtensor has updated it, without + ! touching the other (constant) geometry fields. + subroutine init_tensorvisc_c (ie, tensorvisc_ptr) bind(c) + use iso_c_binding, only: c_int, c_ptr + ! + ! Inputs + ! + integer (kind=c_int), intent(in) :: ie + type (c_ptr) , intent(in) :: tensorvisc_ptr + end subroutine init_tensorvisc_c + + ! Same as init_tensorvisc_c, but for tensorVisc_2 (the sponge-layer + ! tensor coefficient), which is likewise recomputed by dss_hvtensor. + subroutine init_tensorvisc2_c (ie, tensorvisc2_ptr) bind(c) + use iso_c_binding, only: c_int, c_ptr + ! + ! Inputs + ! + integer (kind=c_int), intent(in) :: ie + type (c_ptr) , intent(in) :: tensorvisc2_ptr + end subroutine init_tensorvisc2_c + ! Copies geopotential from f90 arrays to C++ views subroutine init_geopotential_c (ie, phis_ptr, gradphis_ptr) bind(c) use iso_c_binding, only: c_int, c_ptr diff --git a/components/homme/test_execs/preqx_kokkos_ut/random_init_ut.cpp b/components/homme/test_execs/preqx_kokkos_ut/random_init_ut.cpp index a217de2f768d..abb4c8c68ea7 100644 --- a/components/homme/test_execs/preqx_kokkos_ut/random_init_ut.cpp +++ b/components/homme/test_execs/preqx_kokkos_ut/random_init_ut.cpp @@ -62,7 +62,7 @@ TEST_CASE("d_dinv_check", "Testing Elements::random_init") { std::cout << "seed: " << seed << (catchRngSeed==0 ? " (catch rng seed was 0)\n" : "\n"); ElementsGeometry geometry; - geometry.init(num_elems,false,true,PhysicalConstants::rearth0); + geometry.init(num_elems,true,PhysicalConstants::rearth0); geometry.randomize(seed); HostViewManaged d("host d", num_elems); diff --git a/components/homme/test_execs/preqx_kokkos_ut/remap_preqx_ut.cpp b/components/homme/test_execs/preqx_kokkos_ut/remap_preqx_ut.cpp index 771376e959fa..d2def27cde39 100644 --- a/components/homme/test_execs/preqx_kokkos_ut/remap_preqx_ut.cpp +++ b/components/homme/test_execs/preqx_kokkos_ut/remap_preqx_ut.cpp @@ -26,7 +26,7 @@ TEST_CASE("remap_interface", "vertical remap") { std::cout << "seed: " << seed << (catchRngSeed==0 ? " (catch rng seed was 0)\n" : "\n"); Elements elements; - elements.init(num_elems,seed, /*alloc_gradphis = */ false, + elements.init(num_elems, /*alloc_gradphis = */ false, PhysicalConstants::rearth0); elements.randomize(seed); diff --git a/components/homme/test_execs/share_kokkos_ut/sphere_op_interface.F90 b/components/homme/test_execs/share_kokkos_ut/sphere_op_interface.F90 index 4cae876ed91d..756df3842c31 100644 --- a/components/homme/test_execs/share_kokkos_ut/sphere_op_interface.F90 +++ b/components/homme/test_execs/share_kokkos_ut/sphere_op_interface.F90 @@ -280,7 +280,7 @@ subroutine vlaplace_sphere_wk_cartesian_c_callable(v, dvv, dinv, spheremp, & real(kind=real_kind), intent(in) :: dinv(np, np, 2, 2) real(kind=real_kind), intent(in) :: spheremp(np, np) real(kind=real_kind), intent(in) :: tensorVisc(np, np, 2, 2) - real(kind=real_kind), intent(in) :: vec_sph2cart(np, np, 3, 2) + real(kind=real_kind), intent(in) :: vec_sph2cart(np, np, 3, 3) logical, value, intent(in) :: var_coef real(kind=real_kind), intent(in) :: hvpower, hvscaling real(kind=real_kind), intent(out) :: laplace(np,np,2) @@ -300,7 +300,7 @@ subroutine vlaplace_sphere_wk_cartesian_c_callable(v, dvv, dinv, spheremp, & elem%tensorVisc = tensorVisc elem%vec_sphere2cart = vec_sph2cart - laplace=vlaplace_sphere_wk_cartesian(v,deriv,elem,var_coef) + laplace=vlaplace_sphere_wk_cartesian(v,deriv,elem,var_coef,tensorVisc) end subroutine vlaplace_sphere_wk_cartesian_c_callable @@ -346,7 +346,7 @@ subroutine vlaplace_sphere_wk_contra_c_callable(v, dvv, d, dinv, mp, spheremp, m elem%Dinv = dinv elem%rmetdet = rmetdet - laplace = vlaplace_sphere_wk_contra(v,deriv,elem,.false.,nu_ratio) + laplace = vlaplace_sphere_wk_contra(v,deriv,elem,nu_ratio) end subroutine vlaplace_sphere_wk_contra_c_callable diff --git a/components/homme/test_execs/share_kokkos_ut/sphere_op_ml.cpp b/components/homme/test_execs/share_kokkos_ut/sphere_op_ml.cpp index e289fcf70f68..190a78ca0b8e 100644 --- a/components/homme/test_execs/share_kokkos_ut/sphere_op_ml.cpp +++ b/components/homme/test_execs/share_kokkos_ut/sphere_op_ml.cpp @@ -252,7 +252,7 @@ class compute_sphere_operator_test_ml { ExecViewManaged vector_input_d; ExecViewManaged tensor_d; - ExecViewManaged vec_sph2cart_d; + ExecViewManaged vec_sph2cart_d; ExecViewManaged scalar_output_d; ExecViewManaged @@ -298,10 +298,10 @@ class compute_sphere_operator_test_ml { tensor_host; const int tensor_len = 2 * 2 * NP * NP; // temp code - ExecViewManaged::host_mirror_type + ExecViewManaged::host_mirror_type vec_sph2cart_host; const int vec_sph2cart_len = - 2 * 3 * NP * NP; // temp code + 3 * 3 * NP * NP; // temp code ExecViewManaged::host_mirror_type scalar_output_host; @@ -1111,7 +1111,7 @@ TEST_CASE( Real dvvf[NP][NP]; Real dinvf[2][2][NP][NP]; Real tensorf[2][2][NP][NP]; - Real vec_sph2cartf[2][3][NP][NP]; + Real vec_sph2cartf[3][3][NP][NP]; Real sphf[NP][NP]; for(int _i = 0; _i < NP; _i++) diff --git a/components/homme/test_execs/thetal_kokkos_ut/caar_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/caar_ut.cpp index 7089e1187f3c..3fcdb7fa93b7 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/caar_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/caar_ut.cpp @@ -116,7 +116,7 @@ TEST_CASE("caar", "caar_testing") { const int num_elems = c.get().get_num_local_elements(); auto& elems = c.create(); - elems.init(num_elems,false,true,PhysicalConstants::rearth0); + elems.init(num_elems,true,PhysicalConstants::rearth0); const auto max_pressure = 1000.0 + hvcoord.ps0; // This ensures max_p > ps0 auto& geo = elems.m_geometry; elems.m_geometry.randomize(seed); // Only needed for phis and gradphis diff --git a/components/homme/test_execs/thetal_kokkos_ut/compose_interface.F90 b/components/homme/test_execs/thetal_kokkos_ut/compose_interface.F90 index 4a9e708b8261..a72009da2fbf 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/compose_interface.F90 +++ b/components/homme/test_execs/thetal_kokkos_ut/compose_interface.F90 @@ -85,10 +85,12 @@ subroutine init_geometry_f90() bind(c) elem_rspheremp, elem_metdet, elem_state_phis real (real_kind), target, dimension(np,np,2) :: elem_gradphis real (real_kind), target, dimension(np,np,2,2) :: elem_D, elem_Dinv, elem_metinv, elem_tensorvisc - real (real_kind), target, dimension(np,np,3,2) :: elem_vec_sph2cart + real (real_kind), target, dimension(np,np,2,2) :: elem_tensorvisc2 + real (real_kind), target, dimension(np,np,3,3) :: elem_vec_sph2cart type (c_ptr) :: elem_D_ptr, elem_Dinv_ptr, elem_fcor_ptr, elem_spheremp_ptr, & elem_rspheremp_ptr, elem_metdet_ptr, elem_metinv_ptr, elem_tensorvisc_ptr, & elem_vec_sph2cart_ptr, elem_state_phis_ptr, elem_gradphis_ptr + type (c_ptr) :: elem_tensorvisc2_ptr type (cartesian3D_t) :: sphere_cart real (kind=real_kind) :: sphere_cart_vec(3,np,np), sphere_latlon_vec(2,np,np) @@ -106,6 +108,7 @@ subroutine init_geometry_f90() bind(c) elem_vec_sph2cart_ptr = c_loc(elem_vec_sph2cart) elem_state_phis_ptr = c_loc(elem_state_phis) elem_gradphis_ptr = c_loc(elem_gradphis) + elem_tensorvisc2_ptr = c_loc(elem_tensorvisc2) do ie = 1,nelemd elem_D = elem(ie)%D elem_Dinv = elem(ie)%Dinv @@ -117,6 +120,7 @@ subroutine init_geometry_f90() bind(c) elem_state_phis = elem(ie)%state%phis elem_gradphis = elem(ie)%derived%gradphis elem_tensorvisc = elem(ie)%tensorVisc + elem_tensorvisc2 = elem(ie)%tensorVisc_2 elem_vec_sph2cart = elem(ie)%vec_sphere2cart do j = 1,np do i = 1,np @@ -131,7 +135,7 @@ subroutine init_geometry_f90() bind(c) call init_elements_2d_c(ie-1, elem_D_ptr, elem_Dinv_ptr, elem_fcor_ptr, & elem_spheremp_ptr, elem_rspheremp_ptr, elem_metdet_ptr, elem_metinv_ptr, & elem_tensorvisc_ptr, elem_vec_sph2cart_ptr, sphere_cart_vec, & - sphere_latlon_vec) + sphere_latlon_vec, elem_tensorvisc2_ptr) call init_geopotential_c(ie-1, elem_state_phis_ptr, elem_gradphis_ptr) enddo end subroutine init_geometry_f90 diff --git a/components/homme/test_execs/thetal_kokkos_ut/dirk_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/dirk_ut.cpp index 911bdf3243e7..7fb17ce9d897 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/dirk_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/dirk_ut.cpp @@ -631,7 +631,7 @@ static void init_elems (int, int nelemd, Random& r, const HybridVCoord& hvcoord, const int nlev = NUM_PHYSICAL_LEV, np = NP; const auto all = Kokkos::ALL(); - e.init(nelemd, false, true, PhysicalConstants::rearth0); + e.init(nelemd, true, PhysicalConstants::rearth0); const auto max_pressure = 1000 + hvcoord.ps0; auto& geo = e.m_geometry; e.m_geometry.randomize(r.gen_seed()); @@ -852,7 +852,7 @@ TEST_CASE ("dirk_toplevel_testing") { c2f(e); compute_stage_value_dirk_f90(nm1+1, alphadtwt_nm1*dt2, n0+1, alphadtwt_n0*dt2, np1+1, dt2); Elements ef90; - ef90.init(nelemd, false, true, PhysicalConstants::rearth0); + ef90.init(nelemd, true, PhysicalConstants::rearth0); f2c(ef90); const auto phif = cmvdc(ef90.m_state.m_phinh_i); diff --git a/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp index fb301166f429..a5fc0f35c786 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp @@ -66,7 +66,7 @@ TEST_CASE("forcing", "forcing") { hv.random_init(seed); auto& geo = c.create(); - geo.init(num_elems,true, /* alloc_gradphis = */ true, + geo.init(num_elems, /* alloc_gradphis = */ true, PhysicalConstants::rearth0); geo.randomize(seed); diff --git a/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp index 6ad29cc8f7e0..6b222cdf73eb 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp @@ -799,9 +799,15 @@ test_dyn_to_fv_phys (Session& s, const int nf, const bool theta_hydrostatic_mode const ExecView dps("dps", s.nelemd, nf2), dphis("dphis", s.nelemd, nf2); const ExecView dT("dT", s.nelemd, nf2, g::num_lev_aligned), domega("domega", s.nelemd, nf2, g::num_lev_aligned); - const ExecView duv("duv", s.nelemd, nf2, 2, g::num_lev_aligned), + // dstrain's buffer is reused both as GLL-side input (requires >= np2 in the + // 2nd extent) and as FV-side output (requires >= nf2), so it must be sized + // to fit the larger of the two. + const int dstrain_dim1 = std::max(nf2, static_cast(g::np2)); + const ExecView dstrain("dstrain", s.nelemd, dstrain_dim1, 6, g::num_lev_aligned), + duv("duv", s.nelemd, nf2, 2, g::num_lev_aligned), dq("dq", s.nelemd, nf2, s.qsize, g::num_lev_aligned), dq1("dq", s.nelemd, nf2, nq, g::num_lev_aligned); + Kokkos::deep_copy(dstrain, 0.0); const auto& c = Context::singleton(); auto& gfr = c.get(); @@ -812,11 +818,26 @@ test_dyn_to_fv_phys (Session& s, const int nf, const bool theta_hydrostatic_mode dq1_dyn(g::cpack2real(q), q.extent_int(0), q.extent_int(1), q.extent_int(2)*q.extent_int(3), g::num_lev_aligned); + const GllFvRemap::Phys1T dps_u(dps.data(), dps.extent_int(0), dps.extent_int(1)); + const GllFvRemap::Phys1T dphis_u(dphis.data(), dphis.extent_int(0), dphis.extent_int(1)); + const GllFvRemap::Phys2T dT_u(dT.data(), dT.extent_int(0), dT.extent_int(1), dT.extent_int(2)); + const GllFvRemap::Phys2T domega_u(domega.data(), domega.extent_int(0), domega.extent_int(1), + domega.extent_int(2)); + const GllFvRemap::CPhys3T dstrain_gll_u(dstrain.data(), dstrain.extent_int(0), dstrain.extent_int(1), + dstrain.extent_int(2), dstrain.extent_int(3)); + const GllFvRemap::Phys3T dstrain_fv_u(dstrain.data(), dstrain.extent_int(0), dstrain.extent_int(1), + dstrain.extent_int(2), dstrain.extent_int(3)); + const GllFvRemap::Phys3T duv_u(duv.data(), duv.extent_int(0), duv.extent_int(1), duv.extent_int(2), + duv.extent_int(3)); + const GllFvRemap::Phys3T dq_u(dq.data(), dq.extent_int(0), dq.extent_int(1), dq.extent_int(2), + dq.extent_int(3)); + for (int nt = 0; nt < NUM_TIME_LEVELS; ++nt) { gfr_dyn_to_fv_phys_f90(nf, nt+1, fps.data(), fphis.data(), fT.data(), fuv.data(), fomega.data(), fq.data()); - gfr.run_dyn_to_fv_phys(nt, dps, dphis, dT, domega, duv, dq); + gfr.run_dyn_to_fv_phys(nt, dps_u, dphis_u, dT_u, domega_u, &dstrain_gll_u, &dstrain_fv_u, + duv_u, dq_u); gfr.remap_tracer_dyn_to_fv_phys(nt, nq, dq1_dyn, dq1); @@ -898,10 +919,17 @@ test_fv_phys_to_dyn (Session& s, const int nf, const bool theta_hydrostatic_mode const auto& c = Context::singleton(); auto& gfr = c.get(); + const GllFvRemap::CPhys2T dT_u(dT.data(), dT.extent_int(0), dT.extent_int(1), dT.extent_int(2)); + const GllFvRemap::CPhys3T duv_u(duv.data(), duv.extent_int(0), duv.extent_int(1), duv.extent_int(2), + duv.extent_int(3)); + const GllFvRemap::CPhys3T dfq_u(dfq.data(), dfq.extent_int(0), dfq.extent_int(1), dfq.extent_int(2), + dfq.extent_int(3)); + const GllFvRemap::CPhys2T dKm_u(dKm.data(), dKm.extent_int(0), dKm.extent_int(1), dKm.extent_int(2)); + const GllFvRemap::CPhys2T dKh_u(dKh.data(), dKh.extent_int(0), dKh.extent_int(1), dKh.extent_int(2)); const int nt = 1; gfr_fv_phys_to_dyn_f90(nf, nt+1, fT.data(), fuv.data(), ffq.data()); - gfr.run_fv_phys_to_dyn(nt, dT, duv, dfq, dKm, dKh); + gfr.run_fv_phys_to_dyn(nt, dT_u, duv_u, dfq_u, &dKm_u, &dKh_u); gfr.run_fv_phys_to_dyn_dss(); } diff --git a/components/homme/test_execs/thetal_kokkos_ut/hv_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/hv_ut.cpp index 55ea92f6c91c..5210bd22b734 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/hv_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/hv_ut.cpp @@ -200,7 +200,7 @@ TEST_CASE("hvf", "biharmonic") { const int num_elems = c.get().get_num_local_elements(); auto& geo = c.create(); - geo.init(num_elems,false,true,PhysicalConstants::rearth0); + geo.init(num_elems,true,PhysicalConstants::rearth0); geo.randomize(seed); auto& state = c.create(); diff --git a/components/homme/test_execs/thetal_kokkos_ut/remap_theta_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/remap_theta_ut.cpp index b8e885372e13..ba0569906e12 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/remap_theta_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/remap_theta_ut.cpp @@ -106,7 +106,7 @@ TEST_CASE("remap", "remap_testing") { // Create and init elements/tracers auto& elems = c.create(); - elems.init(num_elems,false,true,PhysicalConstants::rearth0); + elems.init(num_elems,true,PhysicalConstants::rearth0); const auto max_pressure = 1000.0 + hvcoord.ps0; // This ensures max_p > ps0 auto& geo = elems.m_geometry; elems.m_geometry.randomize(seed); // Only needed for phis and gradphis diff --git a/components/homme/test_execs/thetal_kokkos_ut/thetal_test_interface.F90 b/components/homme/test_execs/thetal_kokkos_ut/thetal_test_interface.F90 index 7a46a99dca76..b0402982ee9e 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/thetal_test_interface.F90 +++ b/components/homme/test_execs/thetal_kokkos_ut/thetal_test_interface.F90 @@ -210,7 +210,7 @@ subroutine init_geo_views_f90 (d_ptr, dinv_ptr, & scalar2d(:,:,ie) = elem(ie)%rspheremp enddo - call c_f_pointer(sph2c_ptr, tensor2d, [np,np,3,2,nelemd]) + call c_f_pointer(sph2c_ptr, tensor2d, [np,np,3,3,nelemd]) call c_f_pointer(mdet_ptr, scalar2d, [np,np, nelemd]) do ie=1,nelemd tensor2d(:,:,:,:,ie) = elem(ie)%vec_sphere2cart diff --git a/components/mpas-framework/Makefile b/components/mpas-framework/Makefile index 24d2d8c1f130..50a2371fb186 100644 --- a/components/mpas-framework/Makefile +++ b/components/mpas-framework/Makefile @@ -412,6 +412,8 @@ gnu-cray: "USE_SHTNS = $(USE_SHTNS)" \ "CPPFLAGS = $(MODEL_FORMULATION) -D_MPI $(FILE_OFFSET) $(ZOLTAN_DEFINE)" ) +# ifx (Intel oneAPI) fpp mishandles the COMMA macro-argument idiom used in the +# MPAS framework, so preprocess .F with GNU cpp via GEN_F90=true instead. intel-cray: ( $(MAKE) all \ "FC_PARALLEL = ftn" \ @@ -438,6 +440,7 @@ intel-cray: "USE_PAPI = $(USE_PAPI)" \ "OPENMP = $(OPENMP)" \ "USE_SHTNS = $(USE_SHTNS)" \ + "GEN_F90 = true" \ "CPPFLAGS = $(MODEL_FORMULATION) -D_MPI" ) cray-cray: diff --git a/components/mpas-ocean/cime_config/config_pes.xml b/components/mpas-ocean/cime_config/config_pes.xml index 71266d799257..132a976f80a3 100644 --- a/components/mpas-ocean/cime_config/config_pes.xml +++ b/components/mpas-ocean/cime_config/config_pes.xml @@ -354,6 +354,56 @@ + + + + none + + 640 + 640 + 640 + 640 + 640 + 1 + 640 + 640 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 640 + + + + + + --res TL319_IcoswISC30E3r5_wQU225Icos30E3r5 --compset GMPAS-JRA1p5-WW3 + + -1 + -1 + -4 + -4 + -4 + -4 + + + + diff --git a/components/ww3/cime_config/buildlib_cmake b/components/ww3/cime_config/buildlib_cmake index cd180f432c83..343d7be0f4b0 100755 --- a/components/ww3/cime_config/buildlib_cmake +++ b/components/ww3/cime_config/buildlib_cmake @@ -4,7 +4,6 @@ build ww3 library """ import sys, os, shutil -from distutils.spawn import find_executable _CIMEROOT = os.environ.get("CIMEROOT") if _CIMEROOT is None: diff --git a/driver-mct/shr/seq_flds_mod.F90 b/driver-mct/shr/seq_flds_mod.F90 index f9960c1761ec..6b9bce369f0e 100644 --- a/driver-mct/shr/seq_flds_mod.F90 +++ b/driver-mct/shr/seq_flds_mod.F90 @@ -2887,6 +2887,32 @@ subroutine seq_flds_set(nmlfile, ID, infodata) end if end do + ! Scalar per-gridcell lnd->iac fields + if (add_iac_to_cplstate) call seq_flds_add(l2x_states, 'Sl_forc_hdm') + call seq_flds_add(x2z_states, 'Sl_forc_hdm') + longname = 'Population density' + stdname = 'lnd_population_density' + units = 'ind/km2' + attname = 'Sl_forc_hdm' + call metadata_set(attname, longname, stdname, units) + + ! heating and cooling degree days for EHC + if(add_iac_to_cplstate)call seq_flds_add(l2x_states,'Sl_hdd') + call seq_flds_add(x2z_states,'Sl_hdd') + longname = 'heating degree days' + stdname = 'lnd_hdd' + units = 'K-days' + attname = 'Sl_hdd' + call metadata_set(attname, longname, stdname, units) + + if(add_iac_to_cplstate)call seq_flds_add(l2x_states,'Sl_cdd') + call seq_flds_add(x2z_states,'Sl_cdd') + longname = 'cooling degree days' + stdname = 'lnd_cdd' + units = 'K-days' + attname = 'Sl_cdd' + call metadata_set(attname, longname, stdname, units) + ! iac->atm flux. ! Monthly values of surface, low alt, high alt co2 fluxes, so we ! loop over 36 total fields. diff --git a/externals/ekat b/externals/ekat index d713b7db20f6..3a1485aecfed 160000 --- a/externals/ekat +++ b/externals/ekat @@ -1 +1 @@ -Subproject commit d713b7db20f66072cdb73b849632e08dc2cb0d8d +Subproject commit 3a1485aecfed7a036e4fa30ce845498fd37975e2 diff --git a/externals/mam4xx b/externals/mam4xx index 23877829346a..8fbf7750bafa 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 23877829346a49d62cfa035c22038d8d78d85eb8 +Subproject commit 8fbf7750bafa8b14533d72397aeb027184d0ec37 diff --git a/share/build/buildlib.ekat b/share/build/buildlib.ekat index 8a651c036289..92a86e17fb2b 100755 --- a/share/build/buildlib.ekat +++ b/share/build/buildlib.ekat @@ -135,7 +135,7 @@ def buildlib(bldroot, installpath, case): if case.get_value("EXTRA_DEBUG"): ekat_cmake_options += " -DKokkos_ENABLE_DEBUG_BOUNDS_CHECK=On" - gen_makefile_cmd = f"cmake {kokkos_options} {cc} {cxx} -DCMAKE_CXX_STANDARD=17 {ekat_cmake_options} -DCMAKE_INSTALL_PREFIX={installpath} {ekat_dir}" + gen_makefile_cmd = f"cmake {kokkos_options} {cc} {cxx} {ekat_cmake_options} -DCMAKE_INSTALL_PREFIX={installpath} {ekat_dir}" # When later we use find_package to get kokkos in CMake, the folder # install_sharedpath/kokkos (which is bldroot here) gets picked over diff --git a/share/timing/private.h b/share/timing/private.h index 387e1e083fc6..af2d274884ae 100644 --- a/share/timing/private.h +++ b/share/timing/private.h @@ -44,8 +44,10 @@ #define MAX_AUX 9 #ifndef __cplusplus -// Protect against inclusion of stdbool.h in, e.g., a compiler wrapper. -# ifndef true +// Protect against inclusion of stdbool.h in, e.g., a compiler wrapper, and +// against C23 (and later) where true/false/bool are keywords, not macros +// defined via stdbool.h. +# if !defined(true) && !(defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L) typedef enum {false = 0, true = 1} bool; /* mimic C++ */ # endif #endif diff --git a/tools/generate_domain_files/generate_domain_files_E3SM.py b/tools/generate_domain_files/generate_domain_files_E3SM.py index 86dbcc9cff6d..2660c11161ba 100644 --- a/tools/generate_domain_files/generate_domain_files_E3SM.py +++ b/tools/generate_domain_files/generate_domain_files_E3SM.py @@ -237,7 +237,7 @@ def main(): # Get ocn mask on ocn grid omask = get_mask(ds,opts,suffix='_a') - ofrac = xr.zeros_like(ds['area_a']) + ofrac = xr.where( omask!=0, xr.ones_like(ds['area_a']), xr.zeros_like(ds['area_a']) ) ds_out = xr.Dataset()