Skip to content

Put OpenACC parallelized versions of biogeophys modules onto new branch #8669

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
peterdschwartz wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

Put OpenACC parallelized versions of biogeophys modules onto new branch #8669

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 11 additions & 7 deletions components/elm/src/biogeophys/ActiveLayerMod.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ module ActiveLayerMod
use ColumnDataType , only : col_es, col_ws
use LandunitType , only : lun_pp
use landunit_varcon , only : ilowcenpoly, iflatcenpoly, ihighcenpoly
use timeinfoMod
!
implicit none
save
Expand All @@ -23,12 +24,11 @@ module ActiveLayerMod
! !PUBLIC MEMBER FUNCTIONS:
public:: alt_calc
!-----------------------------------------------------------------------

contains

!-----------------------------------------------------------------------
subroutine alt_calc(num_soilc, filter_soilc, &
temperature_vars, canopystate_vars)
subroutine alt_calc(num_soilc, filter_soilc, canopystate_vars)
!
! !DESCRIPTION:
! define active layer thickness similarly to frost_table, except set as deepest thawed layer and define on nlevgrnd
Expand All @@ -45,6 +45,7 @@ subroutine alt_calc(num_soilc, filter_soilc, &
! initialized to valid values, so I think this is okay.
!
! !USES:
!$acc routine seq
use shr_const_mod , only : SHR_CONST_TKFRZ
use elm_varpar , only : nlevgrnd
use elm_time_manager , only : get_curr_date, get_step_size
Expand All @@ -54,7 +55,6 @@ subroutine alt_calc(num_soilc, filter_soilc, &
! !ARGUMENTS:
integer , intent(in) :: num_soilc ! number of soil columns in filter
integer , intent(in) :: filter_soilc(:) ! filter for soil columns
type(temperature_type) , intent(in) :: temperature_vars
type(canopystate_type) , intent(inout) :: canopystate_vars
!
! !LOCAL VARIABLES:
Expand Down Expand Up @@ -94,9 +94,13 @@ subroutine alt_calc(num_soilc, filter_soilc, &

! on a set annual timestep, update annual maxima
! make this 1 January for NH columns, 1 July for SH columns
call get_curr_date(year, mon, day, sec)
dtime = get_step_size()
if ( (mon .eq. 1) .and. (day .eq. 1) .and. ( sec / dtime .eq. 1) ) then
! call get_curr_date(year, mon, day, sec)
dtime = int(dtime_mod)
year = year_curr
mon = mon_curr
day = day_curr
sec = secs_curr
if ( (mon .eq. 1) .and. (day .eq. 1) .and. ( sec / int(dtime) .eq. 1) ) then
do fc = 1,num_soilc
c = filter_soilc(fc)
g = col_pp%gridcell(c)
Expand Down
162 changes: 96 additions & 66 deletions components/elm/src/biogeophys/AerosolMod.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ module AerosolMod
use ColumnType , only : col_pp
use ColumnDataType , only : col_ws, col_wf

use timeinfoMod
use timeinfoMod, only : dtime_mod
!
! !PUBLIC TYPES:
implicit none
Expand All @@ -38,7 +38,6 @@ subroutine AerosolMasses(bounds, num_on, filter_on, num_off, filter_off, aerosol
! (based on new snow level state, after SnowFilter is rebuilt.
! NEEDS TO BE AFTER SnowFiler is rebuilt in Hydrology2, otherwise there
! can be zero snow layers but an active column in filter)
!$acc routine seq
! !ARGUMENTS:
type(bounds_type) , intent(in ) :: bounds
integer , intent(in) :: num_on ! number of column filter-ON points
Expand All @@ -48,10 +47,10 @@ subroutine AerosolMasses(bounds, num_on, filter_on, num_off, filter_off, aerosol
type(aerosol_type) , intent(inout) :: aerosol_vars
!
! !LOCAL VARIABLES:
real(r8) :: dtime ! land model time step (sec)
integer :: g,l,c,j,fc ! indices
integer :: g,l,c,j,fc,snl_idx ! indices
real(r8) :: snowmass ! liquid+ice snow mass in a layer [kg/m2]
real(r8) :: snowcap_scl_fct ! temporary factor used to correct for snow capping
real(r8) :: sum_bc,sum_oc,sum_dst
!-----------------------------------------------------------------------

associate( &
Expand Down Expand Up @@ -92,30 +91,22 @@ subroutine AerosolMasses(bounds, num_on, filter_on, num_off, filter_off, aerosol
mss_cnc_dst4 => aerosol_vars%mss_cnc_dst4_col & ! Output: [real(r8) (:,:) ] mass concentration of dust species 4 (col,lyr) [kg/kg]
)

dtime = dtime_mod

do fc = 1, num_on
c = filter_on(fc)

! Zero column-integrated aerosol mass before summation
mss_bc_col(c) = 0._r8
mss_oc_col(c) = 0._r8
mss_dst_col(c) = 0._r8

!$acc enter data create(sum_bc,sum_oc,sum_dst)
if (.not. use_firn_percolation_and_compaction) then
!$acc parallel loop independent gang vector collapse(2) default(present)
do j = -nlevsno+1, 0
do fc = 1, num_on
c = filter_on(fc)
! layer mass of snow:
snowmass = h2osoi_ice(c,j) + h2osoi_liq(c,j)

! layer mass of snow:
snowmass = h2osoi_ice(c,j) + h2osoi_liq(c,j)

if (.not. use_firn_percolation_and_compaction) then
! Correct the top layer aerosol mass to account for snow capping.
! This approach conserves the aerosol mass concentration
! (but not the aerosol amss) when snow-capping is invoked

if (j == snl(c)+1) then
if (do_capsnow(c)) then

snowcap_scl_fct = snowmass / (snowmass + (qflx_snwcp_ice(c)*dtime))
snowcap_scl_fct = snowmass / (snowmass + (qflx_snwcp_ice(c)*dtime_mod))

mss_bcpho(c,j) = mss_bcpho(c,j)*snowcap_scl_fct
mss_bcphi(c,j) = mss_bcphi(c,j)*snowcap_scl_fct
Expand All @@ -128,22 +119,28 @@ subroutine AerosolMasses(bounds, num_on, filter_on, num_off, filter_off, aerosol
mss_dst4(c,j) = mss_dst4(c,j)*snowcap_scl_fct
endif
endif
endif
end do
end do
endif

!$acc parallel loop independent gang vector collapse(2) default(present)
do j = -nlevsno+1, 0
do fc = 1, num_on
c = filter_on(fc)

if (j >= snl(c)+1) then
! layer mass of snow:
snowmass = h2osoi_ice(c,j) + h2osoi_liq(c,j)

if (j >= snl(c)+1) then
mss_bctot(c,j) = mss_bcpho(c,j) + mss_bcphi(c,j)
mss_bc_col(c) = mss_bc_col(c) + mss_bctot(c,j)
mss_cnc_bcphi(c,j) = mss_bcphi(c,j) / snowmass
mss_cnc_bcpho(c,j) = mss_bcpho(c,j) / snowmass

mss_octot(c,j) = mss_ocpho(c,j) + mss_ocphi(c,j)
mss_oc_col(c) = mss_oc_col(c) + mss_octot(c,j)
mss_cnc_ocphi(c,j) = mss_ocphi(c,j) / snowmass
mss_cnc_ocpho(c,j) = mss_ocpho(c,j) / snowmass

mss_dsttot(c,j) = mss_dst1(c,j) + mss_dst2(c,j) + mss_dst3(c,j) + mss_dst4(c,j)
mss_dst_col(c) = mss_dst_col(c) + mss_dsttot(c,j)
mss_cnc_dst1(c,j) = mss_dst1(c,j) / snowmass
mss_cnc_dst2(c,j) = mss_dst2(c,j) / snowmass
mss_cnc_dst3(c,j) = mss_dst3(c,j) / snowmass
Expand Down Expand Up @@ -176,50 +173,81 @@ subroutine AerosolMasses(bounds, num_on, filter_on, num_off, filter_off, aerosol
mss_cnc_dst4(c,j) = 0._r8
endif
enddo
enddo

!$acc parallel loop independent gang vector default(present)
do fc = 1, num_on
c = filter_on(fc)
snl_idx = snl(c)+1
! top-layer diagnostics
h2osno_top(c) = h2osoi_ice(c,snl(c)+1) + h2osoi_liq(c,snl(c)+1) !TODO MV - is this correct to be placed here???
mss_bc_top(c) = mss_bctot(c,snl(c)+1)
mss_oc_top(c) = mss_octot(c,snl(c)+1)
mss_dst_top(c) = mss_dsttot(c,snl(c)+1)
enddo
h2osno_top(c) = h2osoi_ice(c,snl_idx) + h2osoi_liq(c,snl_idx) !TODO MV - is this correct to be placed here???
mss_bc_top(c) = mss_bctot(c,snl_idx)
mss_oc_top(c) = mss_octot(c,snl_idx)
mss_dst_top(c) = mss_dsttot(c,snl_idx)
end do

! Zero mass variables in columns without snow
!$acc parallel loop gang worker independent default(present) private(sum_bc,sum_oc,sum_dst)
do fc = 1, num_on
c = filter_on(fc)
sum_bc = 0._r8
sum_oc = 0._r8
sum_dst = 0._r8
!$acc loop vector reduction(+:sum_bc,sum_oc,sum_dst)
do j = -nlevsno+1,0
if (j >= snl(c)+1) then
sum_bc = sum_bc + mss_bctot(c,j)
sum_oc = sum_oc + mss_octot(c,j)
sum_dst = sum_dst + mss_dsttot(c,j)
end if
end do
mss_bc_col(c) = sum_bc
mss_oc_col(c) = sum_oc
mss_dst_col(c) = sum_dst
end do

! Zero mass variables in columns without snow
!$acc parallel loop independent gang vector default(present)
do fc = 1, num_off
c = filter_off(fc)

mss_bc_top(c) = 0._r8
mss_bc_col(c) = 0._r8
mss_bcpho(c,:) = 0._r8
mss_bcphi(c,:) = 0._r8
mss_bctot(c,:) = 0._r8
mss_cnc_bcphi(c,:) = 0._r8
mss_cnc_bcpho(c,:) = 0._r8

mss_oc_top(c) = 0._r8
mss_oc_col(c) = 0._r8
mss_ocpho(c,:) = 0._r8
mss_ocphi(c,:) = 0._r8
mss_octot(c,:) = 0._r8
mss_cnc_ocphi(c,:) = 0._r8
mss_cnc_ocpho(c,:) = 0._r8

mss_dst_top(c) = 0._r8
mss_dst_col(c) = 0._r8
mss_dst1(c,:) = 0._r8
mss_dst2(c,:) = 0._r8
mss_dst3(c,:) = 0._r8
mss_dst4(c,:) = 0._r8
mss_dsttot(c,:) = 0._r8

mss_cnc_dst1(c,:) = 0._r8
mss_cnc_dst2(c,:) = 0._r8
mss_cnc_dst3(c,:) = 0._r8
mss_cnc_dst4(c,:) = 0._r8

enddo

!$acc parallel loop independent gang vector collapse(2) default(present)
do j=-nlevsno+1,0
do fc = 1, num_off
c = filter_off(fc)
mss_dst1(c,j) = 0._r8
mss_dst2(c,j) = 0._r8
mss_dst3(c,j) = 0._r8
mss_dst4(c,j) = 0._r8
mss_dsttot(c,j) = 0._r8

mss_cnc_dst1(c,j) = 0._r8
mss_cnc_dst2(c,j) = 0._r8
mss_cnc_dst3(c,j) = 0._r8
mss_cnc_dst4(c,j) = 0._r8
mss_ocpho(c,j) = 0._r8
mss_ocphi(c,j) = 0._r8
mss_octot(c,j) = 0._r8
mss_cnc_ocphi(c,j) = 0._r8
mss_cnc_ocpho(c,j) = 0._r8

mss_bcpho(c,j) = 0._r8
mss_bcphi(c,j) = 0._r8
mss_bctot(c,j) = 0._r8
mss_cnc_bcphi(c,j) = 0._r8
mss_cnc_bcpho(c,j) = 0._r8
end do
end do
!$acc exit data delete(sum_bc,sum_oc,sum_dst)

end associate

end subroutine AerosolMasses
Expand All @@ -230,7 +258,6 @@ subroutine AerosolFluxes(bounds, num_snowc, filter_snowc, &
!
! !DESCRIPTION:
! Compute aerosol fluxes through snowpack and aerosol deposition fluxes into top layere
!$acc routine seq
! !ARGUMENTS:
type(bounds_type) , intent(in) :: bounds
integer , intent(in) :: num_snowc ! number of snow points in column filter
Expand All @@ -239,7 +266,6 @@ subroutine AerosolFluxes(bounds, num_snowc, filter_snowc, &
type(aerosol_type) , intent(inout) :: aerosol_vars
!
! !LOCAL VARIABLES:
real(r8) :: dtime ! land model time step (sec)
integer :: c,g,j,fc
!-----------------------------------------------------------------------

Expand Down Expand Up @@ -289,6 +315,8 @@ subroutine AerosolFluxes(bounds, num_snowc, filter_snowc, &
! (cloud-borne) aerosol, and "pho" flavors are interstitial
! aerosol. "wet" and "dry" fluxes of BC and OC specified here are
! purely diagnostic

!$acc parallel loop independent gang vector default(present)
do c = bounds%begc,bounds%endc
g = col_pp%gridcell(c)

Expand Down Expand Up @@ -324,6 +352,7 @@ subroutine AerosolFluxes(bounds, num_snowc, filter_snowc, &
! species are distinguished in model, other fluxes (e.g., dry and
! wet BC/OC) are purely diagnostic.

!$acc parallel loop independent gang vector default(present)
do c = bounds%begc,bounds%endc
g = col_pp%gridcell(c)

Expand Down Expand Up @@ -359,19 +388,20 @@ subroutine AerosolFluxes(bounds, num_snowc, filter_snowc, &
! is in the top layer after deposition, and is not immediately
! washed out before radiative calculations are done

dtime = dtime_mod

!$acc parallel loop independent gang vector default(present) &
!$acc present(mss_bcphi(:,:),mss_bcpho(:,:),mss_ocphi(:,:),mss_ocpho(:,:), &
!$acc mss_dst1(:,:),mss_dst2(:,:),mss_dst3(:,:),mss_dst4(:,:))
do fc = 1, num_snowc
c = filter_snowc(fc)
mss_bcphi(c,snl(c)+1) = mss_bcphi(c,snl(c)+1) + (flx_bc_dep_phi(c)*dtime)
mss_bcpho(c,snl(c)+1) = mss_bcpho(c,snl(c)+1) + (flx_bc_dep_pho(c)*dtime)
mss_ocphi(c,snl(c)+1) = mss_ocphi(c,snl(c)+1) + (flx_oc_dep_phi(c)*dtime)
mss_ocpho(c,snl(c)+1) = mss_ocpho(c,snl(c)+1) + (flx_oc_dep_pho(c)*dtime)

mss_dst1(c,snl(c)+1) = mss_dst1(c,snl(c)+1) + (flx_dst_dep_dry1(c) + flx_dst_dep_wet1(c))*dtime
mss_dst2(c,snl(c)+1) = mss_dst2(c,snl(c)+1) + (flx_dst_dep_dry2(c) + flx_dst_dep_wet2(c))*dtime
mss_dst3(c,snl(c)+1) = mss_dst3(c,snl(c)+1) + (flx_dst_dep_dry3(c) + flx_dst_dep_wet3(c))*dtime
mss_dst4(c,snl(c)+1) = mss_dst4(c,snl(c)+1) + (flx_dst_dep_dry4(c) + flx_dst_dep_wet4(c))*dtime
mss_bcphi(c,snl(c)+1) = mss_bcphi(c,snl(c)+1) + (flx_bc_dep_phi(c)*dtime_mod)
mss_bcpho(c,snl(c)+1) = mss_bcpho(c,snl(c)+1) + (flx_bc_dep_pho(c)*dtime_mod)
mss_ocphi(c,snl(c)+1) = mss_ocphi(c,snl(c)+1) + (flx_oc_dep_phi(c)*dtime_mod)
mss_ocpho(c,snl(c)+1) = mss_ocpho(c,snl(c)+1) + (flx_oc_dep_pho(c)*dtime_mod)

mss_dst1(c,snl(c)+1) = mss_dst1(c,snl(c)+1) + (flx_dst_dep_dry1(c) + flx_dst_dep_wet1(c))*dtime_mod
mss_dst2(c,snl(c)+1) = mss_dst2(c,snl(c)+1) + (flx_dst_dep_dry2(c) + flx_dst_dep_wet2(c))*dtime_mod
mss_dst3(c,snl(c)+1) = mss_dst3(c,snl(c)+1) + (flx_dst_dep_dry3(c) + flx_dst_dep_wet3(c))*dtime_mod
mss_dst4(c,snl(c)+1) = mss_dst4(c,snl(c)+1) + (flx_dst_dep_dry4(c) + flx_dst_dep_wet4(c))*dtime_mod
end do

end associate
Expand Down
Loading