Sync from NCAR/main (many combined updates)

CMakeLists.txt

@@ -207,9 +207,13 @@ target_include_directories(ccpp_physics PUBLIC
 
 target_link_libraries(ccpp_physics PRIVATE MPI::MPI_Fortran)
 target_link_libraries(ccpp_physics PUBLIC w3emc::w3emc_d
-                                          sp::sp_d
                                           NetCDF::NetCDF_Fortran
 				             )
+if(ip_FOUND)
+  target_link_libraries(ccpp_physics PUBLIC ip::ip_d)
+else()
+  target_link_libraries(ccpp_physics PUBLIC sp::sp_d)
+endif()
 #add FMS for FV3 only
 if(FV3 OR MPAS)
   target_link_libraries(ccpp_physics PUBLIC fms)

CODEOWNERS

@@ -30,7 +30,7 @@ physics/GWD/drag_suite.*                                                     @md
 physics/GWD/gwdc.*                                                           @Songyou184                               @grantfirl @rhaesung @Qingfu-Liu @dustinswales
 physics/GWD/gwdps.*                                                          @Songyou184                               @grantfirl @rhaesung @Qingfu-Liu @dustinswales
 physics/GWD/rayleigh_damp.*                                                  @yangfanglin                              @grantfirl @rhaesung @Qingfu-Liu @dustinswales
-physics/GWD/ugwp_driver_v0.F                                                 @mdtoyNOAA                                @grantfirl @rhaesung @Qingfu-Liu @dustinswales
+physics/GWD/ugwp_driver_v0.F90                                               @mdtoyNOAA                                @grantfirl @rhaesung @Qingfu-Liu @dustinswales
 physics/GWD/ugwpv1_gsldrag.*                                                 @mdtoyNOAA @BoYang-NOAA                   @grantfirl @rhaesung @Qingfu-Liu @dustinswales
 physics/GWD/ugwpv1_gsldrag_post.*                                            @mdtoyNOAA @BoYang-NOAA                   @grantfirl @rhaesung @Qingfu-Liu @dustinswales
 physics/GWD/unified_ugwp*                                                    @mdtoyNOAA                                @grantfirl @rhaesung @Qingfu-Liu @dustinswales
@@ -149,7 +149,7 @@ physics/Interstitials/UFS_SCM_NEPTUNE/GFS_PBL_generic_pre.*
 physics/Interstitials/UFS_SCM_NEPTUNE/GFS_phys_time_vary.fv3.*                                                         @grantfirl @rhaesung @Qingfu-Liu @dustinswales
 physics/Interstitials/UFS_SCM_NEPTUNE/GFS_phys_time_vary.scm.*                                                         @grantfirl @rhaesung @Qingfu-Liu @dustinswales
 physics/Interstitials/UFS_SCM_NEPTUNE/GFS_physics_post.*                                                               @grantfirl @rhaesung @Qingfu-Liu @dustinswales
-physics/Interstitials/UFS_SCM_NEPTUNE/GFS_radiation_surface.*                                                          @grantfirl @rhaesung @Qingfu-Liu @dustinswales 
+physics/Interstitials/UFS_SCM_NEPTUNE/GFS_radiation_surface.*                                                          @grantfirl @rhaesung @Qingfu-Liu @dustinswales
 physics/Interstitials/UFS_SCM_NEPTUNE/GFS_rad_time_vary.fv3.*                                                          @grantfirl @rhaesung @Qingfu-Liu @dustinswales
 physics/Interstitials/UFS_SCM_NEPTUNE/GFS_rad_time_vary.scm.*                                                          @grantfirl @rhaesung @Qingfu-Liu @dustinswales
 physics/Interstitials/UFS_SCM_NEPTUNE/GFS_rrtmgp_cloud_mp.*                  @dustinswales @Qingfu-Liu                 @grantfirl @rhaesung @Qingfu-Liu @dustinswales

physics/GWD/cires_tauamf_data.F90

@@ -4,6 +4,7 @@
 module cires_tauamf_data
 
   use machine, only: kind_phys
+  use mpi_f08
 !...........................................................................................
 ! tabulated GW-sources: GRACILE/Ern et al., 2018 and/or Resolved GWs from C384-Annual run
 !...........................................................................................
@@ -20,27 +21,28 @@ module cires_tauamf_data
 contains
 
 !>
-   subroutine read_tau_amf(me, master, errmsg, errflg)
+   subroutine read_tau_amf(mpicomm, mpirank, mpiroot, errmsg, errflg)
 
-    use  netcdf
-    integer, intent(in) ::  me, master   
+    use netcdf
+    use mpiutil, only: ccpp_bcast
+    type(MPI_Comm), intent(in) :: mpicomm
+    integer, intent(in) :: mpirank, mpiroot
     integer :: ncid,  iernc, vid, dimid, status         
     integer :: k
 
     character(len=*), intent(out) :: errmsg
     integer,          intent(out) :: errflg    
 !
-      
-      iernc=NF90_OPEN(trim(ugwp_taufile), nf90_nowrite, ncid)
+    read_and_broadcast_1: if (mpirank==mpiroot) then
+       iernc=NF90_OPEN(trim(ugwp_taufile), nf90_nowrite, ncid)
 
        if(iernc.ne.0) then         
           write(errmsg,'(*(a))') "read_tau_amf: cannot open file_limb_tab data-file ",  &
                                     trim(ugwp_taufile)
           print *, 'cannot open ugwp-v1 tau-file=',trim(ugwp_taufile)
           errflg = 1
           return
-        else
-
+       endif
 
        status = nf90_inq_dimid(ncid, "lat", DimID)
 !      if (status /= nf90_noerr) call handle_err(status)
@@ -50,28 +52,36 @@ subroutine read_tau_amf(me, master, errmsg, errflg)
        status = nf90_inq_dimid(ncid, "days", DimID)
        status = nf90_inquire_dimension(ncid, DimID,  len =ntau_d2t )
 
-           if (me == master)  print *, ntau_d1y, ntau_d2t, ' dimd of tau_ngw ugwp-v1 '
-           if (ntau_d2t .le. 0 .or. ntau_d1y .le. 0) then 
-               print *, 'ugwp-v1 tau-file=',    trim(ugwp_taufile)
-               print *, '  ugwp-v1: ', 'ntau_d2t=',ntau_d2t, 'ntau_d2t=',ntau_d1y
-               stop
-           endif
-
-        if (.not.allocated(ugwp_taulat))  allocate (ugwp_taulat(ntau_d1y ))
-        if (.not.allocated(days_limb))    allocate (days_limb(ntau_d2t))
-        if (.not.allocated(tau_limb))     allocate (tau_limb(ntau_d1y, ntau_d2t ))
-
-        iernc=nf90_inq_varid( ncid, 'DAYS', vid )
-        iernc= nf90_get_var( ncid, vid, days_limb)
-        iernc=nf90_inq_varid( ncid, 'LATS', vid )
-        iernc= nf90_get_var( ncid, vid, ugwp_taulat)
-        iernc=nf90_inq_varid( ncid, 'ABSMF', vid )
-        iernc= nf90_get_var( ncid, vid, tau_limb)
-
-        iernc=nf90_close(ncid)
-
-        endif
-
+       print *, ntau_d1y, ntau_d2t, ' dimd of tau_ngw ugwp-v1 '
+       if (ntau_d2t .le. 0 .or. ntau_d1y .le. 0) then 
+          print *, 'ugwp-v1 tau-file=',    trim(ugwp_taufile)	   
+          print *, '  ugwp-v1: ', 'ntau_d2t=',ntau_d2t, 'ntau_d2t=',ntau_d1y
+          errflg = 1
+          return
+       endif
+    endif read_and_broadcast_1
+
+    call ccpp_bcast(ntau_d1y, mpiroot, mpicomm, errflg)
+    call ccpp_bcast(ntau_d2t, mpiroot, mpicomm, errflg)
+
+    if (.not.allocated(ugwp_taulat))  allocate (ugwp_taulat(ntau_d1y ))
+    if (.not.allocated(days_limb))    allocate (days_limb(ntau_d2t))
+    if (.not.allocated(tau_limb))     allocate (tau_limb(ntau_d1y, ntau_d2t ))   	   
+
+    read_and_broadcast_2: if (mpirank==mpiroot) then
+      iernc=nf90_inq_varid( ncid, 'DAYS', vid )
+      iernc= nf90_get_var( ncid, vid, days_limb)
+      iernc=nf90_inq_varid( ncid, 'LATS', vid )
+      iernc= nf90_get_var( ncid, vid, ugwp_taulat)
+      iernc=nf90_inq_varid( ncid, 'ABSMF', vid )
+      iernc= nf90_get_var( ncid, vid, tau_limb)
+      iernc=nf90_close(ncid)
+    endif read_and_broadcast_2
+
+    call ccpp_bcast(days_limb,   mpiroot, mpicomm, errflg)
+    call ccpp_bcast(tau_limb,    mpiroot, mpicomm, errflg)
+    call ccpp_bcast(ugwp_taulat, mpiroot, mpicomm, errflg)
+
   end  subroutine read_tau_amf  
 
 !>

physics/GWD/cires_ugwp.F90

@@ -39,7 +39,7 @@ module cires_ugwp
 !! \htmlinclude cires_ugwp_init.html
 !!
     subroutine cires_ugwp_init (me, master, nlunit, input_nml_file, logunit, &
-                fn_nml2, lonr, latr, levs, ak, bk, dtp, cdmbgwd, cgwf,       &
+                fn_nml2, lonr, levs, ak, bk, dtp, cdmbgwd, cgwf,       &
                 pa_rf_in, tau_rf_in, con_p0, gwd_opt,do_ugwp, errmsg, errflg)
 
 !----  initialization of cires_ugwp
@@ -52,7 +52,6 @@ subroutine cires_ugwp_init (me, master, nlunit, input_nml_file, logunit, &
     integer,              intent (in) :: logunit
     integer,              intent (in) :: lonr
     integer,              intent (in) :: levs
-    integer,              intent (in) :: latr
     real(kind=kind_phys), intent (in) :: ak(:), bk(:)
     real(kind=kind_phys), intent (in) :: dtp
     real(kind=kind_phys), intent (in) :: cdmbgwd(:), cgwf(:) ! "scaling" controls for "old" GFS-GW schemes
@@ -64,11 +63,6 @@ subroutine cires_ugwp_init (me, master, nlunit, input_nml_file, logunit, &
     character(len=*), intent (in) :: fn_nml2
     !character(len=*), parameter   :: fn_nml='input.nml'
 
-    integer :: ios
-    logical :: exists
-    real    :: dxsg
-    integer :: k
-
     character(len=*), intent(out) :: errmsg
     integer,          intent(out) :: errflg
 
@@ -77,19 +71,21 @@ subroutine cires_ugwp_init (me, master, nlunit, input_nml_file, logunit, &
     errflg = 0
 
     if (is_initialized) return
-    
+
     ! Consistency checks
     if (gwd_opt/=1) then
       write(errmsg,'(*(a))') "Logic error: namelist choice of gravity wave &
       & drag is different from cires_ugwp scheme"
       errflg = 1
       return
-    end if  
+    end if
 
     if (do_ugwp .or. cdmbgwd(3) > 0.0) then
       call cires_ugwpv0_mod_init (me, master, nlunit, input_nml_file, logunit, &
-                                fn_nml2, lonr, latr, levs, ak, bk, con_p0, dtp, &
-                                cdmbgwd(1:2), cgwf, pa_rf_in, tau_rf_in)
+                                fn_nml2, lonr, levs, ak, bk, con_p0, dtp, &
+                                cdmbgwd(1:2), cgwf, pa_rf_in, tau_rf_in, &
+                                errmsg, errflg)
+      if (errflg/=0) return
     else
       write(errmsg,'(*(a))') "Logic error: cires_ugwp_init called but do_ugwp is false and cdmbgwd(3) <= 0"
       errflg = 1
@@ -106,7 +102,6 @@ subroutine cires_ugwp_init (me, master, nlunit, input_nml_file, logunit, &
 
     end subroutine cires_ugwp_init
 
-
 ! -----------------------------------------------------------------------
 ! finalize of cires_ugwp   (_finalize)
 ! -----------------------------------------------------------------------
@@ -136,7 +131,6 @@ subroutine cires_ugwp_finalize(errmsg, errflg)
 
     end subroutine cires_ugwp_finalize
 
-
 ! -----------------------------------------------------------------------
 !    originally from ugwp_driver_v0.f
 !    driver of cires_ugwp   (_driver)
@@ -147,45 +141,45 @@ end subroutine cires_ugwp_finalize
 ! -----------------------------------------------------------------------
 !>@brief These subroutines and modules execute the CIRES UGWP Version 0.
 !> \section gen_cires_ugwp CIRES UGWP V0 Scheme General Algorithm
-!! The physics of Non-Orographic Gravity Waves (NGWs) in the UGWP framework 
-!!(Yudin et al. 2018 \cite yudin_et_al_2018) is represented by four GW-solvers, introduced in 
-!!Lindzen (1981) \cite lindzen_1981, Hines (1997) \cite hines_1997, Alexander 
-!!and Dunkerton (1999) \cite alexander_and_dunkerton_1999, and Scinocca (2003) \cite scinocca_2003. 
-!!A major modification of these GW solvers was introduced with the addition of the 
-!!background dissipation of temperature and winds to the saturation criteria for wave breaking. 
-!!This feature is important in the mesosphere and thermosphere for WAM applications and it 
-!!considers appropriate scale-dependent dissipation of waves near the model top lid providing 
-!!the momentum and energy conservation in the vertical column physics (Shaw and 
-!!Shepherd (2009) \cite shaw_and_shepherd_2009). In the UGWP-v0 scheme, a modification of the 
-!!Scinocca (2003) \cite scinocca_2003 algorithm for NGWs with non-hydrostatic and rotational 
-!!effects for GW propagations and background dissipation is contained in the subroutine 
-!!fv3_ugwp_solv2_v0. Future development plans for the UGWP scheme include additional GW-solvers 
-!!to be implemented along with physics-based triggering of waves and stochastic approaches 
-!!for selection of GW modes characterized by horizontal phase velocities, azimuthal 
+!! The physics of Non-Orographic Gravity Waves (NGWs) in the UGWP framework
+!!(Yudin et al. 2018 \cite yudin_et_al_2018) is represented by four GW-solvers, introduced in
+!!Lindzen (1981) \cite lindzen_1981, Hines (1997) \cite hines_1997, Alexander
+!!and Dunkerton (1999) \cite alexander_and_dunkerton_1999, and Scinocca (2003) \cite scinocca_2003.
+!!A major modification of these GW solvers was introduced with the addition of the
+!!background dissipation of temperature and winds to the saturation criteria for wave breaking.
+!!This feature is important in the mesosphere and thermosphere for WAM applications and it
+!!considers appropriate scale-dependent dissipation of waves near the model top lid providing
+!!the momentum and energy conservation in the vertical column physics (Shaw and
+!!Shepherd (2009) \cite shaw_and_shepherd_2009). In the UGWP-v0 scheme, a modification of the
+!!Scinocca (2003) \cite scinocca_2003 algorithm for NGWs with non-hydrostatic and rotational
+!!effects for GW propagations and background dissipation is contained in the subroutine
+!!fv3_ugwp_solv2_v0. Future development plans for the UGWP scheme include additional GW-solvers
+!!to be implemented along with physics-based triggering of waves and stochastic approaches
+!!for selection of GW modes characterized by horizontal phase velocities, azimuthal
 !!directions and magnitude of the vertical momentum flux (VMF).
 !!
-!! In UGWP-v0, the specification for the VMF function is adopted from the 
-!! GEOS-5 global atmosphere model of GMAO NASA/GSFC, as described in 
-!! Molod et al. (2015) \cite molod_et_al_2015 and employed in the MERRRA-2 
-!! reanalysis (Gelaro et al., 2017 \cite gelaro_et_al_2017). The Fortran 
-!! subroutine slat_geos5_tamp_v0() describes the latitudinal shape of 
-!! VMF-function as displayed in Figure 3 of Molod et al. (2015) 
-!! \cite molod_et_al_2015. It shows that the enhanced values of 
-!! VMF in the equatorial region gives opportunity to simulate the 
-!! QBO-like oscillations in the equatorial zonal winds and lead to more 
-!! realistic simulations of the equatorial dynamics in GEOS-5 operational 
-!! and MERRA-2 reanalysis products. For the first vertically extended 
-!! version of FV3GFS in the stratosphere and mesosphere, this simplified 
-!! function of VMF allows us to tune the model climate and to evaluate 
-!! multi-year simulations of FV3GFS with the MERRA-2 and ERA-5 reanalysis 
-!! products, along with temperature, ozone, and water vapor observations 
-!! of current satellite missions. After delivery of the UGWP-code, the 
-!! EMC group developed and tested approach to modulate the zonal mean 
-!! NGW forcing by 3D-distributions of the total precipitation as a proxy 
-!! for the excitation of NGWs by convection and the vertically-integrated  
-!! (surface - tropopause) Turbulent Kinetic Energy (TKE). The verification 
-!! scores with updated NGW forcing, as reported elsewhere by EMC researchers, 
-!! display noticeable improvements in the forecast scores produced by 
+!! In UGWP-v0, the specification for the VMF function is adopted from the
+!! GEOS-5 global atmosphere model of GMAO NASA/GSFC, as described in
+!! Molod et al. (2015) \cite molod_et_al_2015 and employed in the MERRRA-2
+!! reanalysis (Gelaro et al., 2017 \cite gelaro_et_al_2017). The Fortran
+!! subroutine slat_geos5_tamp_v0() describes the latitudinal shape of
+!! VMF-function as displayed in Figure 3 of Molod et al. (2015)
+!! \cite molod_et_al_2015. It shows that the enhanced values of
+!! VMF in the equatorial region gives opportunity to simulate the
+!! QBO-like oscillations in the equatorial zonal winds and lead to more
+!! realistic simulations of the equatorial dynamics in GEOS-5 operational
+!! and MERRA-2 reanalysis products. For the first vertically extended
+!! version of FV3GFS in the stratosphere and mesosphere, this simplified
+!! function of VMF allows us to tune the model climate and to evaluate
+!! multi-year simulations of FV3GFS with the MERRA-2 and ERA-5 reanalysis
+!! products, along with temperature, ozone, and water vapor observations
+!! of current satellite missions. After delivery of the UGWP-code, the
+!! EMC group developed and tested approach to modulate the zonal mean
+!! NGW forcing by 3D-distributions of the total precipitation as a proxy
+!! for the excitation of NGWs by convection and the vertically-integrated
+!! (surface - tropopause) Turbulent Kinetic Energy (TKE). The verification
+!! scores with updated NGW forcing, as reported elsewhere by EMC researchers,
+!! display noticeable improvements in the forecast scores produced by
 !! FV3GFS configuration extended into the mesosphere.
 !!
 !> \section arg_table_cires_ugwp_run Argument Table
@@ -300,8 +294,9 @@ subroutine cires_ugwp_run(do_ugwp, me,  master, im,  levs, ntrac, dtp, kdt, lonr
            ugrs, vgrs, tgrs, qgrs(:,:,1), kpbl, prsi,del,prsl, prslk, phii, phil, &
            dtp, kdt, sgh30, hprime, oc, oa4, clx, theta, sigma, gamma, elvmax,    &
            dusfcg, dvsfcg, xlat_d, sinlat, coslat, area, cdmbgwd(1:2),            &
-           me, master, rdxzb, con_g, con_omega, zmtb, zogw, tau_mtb, tau_ogw,     &
-           tau_tofd, dudt_mtb, dudt_ogw, dudt_tms)
+           me, master, rdxzb, zmtb, zogw, tau_mtb, tau_ogw,                       &
+           tau_tofd, dudt_mtb, dudt_ogw, dudt_tms,                                &
+           errmsg, errflg)
 
      else                                    ! calling old GFS gravity wave drag as is
 
@@ -335,7 +330,6 @@ subroutine cires_ugwp_run(do_ugwp, me,  master, im,  levs, ntrac, dtp, kdt, lonr
 
      endif ! do_ugwp
 
-
     if(ldiag3d .and. lssav .and. .not. flag_for_gwd_generic_tend) then
       idtend = dtidx(index_of_x_wind,index_of_process_orographic_gwd)
       if(idtend>=1) then
@@ -350,7 +344,6 @@ subroutine cires_ugwp_run(do_ugwp, me,  master, im,  levs, ntrac, dtp, kdt, lonr
          dtend(:,:,idtend) = dtend(:,:,idtend) + Pdtdt*dtp
       endif
     endif
-
 
     if (cdmbgwd(3) > 0.0) then
 

physics/GWD/cires_ugwp.meta

@@ -3,7 +3,7 @@
   type = scheme
 # DH* 20200804 - this is a result of the nasty hack to call gwdps from within ugwp-v0!
   dependencies=cires_ugwp_triggers.F90,cires_ugwp_initialize.F90
-  dependencies=cires_orowam2017.f,cires_ugwp_module.F90,gwdps.f,../hooks/machine.F,ugwp_driver_v0.F
+  dependencies=cires_orowam2017.f,cires_ugwp_module.F90,gwdps.f,../hooks/machine.F,ugwp_driver_v0.F90
 
 ########################################################################
 [ccpp-arg-table]
@@ -60,13 +60,6 @@
   dimensions = ()
   type = integer
   intent = in
-[latr]
-  standard_name = number_of_latitude_points
-  long_name = number of global points in y-dir (j) along the meridian
-  units = count
-  dimensions = ()
-  type = integer
-  intent = in
 [levs]
   standard_name = vertical_layer_dimension
   long_name = number of vertical levels
@@ -822,7 +815,7 @@
   type = real
   kind = kind_phys
   intent = in
-[dqdt_tke] 
+[dqdt_tke]
   standard_name = process_split_cumulative_tendency_of_turbulent_kinetic_energy
   long_name = turbulent kinetic energy tendency due to model physics
   units = J s-1