Merge branch 'develop' into feature/bmauer/fixes-#3338

.circleci/config.yml

@@ -16,7 +16,7 @@ parameters:
 
 # Anchors to prevent forgetting to update a version
 os_version: &os_version ubuntu24
-baselibs_version: &baselibs_version v7.27.0
+baselibs_version: &baselibs_version v7.29.0
 bcs_version: &bcs_version v11.6.0
 tag_build_arg_name: &tag_build_arg_name maplversion
 

.github/workflows/workflow.yml

@@ -38,7 +38,7 @@ jobs:
     name: Build and Test MAPL GNU
     runs-on: ubuntu-latest
     container:
-      image: gmao/ubuntu24-geos-env-mkl:v7.27.0-openmpi_5.0.5-gcc_14.2.0
+      image: gmao/ubuntu24-geos-env-mkl:v7.29.0-openmpi_5.0.5-gcc_14.2.0
       # Per https://github.com/actions/virtual-environments/issues/1445#issuecomment-713861495
       # It seems like we might not need secrets on GitHub Actions which is good for forked
       # pull requests
@@ -89,7 +89,7 @@ jobs:
     name: Build and Test MAPL Intel
     runs-on: ubuntu-latest
     container:
-      image: gmao/ubuntu24-geos-env:v7.27.0-intelmpi_2021.13-ifort_2021.13
+      image: gmao/ubuntu24-geos-env:v7.29.0-intelmpi_2021.13-ifort_2021.13
       # Per https://github.com/actions/virtual-environments/issues/1445#issuecomment-713861495
       # It seems like we might not need secrets on GitHub Actions which is good for forked
       # pull requests

CHANGELOG.md

@@ -9,28 +9,47 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
-- Added optional start_date and start_time to control the output window for each History collection. No output will be written before then. If not specified, these default to the beginning of the experiment.
-- Added utility to prepare inputs for ExtDatDriver.x so that ExtData can simulate a real GEOS run
+### Changed
+
+### Fixed
+
+### Removed
+
+### Deprecated
+
+## [2.52.0] - 2025-01-17
+
+### Added
+
+- Added subroutine to read nc4 tile file
+- Added optional `start_date` and `start_time` to control the output window for each History collection. No output will be written before then. If not specified, these default to the beginning of the experiment.
+- Added utility to prepare inputs for `ExtDataDriver.x` so that ExtData can simulate a real GEOS run
 - Added loggers when writing or reading weight files
 - Added new option to AGCM.rc `overwrite_checkpoint` to allow checkpoint files to be overwritten. By default still will not overwrite checkpoints
+- The trajectory sampler netCDF output variable `location_index_in_iodafile` can be turned off, after we add two control variables: `use_NWP_1_file` and `restore_2_obs_vector` for users.  When set to true, the two options will select only one obs file at each Epoch interval, and will rotate the output field index back to the location vector inthe obs file before generating netCDF output.
+- Support `splitfield: 1` in HISTORY.rc for trajectory sampler
 
 ### Changed
 
 - Updated ExtData so that if files are missing in a sequence the last value will be perisisted if one has not chosen `exact` option
 - Changed MAPL_ESMFRegridder to require the dstMaskValues to be added as grid attribute to use fixed masking, fixes UFS issue
 - Increased formatting width of time index in ExtData2G diagnostic print
 - Updated GitHub checkout action to use blobless clones
+- Update CI to use Baselibs 7.29.0 by default
+  - This provides ESMF 8.8.0
+- Update `components.yaml`
+  - `ESMA_env` v4.34.0
+    - Update to MPT 2.30 at NAS
+    - Update to Baselibs 7.29.0 (ESMF 8.8.0)
+  - `ESMA_cmake` v3.56.0
+    - Use `LOCATION` Python `FIND_STRATEGY`
 
 ### Fixed
 
 - Free MPI communicators after reading and/or writing of restarts
-- Fixed the behavior of MAPL_MaxMin in presence of NaN
+- Fixed the behavior of `MAPL_MaxMin` in presence of NaN
 - Fixed bug with return codes and macros in udunits2f
 
-### Removed
-
-### Deprecated
-
 ## [2.51.2] - 2024-12-19
 
 ### Changed

CMakeLists.txt

@@ -8,7 +8,7 @@ endif ()
 
 project (
   MAPL
-  VERSION 2.51.2
+  VERSION 2.52.0
   LANGUAGES Fortran CXX C)  # Note - CXX is required for ESMF
 
 # Set the possible values of build type for cmake-gui

base/MAPL_LocStreamMod.F90

@@ -69,8 +69,8 @@ module MAPL_LocStreamMod
 
 !EOP
 
-integer, parameter :: NumGlobalVars=4
-integer, parameter :: NumLocalVars =4
+!integer, parameter :: NumGlobalVars=4
+!integer, parameter :: NumLocalVars =4
 
 
 type MAPL_GeoLocation
@@ -343,18 +343,19 @@ subroutine MAPL_LocStreamCreateFromFile(LocStream, LAYOUT, FILENAME, NAME, MASK,
     integer                           :: UNIT
     integer                           :: N, I, K, L, NT
     type(MAPL_LocStreamType), pointer :: STREAM
-    real,    pointer                  :: AVR(:,:), AVR_transpose(:,:)
+    real,    pointer                  :: AVR(:,:)
     logical, pointer                  :: MSK(:)
     real                              :: X, Y, X0, Y0, XE, DX, DY
     integer                           :: II, JJ
     logical                           :: DoCoeffs
     character(len=MAPL_TileNameLength):: gname
-
+    character(len=MAPL_TileNameLength):: GridNames(2)
+    integer                           :: IMs(2), JMs(2)
     integer                           :: irec
     integer(kind=1)                   :: byte(4)
-    integer                           :: I1, IN, J1, JN
-    integer                           :: iostat
-    logical                           :: isascii
+    integer                           :: I1, IN, J1, JN, N_Grids, N_PfafCat
+    integer                           :: iostat, filetype
+    logical                           :: isascii, isnc4, isbinary
     logical                           :: read_always
     logical, pointer                  :: ISMINE(:)
     type(MAPL_Tiling       ), pointer :: TILING
@@ -404,107 +405,75 @@ subroutine MAPL_LocStreamCreateFromFile(LocStream, LAYOUT, FILENAME, NAME, MASK,
 
 ! Use some heuristics to determine filetype (choices are BINARY and ASCII)
 !------------------------------------------------------------------------
-    ! just get the unit
-    UNIT = GETFILE(FILENAME, DO_OPEN=0, ALL_PES=.true., RC=STATUS)
-    _VERIFY(STATUS)
+    call MAPL_NCIOGetFileType(FILENAME, filetype, _RC)
 
-    INQUIRE(IOLENGTH=IREC) BYTE
-    open (UNIT=UNIT, FILE=FILENAME, FORM='unformatted', ACCESS='DIRECT', RECL=IREC, IOSTAT=status)
-    _VERIFY(STATUS)
-    read (UNIT, REC=1, ERR=100) BYTE
-    call FREE_FILE(UNIT)
-
-    isascii = .true.
-    do i=1,size(byte)
-       if (BYTE(I) < 7) then
-          isascii = .false.
-          exit
-       end if
-    end do
+    isnc4   = (filetype == MAPL_FILETYPE_NC4)
+    isascii = (filetype == MAPL_FILETYPE_TXT)
+    isbinary= (filetype == MAPL_FILETYPE_BIN)
 
-    if (isascii) then
+    if ( .not. isbinary) then
 ! Open file and read header info
 !-------------------------------
 
-       UNIT = GETFILE(FILENAME, form='FORMATTED', RC=status)
-       _VERIFY(STATUS)
-
-! Total number of tiles in exchange grid
-!---------------------------------------
+       if (isnc4) then
+          if (MAPL_AM_I_root()) then
+            call MAPL_ReadTilingNC4(FILENAME, GridName=GridNames, IM=IMs, JM=JMs, N_Grids=N_Grids, N_PfafCat=N_PfafCat, AVR=AVR, _RC)
+            NT = size(AVR,1)
+          endif
+          call MAPL_CommsBcast(layout, NT,        1,    MAPL_Root, status)
+          call MAPL_CommsBcast(layout, N_Grids,   1,    MAPL_Root, status)
+          call MAPL_CommsBcast(layout, IMs,       2,    MAPL_Root, status)
+          call MAPL_CommsBcast(layout, JMs,       2,    MAPL_Root, status)
 
-       if (use_pfaf_) then
-          call READ_PARALLEL(layout, hdr, UNIT=UNIT, rc=status)
-          _VERIFY(STATUS)
-          nt=hdr(1)
-          stream%pfafstetter_catchments=hdr(2)
-       else
-          call READ_PARALLEL(layout, nt, UNIT=UNIT, rc=status)
-          _VERIFY(STATUS)
-       end if
+          if (use_pfaf_) then
+            call MAPL_CommsBcast(layout, N_PfafCat, 1,    MAPL_Root, status)
+          endif
 
-! Number of grids that can be attached
-!-------------------------------------
+          do N = 1, N_Grids
+            call MAPL_CommsBcast(layout, GridNames(N), MAPL_TileNameLength, MAPL_Root, status)
+          enddo
 
-       call READ_PARALLEL(layout, STREAM%N_GRIDS, unit=UNIT, rc=status)
-       _VERIFY(STATUS)
+          if (.not. MAPL_AM_I_root()) then
+            allocate(AVR(NT, NumGlobalVars+NumLocalVars*N_GRIDS))
+          endif         
+          call MAPL_CommsBcast(layout, AVR, NT*(NumGlobalVars+NumLocalVars*N_GRIDS), MAPL_Root, status)
+       endif
 
-! The exchange grid is used to tile each attached grid
-!-----------------------------------------------------
+       if (isascii) then
+          call MAPL_ReadTilingASCII(layout, FILENAME, GridNames, NT, IMs, JMs, N_Grids, N_PfafCat, AVR, _RC)
+       endif
 
+       STREAM%N_GRIDS = N_Grids
        allocate(STREAM%TILING(STREAM%N_GRIDS), STAT=STATUS)
        _VERIFY(STATUS)
 
-! The names and sizes of the grids to be tiled
-!---------------------------------------------
-
-       do N=1,STREAM%N_GRIDS
-          call READ_PARALLEL(layout, STREAM%TILING(N)%NAME, unit=UNIT, rc=status)
-          _VERIFY(STATUS)
+       do N = 1, N_Grids
+          STREAM%TILING(N)%NAME = GridNames(N)
           if (NewGridNames_) then
-             call GenOldGridName_(STREAM%TILING(N)%NAME)
+              call GenOldGridName_(STREAM%TILING(N)%NAME)
           end if
-          call READ_PARALLEL(layout, STREAM%TILING(N)%IM, unit=UNIT, rc=status)
-          _VERIFY(STATUS)
-          call READ_PARALLEL(layout, STREAM%TILING(N)%JM, unit=UNIT, rc=status)
-          _VERIFY(STATUS)
+          STREAM%TILING(N)%IM   = IMs(N)
+          STREAM%TILING(N)%JM   = JMs(N)
        enddo
        if (use_pfaf_) then
+          stream%pfafstetter_catchments = N_PfafCat
           STREAM%TILING(2)%IM = stream%pfafstetter_catchments
           STREAM%TILING(2)%JM = 1
           STREAM%TILING(2)%name = "CATCHMENT_GRID"
        end if
 
-
-! Read location stream file into AVR
-!---------------------------------------
-       if(index(STREAM%TILING(1)%NAME,'EASE') /=0 ) then
-          allocate(AVR(NT,9), STAT=STATUS) ! 9 columns for EASE grid
-          _VERIFY(STATUS)
-          allocate(AVR_transpose(9,NT))
-       else
-          allocate(AVR(NT,NumGlobalVars+NumLocalVars*STREAM%N_GRIDS), STAT=STATUS)
-          _VERIFY(STATUS)
-          allocate(AVR_transpose(NumGlobalVars+NumLocalVars*STREAM%N_GRIDS,NT), STAT=STATUS)
-          _VERIFY(STATUS)
-       endif
-
-       call READ_PARALLEL(layout, AVR_transpose(:,:), unit=UNIT, rc=status)
-       AVR= transpose(AVR_transpose)
-       deallocate(AVR_transpose)
-
        ! adjust EASE grid starting index.  Internally, the starting index is 1 instead of 0.
        do N=1,STREAM%N_GRIDS
          if(index(STREAM%TILING(N)%NAME,'EASE') /=0 ) then
              AVR(:,NumGlobalVars+1+NumLocalVars*(N-1)) = AVR(:,NumGlobalVars+1+NumLocalVars*(N-1))+1
              AVR(:,NumGlobalVars+2+NumLocalVars*(N-1)) = AVR(:,NumGlobalVars+2+NumLocalVars*(N-1))+1
          endif
        enddo
-
+     
        !if (use_pfaf_) then
           !AVR(:,NumGlobalVars+2+NumLocalVars) = 1
        !end if
 
-       call FREE_FILE(UNIT)
 
 ! Allocate msk for which tiles to include in the stream being created.
 !--------------------------------------------------------------------
@@ -880,7 +849,7 @@ subroutine MAPL_LocStreamCreateFromFile(LocStream, LAYOUT, FILENAME, NAME, MASK,
 !   and are expensive to reread.
 !------------------------------------------------------------------
 
-    if(.not.isascii) then
+    if( isbinary ) then
 
        if ( MAPL_am_I_root() ) then
          rewind(UNIT)

base/MAPL_ObsUtil.F90

@@ -26,6 +26,8 @@ module MAPL_ObsUtilMod
   type :: obs_unit
      integer :: nobs_epoch
      integer :: ngeoval
+     integer :: count_location_until_matching_file
+     integer :: count_location_in_matching_file
      logical :: export_all_geoval
      type(FileMetadata), allocatable :: metadata
      type(NetCDF4_FileFormatter), allocatable :: file_handle
@@ -38,6 +40,7 @@ module MAPL_ObsUtilMod
      real(kind=REAL64), allocatable :: lats(:)
      real(kind=REAL64), allocatable :: times_R8(:)
      integer,           allocatable :: location_index_ioda(:)
+     integer,           allocatable :: restore_index(:)
      real(kind=REAL32), allocatable :: p2d(:)
      real(kind=REAL32), allocatable :: p3d(:,:)
   end type obs_unit
@@ -50,7 +53,7 @@ module MAPL_ObsUtilMod
      character (len=ESMF_MAXSTR) :: var_name_time=''
      character (len=ESMF_MAXSTR) :: file_name_template=''
      integer :: ngeoval=0
-     integer :: nentry_name=0
+     integer :: nfield_name_mx=12
      character (len=ESMF_MAXSTR), allocatable :: field_name(:,:)
      !character (len=ESMF_MAXSTR), allocatable :: field_name(:)
   end type obs_platform
@@ -768,7 +771,7 @@ function copy_platform_nckeys(a, rc)
     copy_platform_nckeys%var_name_lon = a%var_name_lon
     copy_platform_nckeys%var_name_lat = a%var_name_lat
     copy_platform_nckeys%var_name_time = a%var_name_time
-    copy_platform_nckeys%nentry_name = a%nentry_name
+    copy_platform_nckeys%nfield_name_mx = a%nfield_name_mx
     _RETURN(_SUCCESS)
 
   end function copy_platform_nckeys
@@ -781,7 +784,7 @@ function union_platform(a, b, rc)
     integer, optional, intent(out) :: rc
 
     character (len=ESMF_MAXSTR), allocatable :: field_name_loc(:,:)
-    integer :: nfield, nentry_name
+    integer :: nfield, nfield_name_mx
     integer, allocatable :: tag(:)
     integer :: i, j, k
     integer :: status
@@ -802,9 +805,9 @@ function union_platform(a, b, rc)
     enddo
     union_platform%ngeoval=k
     nfield=k
-    nentry_name=union_platform%nentry_name
+    nfield_name_mx=union_platform%nfield_name_mx
     if ( allocated (union_platform%field_name) ) deallocate(union_platform%field_name)
-    allocate(union_platform%field_name(nentry_name, nfield))
+    allocate(union_platform%field_name(nfield_name_mx, nfield))
     do i=1, a%ngeoval
        union_platform%field_name(:,i) = a%field_name(:,i)
     enddo
@@ -950,6 +953,7 @@ subroutine fglob(search_name, filename, rc)     ! give the last name
     if (lenmax < slen) then
        if (MAPL_AM_I_ROOT())  write(6,*) 'pathlen vs filename_max_char_len: ', slen, lenmax
        _FAIL ('PATHLEN is greater than filename_max_char_len')
+       STOP 'lenmax < slen'
     end if
     if (slen>0) filename(1:slen)=c_filename(1:slen)