Merge pull request #3351 from GEOS-ESM/feature/bmauer/fixes-#3338

Feature/bmauer/fixes #3338

CHANGELOG.md

@@ -31,7 +31,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
-- Changed `MAPL_ESMFRegridder` to require the dstMaskValues to be added as grid attribute to use fixed masking, fixes UFS issue
+- Updated ExtData so that if files are missing in a sequence the last value will be perisisted if one has not chosen `exact` option
+- Changed MAPL_ESMFRegridder to require the dstMaskValues to be added as grid attribute to use fixed masking, fixes UFS issue
 - Increased formatting width of time index in ExtData2G diagnostic print
 - Updated GitHub checkout action to use blobless clones
 - Update CI to use Baselibs 7.29.0 by default

Tests/ExtDataRoot_GridComp.F90

@@ -677,12 +677,8 @@ subroutine CompareState(State1,State2,tol,rc)
 
       integer :: status
       integer                             :: i
-      real, pointer                       :: ptr3_1(:,:,:)
-      real, pointer                       :: ptr3_2(:,:,:)
-      real, pointer                       :: ptr2_1(:,:)
-      real, pointer                       :: ptr2_2(:,:)
-      real, pointer                       :: ptr1_1(:)
-      real, pointer                       :: ptr1_2(:)
+      real, pointer                       :: ptr1(:)
+      real, pointer                       :: ptr2(:)
       integer :: itemcount,rank1,rank2
       character(len=ESMF_MAXSTR), allocatable :: NameList(:)
       logical, allocatable :: foundDiff(:)
@@ -706,25 +702,12 @@ subroutine CompareState(State1,State2,tol,rc)
                exit
             end if
             _ASSERT(rank1==rank2,'needs informative message')
+            call assign_fptr(field1, ptr1, _RC)
+            call assign_fptr(field2, ptr2, _RC)
+            _ASSERT(size(ptr1)==size(ptr2),'needs informative message')
             foundDiff(i)=.false.
-            if (rank1==1) then
-               call MAPL_GetPointer(state1,ptr1_1,trim(nameList(i)),_RC)
-               call MAPL_GetPointer(state2,ptr1_2,trim(nameList(i)),_RC)
-               if (any((ptr1_1-ptr1_2) > tol)) then
-                   foundDiff(i) = .true.
-               end if
-            else if (rank1==2) then
-               call MAPL_GetPointer(state1,ptr2_1,trim(nameList(i)),_RC)
-               call MAPL_GetPointer(state2,ptr2_2,trim(nameList(i)),_RC)
-               if (any((ptr2_1-ptr2_2) > tol)) then
-                   foundDiff(i) = .true.
-               end if
-            else if (rank1==3) then
-               call MAPL_GetPointer(state1,ptr3_1,trim(nameList(i)),_RC)
-               call MAPL_GetPointer(state2,ptr3_2,trim(nameList(i)),_RC)
-               if (any((ptr3_1-ptr3_2) > tol)) then
-                   foundDiff(i) = .true.
-               end if
+            if (any(abs(ptr1-ptr2) > tol)) then
+                foundDiff(i) = .true.
             end if
             if (foundDiff(i)) then
                _FAIL('found difference when compare state')
@@ -741,10 +724,8 @@ subroutine ForceAllocation(state,rc)
 
          integer :: status
 
-         real, pointer :: ptr3d(:,:,:)
-         real, pointer :: ptr2d(:,:)
          integer       :: ii
-         integer :: itemcount,dims
+         integer :: itemcount
          character(len=ESMF_MAXSTR), allocatable :: NameList(:)
          type (ESMF_StateItem_Flag), allocatable :: itemTypeList(:)
          type(ESMF_Field) :: Field

Tests/ExtData_Testing_Framework/test_cases/case37/AGCM1.rc

@@ -0,0 +1,22 @@
+NX: 1
+NY: 1
+
+Root.GRID_TYPE: LatLon
+Root.GRIDNAME: DC90x45-PC
+Root.LM: 3
+Root.IM_WORLD: 90
+Root.JM_WORLD: 45
+Root.POLE: 'PC'
+Root.DATELINE: 'DC'
+
+RUN_MODE: GenerateExports
+
+EXPORT_STATE::
+VAR1 , time , days , xy , c
+::
+
+FILL_DEF::
+VAR1 time
+::
+
+REF_TIME: 20070927 000000

Tests/ExtData_Testing_Framework/test_cases/case37/AGCM2.rc

@@ -0,0 +1,22 @@
+NX: 1
+NY: 1
+
+Root.GRID_TYPE: LatLon
+Root.GRIDNAME: DC90x45-PC
+Root.LM: 3
+Root.IM_WORLD: 90
+Root.JM_WORLD: 90
+Root.POLE: 'PC'
+Root.DATELINE: 'DC'
+
+RUN_MODE: GenerateExports
+
+EXPORT_STATE::
+VAR1 , time , days , xy , c
+::
+
+FILL_DEF::
+VAR1 time
+::
+
+REF_TIME: 20070927 000000

Tests/ExtData_Testing_Framework/test_cases/case37/AGCM3.rc

@@ -0,0 +1,26 @@
+NX: 1
+NY: 1
+
+Root.GRID_TYPE: LatLon
+Root.GRIDNAME: DC900x45-PC
+Root.LM: 3
+Root.IM_WORLD: 90
+Root.JM_WORLD: 45
+Root.POLE: 'PC'
+Root.DATELINE: 'DC'
+
+RUN_MODE: CompareImports
+
+IMPORT_STATE::
+VAR1 , time , days , xy , c
+::
+
+EXPORT_STATE::
+VAR1 , time , days , xy , c
+::
+
+FILL_DEF::
+VAR1 3.0
+::
+
+REF_TIME: 20080101 000000

Tests/ExtData_Testing_Framework/test_cases/case37/CAP.rc

@@ -0,0 +1,5 @@
+CASES::
+CAP1.rc
+CAP2.rc
+CAP3.rc
+::

Tests/ExtData_Testing_Framework/test_cases/case37/CAP1.rc

@@ -0,0 +1,8 @@
+ROOT_NAME: Root
+ROOT_CF: AGCM1.rc
+HIST_CF: HISTORY1.rc
+
+BEG_DATE:     20070926 230000
+
+JOB_SGMT:     00000004 000000
+HEARTBEAT_DT:   3600

Tests/ExtData_Testing_Framework/test_cases/case37/CAP2.rc

@@ -0,0 +1,8 @@
+ROOT_NAME: Root
+ROOT_CF: AGCM2.rc
+HIST_CF: HISTORY2.rc
+
+BEG_DATE:     20071001 210000
+
+JOB_SGMT:     00000003 000000
+HEARTBEAT_DT:   3600

Tests/ExtData_Testing_Framework/test_cases/case37/CAP3.rc

@@ -0,0 +1,9 @@
+ROOT_NAME: Root
+ROOT_CF: AGCM3.rc
+HIST_CF: HISTORY3.rc
+
+BEG_DATE:     20070930 000000
+
+JOB_SGMT:     00000001 120000
+HEARTBEAT_DT:   3600
+

Tests/ExtData_Testing_Framework/test_cases/case37/ExtData.rc

@@ -0,0 +1,12 @@
+#CASE_SENSITIVE_VARIABLE_NAMES: .false.
+Ext_AllowExtrap: .false.
+Prefetch: .true.
+DEBUG_LEVEL: 20
+
+PrimaryExports%%
+VAR2D NA  2008 N 0   none     none VAR2D case1.nc4
+%%
+
+
+DerivedExports%%
+%%

Tests/ExtData_Testing_Framework/test_cases/case37/HISTORY1.rc

@@ -0,0 +1,13 @@
+GRID_LABELS:
+::
+
+COLLECTIONS: case1
+::
+
+  case1.template:  '%y4%m2%d2.nc4',
+  case1.format:    'CFIO',
+  case1.frequency:  240000,
+  #case1.duration: 000000,
+  case1.ref_time: 000000,
+  case1.fields: 'VAR1', 'Root',
+                          ::

Tests/ExtData_Testing_Framework/test_cases/case37/HISTORY2.rc

@@ -0,0 +1,13 @@
+GRID_LABELS:
+::
+
+COLLECTIONS: case1
+::
+
+  case1.template:  '%y4%m2%d2.nc4',
+  case1.format:    'CFIO',
+  case1.frequency:  240000,
+  #case1.duration: 000000,
+  case1.ref_time: 000000,
+  case1.fields: 'VAR1', 'Root',
+                          ::

Tests/ExtData_Testing_Framework/test_cases/case37/HISTORY3.rc

@@ -0,0 +1,13 @@
+GRID_LABELS:
+::
+
+COLLECTIONS: 
+::
+
+  case2.template:  'nc4',
+  case2.format:    'CFIO',
+  case2.frequency:  240000,
+  case2.duration: 000000,
+  case2.ref_time: 000000,
+  case2.fields: 'VAR2D', 'Root',
+                          ::

Tests/ExtData_Testing_Framework/test_cases/case37/README

@@ -0,0 +1 @@
+test multiple datasets where an export uses both with with no extrapolation outside and crosses transition date

Tests/ExtData_Testing_Framework/test_cases/case37/extdata.yaml

@@ -0,0 +1,12 @@
+Collections:
+   fstream1: {template: "case1.%y4%m2%d2.nc4", valid_range: "2007-09-22/2007-09-30"}
+   fstream2: {template: "case2.%y4%m2%d2.nc4", valid_range: "2007-10-01/2007-10-08"}
+
+Samplings:
+  s1:
+    time_interpolation: False
+
+Exports:
+   VAR1:
+     - {starting: "2007-09-22", variable: VAR1, collection: fstream1, sample: s1, fail_on_missing_file: false}
+     - {starting: "2007-10-01", variable: VAR1, collection: fstream2, sample: s1, fail_on_missing_file: false}

Tests/ExtData_Testing_Framework/test_cases/extdata_2g_cases.txt

@@ -34,3 +34,4 @@ case33
 case34
 case35
 case36
+case37

gridcomps/ExtData2G/ExtDataBracket.F90

@@ -188,10 +188,8 @@ subroutine interpolate_to_time(this,field,time,rc)
 
       right_node_set = this%right_node%check_if_initialized(_RC)
       left_node_set = this%left_node%check_if_initialized(_RC)
-      call this%right_node%get(file=right_file)
-      call this%left_node%get(file=left_file)
-      right_node_set = right_file /= file_not_found
-      left_node_set = left_file /= file_not_found
+      call ESMF_TimePrint(this%left_node%time,options='string',preString='left bracket time: ')
+      call ESMF_TimePrint(this%right_node%time,options='string',preString='right bracket time: ')
 
       alpha = 0.0
       if ( (.not.this%disable_interpolation) .and. (.not.this%intermittent_disable) .and. right_node_set .and. left_node_set) then
@@ -216,7 +214,9 @@ subroutine interpolate_to_time(this,field,time,rc)
          elsewhere
             var1d = mapl_undef
          endwhere
-      else
+      end if
+
+      if (this%exact .and. (.not.(time == this%left_node%time))) then
          var1d = mapl_undef
       end if
 
@@ -243,6 +243,7 @@ subroutine swap_node_fields(this,rc)
 
       left_created  = ESMF_FieldIsCreated(this%left_node%field,_RC)
       right_created = ESMF_FieldIsCreated(this%right_node%field,_RC)
+      left_created  = ESMF_FieldIsCreated(this%left_node%field,_RC)
       if (left_created .and. right_created) then     
          call assign_fptr(this%left_node%field,left_ptr,_RC) 
          call assign_fptr(this%right_node%field,right_ptr,_RC) 

gridcomps/ExtData2G/ExtDataSimpleFileHandler.F90

@@ -105,8 +105,10 @@ subroutine get_file_bracket(this, input_time, source_time, bracket, fail_on_miss
 
             call bracket%set_node('L',file=current_file,time_index=time_index,time=time,was_set=.true.,_RC)
             if (in_range .and. (bracket%left_node == bracket%right_node)) then
-               call bracket%swap_node_fields(_RC)
-               bracket%new_file_left = .false.
+               if (.not. (current_file == file_not_found)) then
+                  call bracket%swap_node_fields(_RC)
+                  bracket%new_file_left = .false.
+               end if
             else
                if (time_index == time_not_found ) then
                   call this%get_file(current_file,target_time,-1,allow_missing_file,_RC)