Merge pull request #3047 from GEOS-ESM/feature/mathomp4/add-compress-…

…tests

Add extdata tests to test compression, bit-shaving, and quantization

CHANGELOG.md

@@ -9,9 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Added 5 new ExtData tests to test compression, bit-shaving, and quantization
+
 ### Changed
 
-- Add restart benchmark code `restart_simulator.x` in bechmark directory
+- Rename all single-digit ExtData tests to have a leading zero (i.e., `case1` -> `case01`)
+- Add restart benchmark code `restart_simulator.x` in benchmark directory
 - Start implementing changes for vertical regridding in ExtData
 - Made the POSITIVE field attribute defaults to "down" in case it is not found
 - VLOCATION is not querried in MAPL_VerticalMethods.F90 for rank 2 fields

Tests/ExtData_Testing_Framework/CMakeLists.txt

@@ -13,7 +13,7 @@ set(cutoff "7")
 # be skipped for ESSENTIAL testing. Most ExtData tests
 # take 1-2 seconds at most, but some take 20-30 seconds.
 set(SLOW_TESTS
-  "case6"
+  "case06"
   "case14"
   "case15"
   "case16"
@@ -22,6 +22,13 @@ set(SLOW_TESTS
   "case23"
 )
 
+# We have 3 tests that require netcdf Quantize support
+set(QUANTIZE_TESTS
+  "case32"
+  "case33"
+  "case34"
+)
+
 foreach(TEST_CASE ${TEST_CASES_1G})
   if (EXISTS ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc)
     file(READ ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc num_procs)
@@ -52,6 +59,11 @@ file(STRINGS "test_cases/extdata_2g_cases.txt" TEST_CASES_2G)
 
 foreach(TEST_CASE ${TEST_CASES_2G})
 
+  # Skip tests that require Quantize support if we don't have it
+  if (NOT NETCDF_HAS_QUANTIZE AND ${TEST_CASE} IN_LIST QUANTIZE_TESTS)
+    continue()
+  endif()
+
   if (EXISTS ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc)
     file(READ ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc num_procs)
   else()

Tests/ExtData_Testing_Framework/test_cases/case30/AGCM1.rc

@@ -0,0 +1,24 @@
+NX: 1
+NY: 1
+
+Root.GRID_TYPE: LatLon
+Root.GRIDNAME: DC90x45-PC
+Root.LM: 3
+Root.IM_WORLD: 90
+Root.JM_WORLD: 45
+Root.POLE: 'PC'
+Root.DATELINE: 'DC'
+
+RUN_MODE: GenerateExports
+
+EXPORT_STATE::
+VAR2D , time , days , xy , c
+VAR3D , time , days , xyz , c
+::
+
+FILL_DEF::
+VAR2D time
+VAR3D time
+::
+
+REF_TIME: 20040701 000000

Tests/ExtData_Testing_Framework/test_cases/case30/AGCM2.rc

@@ -0,0 +1,29 @@
+NX: 1
+NY: 1
+
+Root.GRID_TYPE: LatLon
+Root.GRIDNAME: DC90x45-PC
+Root.LM: 3
+Root.IM_WORLD: 90
+Root.JM_WORLD: 45
+Root.POLE: 'PC'
+Root.DATELINE: 'DC'
+
+RUN_MODE: CompareImports
+
+IMPORT_STATE::
+VAR2D , time , days , xy , c
+VAR3D , time , days , xyz , c
+::
+
+EXPORT_STATE::
+VAR2D , time , days , xy , c
+VAR3D , time , days , xyz , c
+::
+
+FILL_DEF::
+VAR2D time
+VAR3D time
+::
+
+REF_TIME: 20040701 000000

Tests/ExtData_Testing_Framework/test_cases/case30/CAP.rc

@@ -0,0 +1,4 @@
+CASES::
+CAP1.rc
+CAP2.rc
+::

Tests/ExtData_Testing_Framework/test_cases/case30/CAP1.rc

@@ -0,0 +1,25 @@
+ROOT_NAME: Root
+ROOT_CF: AGCM1.rc
+HIST_CF: HISTORY1.rc
+
+BEG_DATE:     20040101 210000
+
+JOB_SGMT:     00001200 000000
+HEARTBEAT_DT:   3600
+
+#RUN_EXTDATA: .false.
+RUN_TIMES::
+20040115 210000
+20040215 210000
+20040315 210000
+20040415 210000
+20040515 210000
+20040615 210000
+20040715 210000
+20040815 210000
+20040915 210000
+20041015 210000
+20041115 210000
+20041215 210000
+::
+

Tests/ExtData_Testing_Framework/test_cases/case30/CAP2.rc

@@ -0,0 +1,15 @@
+ROOT_NAME: Root
+ROOT_CF: AGCM2.rc
+HIST_CF: HISTORY2.rc
+
+BEG_DATE:     20040101 210000
+
+JOB_SGMT:     00001200 000000
+HEARTBEAT_DT:   3600
+
+#RUN_EXTDATA: .false.
+RUN_TIMES::
+20041125 210000
+20041126 210000
+::
+

Tests/ExtData_Testing_Framework/test_cases/case30/ExtData.rc

@@ -0,0 +1,13 @@
+#CASE_SENSITIVE_VARIABLE_NAMES: .false.
+Ext_AllowExtrap: .false.
+Prefetch: .true.
+#DEBUG_LEVEL: 20
+
+PrimaryExports%%
+VAR2D NA  N N 0   none     none VAR2D case1.%y4.nc4
+VAR3D NA  N N 0   none     none VAR3D case1.%y4.nc4
+%%
+
+
+DerivedExports%%
+%%

Tests/ExtData_Testing_Framework/test_cases/case30/HISTORY1.rc

@@ -0,0 +1,14 @@
+GRID_LABELS:
+::
+
+COLLECTIONS: case1
+::
+
+  case1.template:  '%y4.nc4',
+  case1.format:    'CFIO',
+  case1.frequency:  010000,
+  case1.duration: 000000,
+  case1.deflate: 1,
+  case1.fields: 'VAR2D', 'Root',
+                'VAR3D', 'Root',
+                          ::

Tests/ExtData_Testing_Framework/test_cases/case30/HISTORY2.rc

@@ -0,0 +1,5 @@
+GRID_LABELS:
+::
+
+COLLECTIONS:
+::

Tests/ExtData_Testing_Framework/test_cases/case30/README

@@ -0,0 +1 @@
+Case, 12-month/12 time 2004 file with 2 updates, non-climatology

Tests/ExtData_Testing_Framework/test_cases/case30/extdata.yaml

@@ -0,0 +1,5 @@
+Collections:
+   fstream1: {template: case1.%y4.nc4, valid_range: "2004-01-01/2005-01-01" }
+Exports:
+   VAR2D: {variable: VAR2D, collection: fstream1}
+   VAR3D: {variable: VAR3D, collection: fstream1}

Tests/ExtData_Testing_Framework/test_cases/case31/AGCM1.rc

@@ -0,0 +1,24 @@
+NX: 1
+NY: 1
+
+Root.GRID_TYPE: LatLon
+Root.GRIDNAME: DC90x45-PC
+Root.LM: 3
+Root.IM_WORLD: 90
+Root.JM_WORLD: 45
+Root.POLE: 'PC'
+Root.DATELINE: 'DC'
+
+RUN_MODE: GenerateExports
+
+EXPORT_STATE::
+VAR2D , time , days , xy , c
+VAR3D , time , days , xyz , c
+::
+
+FILL_DEF::
+VAR2D time
+VAR3D time
+::
+
+REF_TIME: 20040701 000000

Tests/ExtData_Testing_Framework/test_cases/case31/AGCM2.rc

@@ -0,0 +1,29 @@
+NX: 1
+NY: 1
+
+Root.GRID_TYPE: LatLon
+Root.GRIDNAME: DC90x45-PC
+Root.LM: 3
+Root.IM_WORLD: 90
+Root.JM_WORLD: 45
+Root.POLE: 'PC'
+Root.DATELINE: 'DC'
+
+RUN_MODE: CompareImports
+
+IMPORT_STATE::
+VAR2D , time , days , xy , c
+VAR3D , time , days , xyz , c
+::
+
+EXPORT_STATE::
+VAR2D , time , days , xy , c
+VAR3D , time , days , xyz , c
+::
+
+FILL_DEF::
+VAR2D time
+VAR3D time
+::
+
+REF_TIME: 20040701 000000

Tests/ExtData_Testing_Framework/test_cases/case31/CAP.rc

@@ -0,0 +1,4 @@
+CASES::
+CAP1.rc
+CAP2.rc
+::

Tests/ExtData_Testing_Framework/test_cases/case31/CAP1.rc

@@ -0,0 +1,25 @@
+ROOT_NAME: Root
+ROOT_CF: AGCM1.rc
+HIST_CF: HISTORY1.rc
+
+BEG_DATE:     20040101 210000
+
+JOB_SGMT:     00001200 000000
+HEARTBEAT_DT:   3600
+
+#RUN_EXTDATA: .false.
+RUN_TIMES::
+20040115 210000
+20040215 210000
+20040315 210000
+20040415 210000
+20040515 210000
+20040615 210000
+20040715 210000
+20040815 210000
+20040915 210000
+20041015 210000
+20041115 210000
+20041215 210000
+::
+

Tests/ExtData_Testing_Framework/test_cases/case31/CAP2.rc

@@ -0,0 +1,15 @@
+ROOT_NAME: Root
+ROOT_CF: AGCM2.rc
+HIST_CF: HISTORY2.rc
+
+BEG_DATE:     20040101 210000
+
+JOB_SGMT:     00001200 000000
+HEARTBEAT_DT:   3600
+
+#RUN_EXTDATA: .false.
+RUN_TIMES::
+20041125 210000
+20041126 210000
+::
+

Tests/ExtData_Testing_Framework/test_cases/case31/ExtData.rc

@@ -0,0 +1,13 @@
+#CASE_SENSITIVE_VARIABLE_NAMES: .false.
+Ext_AllowExtrap: .false.
+Prefetch: .true.
+#DEBUG_LEVEL: 20
+
+PrimaryExports%%
+VAR2D NA  N N 0   none     none VAR2D case1.%y4.nc4
+VAR3D NA  N N 0   none     none VAR3D case1.%y4.nc4
+%%
+
+
+DerivedExports%%
+%%

Tests/ExtData_Testing_Framework/test_cases/case31/HISTORY1.rc

@@ -0,0 +1,15 @@
+GRID_LABELS:
+::
+
+COLLECTIONS: case1
+::
+
+  case1.template:  '%y4.nc4',
+  case1.format:    'CFIO',
+  case1.frequency:  010000,
+  case1.duration: 000000,
+  case1.deflate: 1,
+  case1.nbits: 10,
+  case1.fields: 'VAR2D', 'Root',
+                'VAR3D', 'Root',
+                          ::

Tests/ExtData_Testing_Framework/test_cases/case31/HISTORY2.rc

@@ -0,0 +1,5 @@
+GRID_LABELS:
+::
+
+COLLECTIONS:
+::

Tests/ExtData_Testing_Framework/test_cases/case31/README

@@ -0,0 +1 @@
+Case, 12-month/12 time 2004 file with 2 updates, non-climatology

Tests/ExtData_Testing_Framework/test_cases/case31/extdata.yaml

@@ -0,0 +1,5 @@
+Collections:
+   fstream1: {template: case1.%y4.nc4, valid_range: "2004-01-01/2005-01-01" }
+Exports:
+   VAR2D: {variable: VAR2D, collection: fstream1}
+   VAR3D: {variable: VAR3D, collection: fstream1}

Tests/ExtData_Testing_Framework/test_cases/case32/AGCM1.rc

@@ -0,0 +1,24 @@
+NX: 1
+NY: 1
+
+Root.GRID_TYPE: LatLon
+Root.GRIDNAME: DC90x45-PC
+Root.LM: 3
+Root.IM_WORLD: 90
+Root.JM_WORLD: 45
+Root.POLE: 'PC'
+Root.DATELINE: 'DC'
+
+RUN_MODE: GenerateExports
+
+EXPORT_STATE::
+VAR2D , time , days , xy , c
+VAR3D , time , days , xyz , c
+::
+
+FILL_DEF::
+VAR2D time
+VAR3D time
+::
+
+REF_TIME: 20040701 000000

Tests/ExtData_Testing_Framework/test_cases/case32/AGCM2.rc

@@ -0,0 +1,29 @@
+NX: 1
+NY: 1
+
+Root.GRID_TYPE: LatLon
+Root.GRIDNAME: DC90x45-PC
+Root.LM: 3
+Root.IM_WORLD: 90
+Root.JM_WORLD: 45
+Root.POLE: 'PC'
+Root.DATELINE: 'DC'
+
+RUN_MODE: CompareImports
+
+IMPORT_STATE::
+VAR2D , time , days , xy , c
+VAR3D , time , days , xyz , c
+::
+
+EXPORT_STATE::
+VAR2D , time , days , xy , c
+VAR3D , time , days , xyz , c
+::
+
+FILL_DEF::
+VAR2D time
+VAR3D time
+::
+
+REF_TIME: 20040701 000000

Tests/ExtData_Testing_Framework/test_cases/case32/CAP.rc

@@ -0,0 +1,4 @@
+CASES::
+CAP1.rc
+CAP2.rc
+::