v17: Update NSST dtf file and master idx

.gitignore

@@ -141,6 +141,7 @@ sorc/syndat_maksynrc.fd
 sorc/syndat_qctropcy.fd
 sorc/tave.fd
 sorc/tocsbufr.fd
+sorc/tref_calc.fd
 sorc/upp.fd
 sorc/vint.fd
 sorc/webtitle.fd

dev/jobs/JGLOBAL_ENKF_SELECT_OBS

@@ -67,7 +67,7 @@ export PREPQCPF="${COMIN_OBS}/${OPREFIX}prepbufr.acft_profiles"
 
 # Deterministic analysis and increment files
 export SFCANL="${COMOUT_ATMOS_ANALYSIS_DET}/${APREFIX_DET}analysis.sfc.a006.nc"
-export DTFANL="${COMOUT_ATMOS_ANALYSIS_DET}/${APREFIX_DET}analysis.dtf.a006.nc"
+export DTFINC="${COMOUT_ATMOS_ANALYSIS_DET}/${APREFIX_DET}increment.dtf.i006.nc"
 export ATMANL="${COMOUT_ATMOS_ANALYSIS_DET}/${APREFIX_DET}analysis.atm.a006.nc"
 export ATMINC="${COMOUT_ATMOS_ANALYSIS_DET}/${APREFIX_DET}increment.atm.i006.nc"
 

dev/parm/config/gcafs/config.fetch

@@ -18,9 +18,9 @@ if [[ "${PDY}${cyc}" -gt "${SDATE}" ]]; then
    fi
    export gdas_version
    if [[ "${machine}" == "ORION" || "${machine}" == "HERCULES" ]]; then
-       FETCH_YAML_TMPL_LIST="${PARMgfs}/fetch/${NET}_${APP}_gdas-anl_msu.yaml.j2,${PARMgfs}/fetch/${NET}_${APP}_gdas-dtfanl_msu.yaml.j2"
+       FETCH_YAML_TMPL_LIST="${PARMgfs}/fetch/${NET}_${APP}_gdas-anl_msu.yaml.j2,"
    else
-       FETCH_YAML_TMPL_LIST="${PARMgfs}/fetch/${NET}_${APP}_gdas-anl.yaml.j2,${PARMgfs}/fetch/${NET}_${APP}_gdas-dtfanl.yaml.j2"
+       FETCH_YAML_TMPL_LIST="${PARMgfs}/fetch/${NET}_${APP}_gdas-anl.yaml.j2,"
    fi
    export FETCH_YAML_TMPL_LIST
 else

dev/scripts/exglobal_atmos_analysis.sh

@@ -178,7 +178,6 @@ GRADSTAT=${GRADSTAT:-${COMIN_ATMOS_ANALYSIS_PREV}/${GPREFIX}radstat.tar}
 # Analysis files
 export APREFIX=${APREFIX:-""}
 SFCANL=${SFCANL:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}analysis.sfc.a006.nc}
-DTFANL=${DTFANL:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}analysis.dtf.a006.nc}
 ATMANL=${ATMANL:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}analysis.atm.a006.nc}
 ABIAS=${ABIAS:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias.txt}
 ABIASPC=${ABIASPC:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_pc.txt}
@@ -188,6 +187,7 @@ GSISTAT=${GSISTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}gsistat.txt}
 
 # Increment files
 ATMINC=${ATMINC:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}increment.atm.i006.nc}
+DTFINC=${DTFINC:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}increment.dtf.i006.nc}
 
 # Obs diag
 RUN_SELECT=${RUN_SELECT:-"NO"}
@@ -639,7 +639,7 @@ ${NLN} "${ABIASPC}" satbias_pc.out
 ${NLN} "${ABIASAIR}" aircftbias_out
 
 if [[ "${DONST}" == "YES" ]]; then
-    ${NLN} "${DTFANL}" dtfanl
+    ${NLN} "${DTFINC}" dtfanl
 fi
 
 # If requested, link (and if tarred, de-tar obsinput.tar) into obs_input.* files

dev/scripts/exglobal_atmos_analysis_calc.sh

@@ -73,11 +73,11 @@ ATMG09=${ATMG09:-"${COMIN_ATMOS_HISTORY_PREV}/${GPREFIX}atm.f009.nc"}
 # Analysis files
 export APREFIX=${APREFIX:-""}
 SFCANL=${SFCANL:-"${COMOUT_ATMOS_ANALYSIS}/${APREFIX}analysis.sfc.a006.nc"}
-DTFANL=${DTFANL:-"${COMOUT_ATMOS_ANALYSIS}/${APREFIX}analysis.dtf.a006.nc"}
 ATMANL=${ATMANL:-"${COMOUT_ATMOS_ANALYSIS}/${APREFIX}analysis.atm.a006.nc"}
 
 # Increment files
 ATMINC=${ATMINC:-"${COMOUT_ATMOS_ANALYSIS}/${APREFIX}increment.atm.i006.nc"}
+DTFINC=${DTFINC:-"${COMOUT_ATMOS_ANALYSIS}/${APREFIX}increment.dtf.i006.nc"}
 
 # Set script / GSI control parameters
 DOHYBVAR=${DOHYBVAR:-"NO"}

dev/scripts/exglobal_atmos_products.sh

@@ -40,6 +40,9 @@ IFS=':' read -ra grids <<< "${grid_string}"
 # Files needed by ${USHgfs}/interp_atmos_master.sh
 MASTER_FILE="${COMIN_ATMOS_MASTER}/${PREFIX}master.${fhr3}.grib2"
 
+# Create an index file for the master
+${WGRIB2} -s "${MASTER_FILE}" > "${MASTER_FILE}.idx"
+
 for ((nset = 1; nset <= downset; nset++)); do
 
     echo "INFO: Begin processing nset = ${nset}"

dev/scripts/exglobal_atmos_sfcanl.sh

@@ -112,10 +112,10 @@ done
 # There is only a single NSST analysis at the middle of the window
 # For now use/assume it is the same at the beginning of the window if doing IAU
 if [[ "${DONST}" == "YES" ]]; then
-    export NST_FILE=${NST_FILE:-${COMIN_ATMOS_ANALYSIS}/${APREFIX}analysis.dtf.a006.nc}
+    export NST_FILE=${NST_FILE:-${COMIN_ATMOS_ANALYSIS}/${APREFIX}increment.dtf.i006.nc}
     if [[ -s "${NST_FILE}" ]]; then
-        cpreq "${NST_FILE}" "${DATA}/dtfanl"
-        export NST_FILE="dtfanl"
+        cpreq "${NST_FILE}" "${DATA}/dtfinc"
+        export NST_FILE="dtfinc"
     else
         export NST_FILE="NULL"
     fi

dev/scripts/exglobal_enkf_sfc.sh

@@ -127,7 +127,7 @@ else
 fi
 
 if [[ "${DONST}" == "YES" ]]; then
-    export NST_FILE=${NST_FILE:-${COMIN_ATMOS_ANALYSIS_DET}/${APREFIX}analysis.dtf.a006.nc}
+    export NST_FILE=${NST_FILE:-${COMIN_ATMOS_ANALYSIS_DET}/${APREFIX}increment.dtf.i006.nc}
 else
     export NST_FILE="NULL"
 fi
@@ -159,7 +159,7 @@ export MAX_TASKS_CY=${NMEM_ENS}
 
 if [[ "${DOIAU}" == "YES" ]]; then
     # Update surface restarts at beginning of window when IAU is ON
-    # For now assume/hold analysis.dtf.nc is valid at beginning of window.
+    # For now assume/hold increment.dtf.nc is valid at beginning of window.
 
     for n in $(seq 1 "${ntiles}"); do
 

dev/scripts/exglobal_offline_atmos_analysis.py

@@ -33,5 +33,8 @@
     # Compute the increment between the analysis and background
     offline_anl.calc_increment()
 
+    # Compute the tref increment
+    offline_anl.calc_tref_inc()
+
     # Copy the analysis increment and regridded analysis back to COM
     offline_anl.finalize()

dev/ush/fetch_gdas_for_gcafs.sh

@@ -37,27 +37,23 @@ PDY="${cycle_Y}${cycle_M}${cycle_D}"
 # HPSS path for the two tar files
 hpss_path_root="/NCEPPROD/hpssprod/runhistory/rh${cycle_Y}/${cycle_YM}/${PDY}"
 hpss_file_nc="com_gfs_${gdas_version}_gdas.${PDY}_${cyc}.gdas_nc.tar"
-hpss_file_restart="com_gfs_${gdas_version}_gdas.${PDY}_${cyc}.gdas_restart.tar"
 
 # get the names of the files to extract
 atmanl="./gdas.${PDY}/${cyc}/atmos/gdas.t${cyc}z.atmanl.nc"
-dtfanl="./gdas.${PDY}/${cyc}/atmos/gdas.t${cyc}z.dtfanl.nc"
+sfcanl="./gdas.${PDY}/${cyc}/atmos/gdas.t${cyc}z.sfcanl.nc"
 
 # Fetch the tar files from HPSS
 cd "${OUTPUT_DIR}/tmp"
 
-htar -xvf "${hpss_path_root}/${hpss_file_nc}" "${atmanl}"
-htar -xvf "${hpss_path_root}/${hpss_file_restart}" "${dtfanl}"
+htar -xvf "${hpss_path_root}/${hpss_file_nc}" "${atmanl}" "${sfcanl}"
 
 # create the output tar files
 echo "creating output tar files"
-tar cvf "${hpss_file_nc}" "${atmanl}"
-tar cvf "${hpss_file_restart}" "${dtfanl}"
+tar cvf "${hpss_file_nc}" "${atmanl}" "${sfcanl}"
 
 # Move the tar files to the output directory
 echo "moving tar files to ${OUTPUT_DIR}"
 mv "${hpss_file_nc}" "${OUTPUT_DIR}/"
-mv "${hpss_file_restart}" "${OUTPUT_DIR}/"
 
 # Clean up temporary directory
 rm -rf "${OUTPUT_DIR}/tmp"

dev/ush/make_ee2_links.sh

@@ -111,8 +111,8 @@ for dir in "${gdas_list[@]}" "${gfs_list[@]}" "${gcdas_list[@]}" "${gcafs_list[@
             if [[ -f "${system_prefix}.t${cyc}z.cnvstat" ]]; then
                 link_file "${system_prefix}.t${cyc}z.cnvstat" "${system_prefix}.t${cyc}z.cnvstat.tar"
             fi
-            if [[ -f "${system_prefix}.t${cyc}z.dtfanl.nc" ]]; then
-                link_file "${system_prefix}.t${cyc}z.dtfanl.nc" "${system_prefix}.t${cyc}z.analysis.dtf.a006.nc"
+            if [[ -f "${system_prefix}.t${cyc}z.dtfinc.nc" ]]; then
+                link_file "${system_prefix}.t${cyc}z.dtfinc.nc" "${system_prefix}.t${cyc}z.increment.dtf.i006.nc"
             fi
             if [[ -f "${system_prefix}.t${cyc}z.gsistat" ]]; then
                 link_file "${system_prefix}.t${cyc}z.gsistat" "${system_prefix}.t${cyc}z.gsistat.txt"

parm/archive/gdas_restarta.yaml.j2

@@ -45,7 +45,7 @@ gdas_restarta:
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_pc.txt"
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}cnvstat.tar"
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_int.txt"
-        - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}analysis.dtf.a006.nc"
+        - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}increment.dtf.i006.nc"
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}analysis.done.txt"
         {% else %}
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}rad_varbc_params.tar"

parm/archive/gfs_arcdir.yaml.j2

@@ -103,6 +103,6 @@ copy_opt:
        "{{ ARCDIR }}/abias_air.{{ RUN }}.{{ cycle_YMDH }}"]
     - ["{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}abias_int.txt",
        "{{ ARCDIR }}/abias_int.{{ RUN }}.{{ cycle_YMDH }}"]
-    - ["{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}analysis.dtf.a006.nc",
-       "{{ ARCDIR }}/dtfanl.{{ RUN }}.{{ cycle_YMDH }}.nc"]
+    - ["{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}increment.dtf.i006.nc",
+       "{{ ARCDIR }}/dtfinc.{{ RUN }}.{{ cycle_YMDH }}.nc"]
 {% endif %}

parm/archive/gfs_netcdfa.yaml.j2

@@ -26,6 +26,6 @@ gfs_netcdfa:
         {% endfor %}
     optional:
         {% if not DO_JEDIATMVAR %}
-        - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}analysis.dtf.a006.nc"
+        - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}increment.dtf.i006.nc"
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}increment.done.txt"
         {% endif %}

parm/fetch/gcafs_ATMA_gdas-anl.yaml.j2

@@ -9,4 +9,6 @@ target:
     contents:
         # Atmospheric analysis
         - ./gdas.{{ cycle_YMD }}/{{ cycle_HH }}/atmos/gdas.t{{ cycle_HH }}z.atmanl.nc
+        # Surface analysis
+        - ./gdas.{{ cycle_YMD }}/{{ cycle_HH }}/atmos/gdas.t{{ cycle_HH }}z.sfcanl.nc
     destination: "{{ ROTDIR }}"

parm/fetch/gcafs_ATMA_gdas-anl_msu.yaml.j2

@@ -9,4 +9,6 @@ target:
     contents:
         # Atmospheric analysis
         - ./gdas.{{ cycle_YMD }}/{{ cycle_HH }}/atmos/gdas.t{{ cycle_HH }}z.atmanl.nc
+        # Surface analysis
+        - ./gdas.{{ cycle_YMD }}/{{ cycle_HH }}/atmos/gdas.t{{ cycle_HH }}z.sfcanl.nc
     destination: "{{ ROTDIR }}"

sorc/link_workflow.sh

@@ -362,7 +362,7 @@ cd "${HOMEgfs}/exec" || exit 1
 for utilexe in fbwndgfs.x gaussian_sfcanl.x gfs_bufr.x supvit.x syndat_getjtbul.x \
     syndat_maksynrc.x syndat_qctropcy.x tocsbufr.x overgridid.x rdbfmsua.x \
     mkgfsawps.x enkf_chgres_recenter_nc.x tave.x vint.x ocnicepost.x webtitle.x \
-    ensadd.x ensppf.x ensstat.x wave_stat.x; do
+    ensadd.x ensppf.x ensstat.x wave_stat.x tref_calc.x; do
     if [[ -s "${utilexe}" ]]; then
         rm -f "${utilexe}"
     fi
@@ -576,6 +576,7 @@ for prog in enkf_chgres_recenter_nc.fd \
     syndat_qctropcy.fd \
     tave.fd \
     tocsbufr.fd \
+    tref_calc.fd \
     vint.fd \
     webtitle.fd \
     ocnicepost.fd; do

ush/python/pygfs/task/offline_analysis.py

@@ -81,11 +81,14 @@ def initialize(self) -> None:
         fcst_file_in = os.path.join(self.task_config.COMIN_ATMOS_HISTORY_PREV,
                                     f"{self.task_config.GPREFIX}atm.f006.nc")
         files_to_copy.append([fcst_file_in, os.path.join(self.task_config.DATA, "atmges_mem001")])
+        sfcfcst_file_in = os.path.join(self.task_config.COMIN_ATMOS_HISTORY_PREV,
+                                       f"{self.task_config.GPREFIX}sfc.f006.nc")
+        files_to_copy.append([sfcfcst_file_in, os.path.join(self.task_config.DATA, "sfcges_mem001")])
         # TODO: Re-stage all of the inputs on HPSS to match EE2-compliant filenames
         anl_file_in = os.path.join(self.task_config.COMIN_ATMOS_ANALYSIS.replace('analysis', ''), f"{self.task_config.APREFIX_IN}atmanl.nc")
         files_to_copy.append([anl_file_in, os.path.join(self.task_config.DATA, "atmanl.input.nc")])
-        # sfcanl_file_in = os.path.join(self.task_config.COMIN_ATMOS_ANALYSIS, f"{GPREFIX}analysis.sfc.a006.nc")
-        # files_to_copy.append([sfcanl_file_in, os.path.join(self.task_config.DATA, "sfcanl.input.nc")])
+        sfcanl_file_in = os.path.join(self.task_config.COMIN_ATMOS_ANALYSIS.replace('analysis', ''), f"{self.task_config.APREFIX_IN}sfcanl.nc")
+        files_to_copy.append([sfcanl_file_in, os.path.join(self.task_config.DATA, "sfcanl.input.nc")])
         FileHandler({'copy': files_to_copy}).sync()
 
         # generate namelists for the executables
@@ -129,9 +132,26 @@ def initialize(self) -> None:
             f90nml.write(namelist, nmlfile)
         logger.info(f"Wrote namelist to {os.path.join(self.task_config.DATA, 'calc_increment.nml')}")
 
+        # setup namelist for tref increment calculation
+        logger.info("Generating namelist for 'tref_calc'")
+        namelist = {
+            "tref_calc_setup": {
+                "i_output": self.task_config.nlon_interp,
+                "j_output": self.task_config.nlat_interp,
+                "sfcanl_file": "sfcanl.input.nc",
+                "sfcf006_file": "sfcges_mem001",
+                "output_file": "dtfinc.nc",
+            }
+        }
+
+        logger.info(namelist)
+        with open(os.path.join(self.task_config.DATA, 'tref_calc.nml'), 'w') as nmlfile:
+            f90nml.write(namelist, nmlfile)
+        logger.info(f"Wrote namelist to {os.path.join(self.task_config.DATA, 'tref_calc.nml')}")
+
         # copy executables to $DATA
         executables_to_copy = []
-        executable_list = ['enkf_chgres_recenter_nc.x', 'calc_increment_ens_ncio.x']
+        executable_list = ['enkf_chgres_recenter_nc.x', 'calc_increment_ens_ncio.x', 'tref_calc.x']
         for exec_name in executable_list:
             executables_to_copy.append([os.path.join(self.task_config.EXECgfs, exec_name),
                                         os.path.join(self.task_config.DATA, exec_name)])
@@ -163,6 +183,30 @@ def interpolate_analysis(self) -> None:
             logger.exception(f"An error occured during execution of {exe}")
             raise WorkflowException(f"An error occured during execution of {exe}") from err
 
+    @logit(logger)
+    def calc_tref_inc(self) -> None:
+        """Interpolate the tref analysis and compute the dtf increment.
+
+        Parameters
+        ----------
+        self : OfflineAnalysis
+            Instance of the OfflineAnalysis object
+        """
+
+        # set up and run the executable
+        exe = Executable(self.task_config.APRUN_CHGRES)
+        exe.add_default_arg(os.path.join(self.task_config.DATA, 'tref_calc.x'))
+        exe.add_default_arg(os.path.join(self.task_config.DATA, 'tref_calc.nml'))
+        try:
+            logger.debug(f"Executing {exe}")
+            exe()
+        except OSError:
+            logger.exception(f"Failed to execute {exe}")
+            raise
+        except Exception as err:
+            logger.exception(f"An error occured during execution of {exe}")
+            raise WorkflowException(f"An error occured during execution of {exe}") from err
+
     @logit(logger)
     def calc_increment(self) -> None:
         """Compute the analysis increment for input to the forecast model
@@ -216,9 +260,8 @@ def finalize(self) -> None:
                                os.path.join(self.task_config.COMOUT_OBS, f"{self.task_config.APREFIX}snogrb_t1534.3072.1536")])
         # TODO: Re-stage the inputs for the GCDAS offline analysis on HPSS following EE2-compliant filenames, then update this line
         transfer_files.append([
-            os.path.join(self.task_config.COMIN_ATMOS_ANALYSIS.replace('analysis', ''),
-                         f"{self.task_config.APREFIX_IN}dtfanl.nc"),
+            os.path.join(self.task_config.DATA, "dtfinc.nc"),
             os.path.join(self.task_config.COMOUT_ATMOS_ANALYSIS,
-                         f"{self.task_config.APREFIX}analysis.dtf.a006.nc")
+                         f"{self.task_config.APREFIX}increment.dtf.i006.nc")
         ])
         FileHandler({'copy': transfer_files}).sync()