v17: Use DATAROOT space for gsidiags in dev/gfs.v17 (NOAA-EMC#4537)

This will bring in PRs NOAA-EMC#4476 and NOAA-EMC#4487 to the dev/gfs.v17 branch to
enable the use of DATAROOT space for gsi diagnostic files, which allows
easy use of links rather than sending a tarball to COM.

Author: DavidHuber-NOAA

dev/jobs/JGLOBAL_ATMOS_ANALYSIS

@@ -36,6 +36,7 @@ RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
 MEMDIR='ensstat' RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
     COMIN_ATMOS_HISTORY_ENS_PREV:COM_ATMOS_HISTORY_TMPL
 
+# shellcheck disable=SC2153
 mkdir -p "${COMOUT_ATMOS_ANALYSIS}"
 
 export ATMGES="${COMIN_ATMOS_HISTORY_PREV}/${GPREFIX}atm.f006.nc"
@@ -45,6 +46,15 @@ if [[ ! -f "${ATMGES}" ]]; then
     err_exit "FILE MISSING: ATMGES == ${ATMGES}"
 fi
 
+export GENDIAG="${GENDIAG:-YES}"
+
+# Create a DATAROOT directory variable for the gsidiags if GENDIAG is YES
+if [[ "${GENDIAG}" == "YES" ]]; then
+    # shellcheck disable=SC2311
+    pCOMOUT_ATMOS_ANALYSIS="$(dataroot_com_path "${COMOUT_ATMOS_ANALYSIS}")"
+    export pCOMOUT_ATMOS_ANALYSIS
+fi
+
 # Get LEVS
 export LEVS
 LEVS=$(${NCLEN} "${ATMGES}" pfull) && true

dev/jobs/JGLOBAL_ATMOS_ANALYSIS_DIAG

@@ -26,6 +26,11 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
     COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL
 mkdir -m 775 -p "${COMOUT_ATMOS_ANALYSIS}"
 
+# Create a DATAROOT directory variable where the gsidiags live
+# shellcheck disable=SC2153,SC2311
+pCOMIN_ATMOS_ANALYSIS="$(dataroot_com_path "${COMIN_ATMOS_ANALYSIS}")"
+export pCOMIN_ATMOS_ANALYSIS
+
 ###############################################################
 # Run relevant script
 ${ANALDIAGSH:-${SCRgfs}/exglobal_diag.sh} && true

dev/jobs/JGLOBAL_ENKF_DIAG

@@ -29,6 +29,11 @@ MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
     COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \
     COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL
 
+# Create a DATAROOT directory variable where the gsidiags live
+# shellcheck disable=SC2153,SC2311
+pCOMIN_ATMOS_ANALYSIS="$(dataroot_com_path "${COMIN_ATMOS_ANALYSIS}")"
+export pCOMIN_ATMOS_ANALYSIS
+
 #RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
 #    COMIN_OBS_PREV:COM_OBS_TMPL \
 #    COMIN_ATMOS_ANALYSIS_DET_PREV:COM_ATMOS_ANALYSIS_TMPL

dev/jobs/JGLOBAL_ENKF_SELECT_OBS

@@ -28,6 +28,7 @@ RUN=${RUN/enkf/} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
     COMIN_OBS:COM_OBS_TMPL
 MEMDIR='ensstat' YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
     COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL
+# shellcheck disable=SC2153
 declare -rx COMOUT_ATMOS_ANALYSIS_ENS="${COMOUT_ATMOS_ANALYSIS}"
 
 RUN=${RUN/enkf/} YMD=${PDY} HH=${cyc} declare_from_tmpl -r \
@@ -40,6 +41,7 @@ MEMDIR='ensstat' RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
 RUN="${GDUMP}" YMD=${gPDY} HH=${gcyc} declare_from_tmpl -r \
     COMIN_ATMOS_ANALYSIS_DET_PREV:COM_ATMOS_ANALYSIS_TMPL
 
+# shellcheck disable=SC2153
 mkdir -p "${COMOUT_ATMOS_ANALYSIS}"
 
 export ATMGES_ENSMEAN="${COMIN_ATMOS_HISTORY_PREV}/${GPREFIX}${GSUFFIX}atm.f006.nc"
@@ -48,6 +50,15 @@ if [[ ! -f ${ATMGES_ENSMEAN} ]]; then
     err_exit "FILE MISSING: ATMGES_ENSMEAN == ${ATMGES_ENSMEAN}"
 fi
 
+export GENDIAG="${GENDIAG:-YES}"
+
+# Create a DATAROOT directory variable for the gsidiags if GENDIAG is YES
+if [[ "${GENDIAG}" == "YES" ]]; then
+    # shellcheck disable=SC2311
+    pCOMOUT_ATMOS_ANALYSIS="$(dataroot_com_path "${COMOUT_ATMOS_ANALYSIS}")"
+    export pCOMOUT_ATMOS_ANALYSIS
+fi
+
 LEVS=$(${NCLEN} "${ATMGES}" pfull) && true
 export err=$?
 set_strict

dev/scripts/exglobal_atmos_analysis.sh

@@ -410,6 +410,16 @@ else
     echo "not using correlated obs error"
 fi
 
+# If GENDIAG is selected, verify that pCOMOUT_ATMOS_ANALYSIS is set
+if [[ "${GENDIAG}" == "YES" ]]; then
+    if [[ -z "${pCOMOUT_ATMOS_ANALYSIS}" ]]; then
+        export err=1
+        err_exit "pCOMOUT_ATMOS_ANALYSIS must be set when GENDIAG=YES"
+    fi
+    # Make the gsidiags directory to house the GSI diagnostic data
+    GSIDIAGDIR=${GSIDIAGDIR:-"${pCOMOUT_ATMOS_ANALYSIS}/gsidiags"}
+fi
+
 ##############################################################
 # CRTM Spectral and Transmittance coefficients
 mkdir -p crtm_coeffs
@@ -896,20 +906,36 @@ if [[ "${SENDECF}" == "YES" && "${RUN}" != "enkf" ]]; then
     ecflow_client --event release_fcst
 fi
 
+# shellcheck disable=SC2312
+echo "${rCDUMP} ${PDY}${cyc} atminc done at $(date)" > "${COMOUT_ATMOS_ANALYSIS}/${APREFIX}increment.done.txt"
+
 # Diagnostic files
 # if requested, GSI diagnostic file directories for use later
 if [[ "${GENDIAG}" == "YES" ]]; then
-    tar -cvf gsidiags.tar dir.????
-    export err=$?
-    if [[ ${err} -ne 0 ]]; then
-        err_exit "Failed to tar GSI diagnostic directories!"
+    # Move the gsidiags dir.* directories to pCOMOUT_ATMOS_ANALYSIS for diagnostic jobs
+    # First, check that the directories exist (we need at least one, so stop after the first match)
+    # shellcheck disable=SC2312
+    count_dirs=$(find . -maxdepth 1 -type d -name 'dir.????' -printf "." -quit | wc -c)
+    if [[ ${count_dirs:-0} -gt 0 ]]; then
+        mkdir -p "${GSIDIAGDIR}"
+        err=$?
+
+        if [[ ! -d "${GSIDIAGDIR}" || ${err} -ne 0 ]]; then
+            err_exit "Failed to create gsidiags directory at ${GSIDIAGDIR}"
+        fi
+
+        for dir in dir.????; do
+            mv "${dir}" "${GSIDIAGDIR}/"
+            export err=$?
+            if [[ ${err} -ne 0 ]]; then
+                err_exit "Failed to move ${dir} to ${GSIDIAGDIR}/"
+            fi
+        done
+    else
+        echo "WARNING: No gsidiags dir.* directories found to move."
     fi
-    cpfs gsidiags.tar "${COMOUT_ATMOS_ANALYSIS}/${APREFIX}gsidiags${DIAG_SUFFIX:-}.tar"
 fi
 
-# shellcheck disable=SC2312
-echo "${rCDUMP} ${PDY}${cyc} atminc done at $(date)" > "${COMOUT_ATMOS_ANALYSIS}/${APREFIX}increment.done.txt"
-
 ################################################################################
 
 exit "${err}"

dev/scripts/exglobal_diag.sh

@@ -39,7 +39,7 @@ rm -f "${RADSTAT}" "${PCPSTAT}" "${CNVSTAT}" "${OZNSTAT}"
 
 # Obs diag
 GENDIAG=${GENDIAG:-"YES"}
-GSIDIAG=${GSIDIAG:-"${COMIN_ATMOS_ANALYSIS}/${APREFIX}gsidiags${DIAG_SUFFIX:-}.tar"}
+GSIDIAGDIR=${GSIDIAGDIR:-"${pCOMIN_ATMOS_ANALYSIS}/gsidiags"}
 USE_BUILD_GSINFO=${USE_BUILD_GSINFO:-"NO"}
 DIAG_COMPRESS=${DIAG_COMPRESS:-"YES"}
 if [[ "${DIAG_COMPRESS:-}" == "YES" ]]; then
@@ -55,15 +55,33 @@ if [[ "${GENDIAG}" != "YES" ]]; then
     exit 0
 fi
 
+# Check that the gsidiags directory exists
+
+if [[ ! -d "${GSIDIAGDIR}" ]]; then
+    export err=1
+    err_exit "gsidiags directory ${GSIDIAGDIR} does not exist"
+fi
+
 ################################################################################
-# Copy gsidiags.tar file from COMIN to DATA and untar
-cpreq "${GSIDIAG}" ./gsidiags.tar
-tar -xvf gsidiags.tar
-export err=$?
-if [[ ${err} -ne 0 ]]; then
-    err_exit "Unable to unpack gsidiags.tar file!"
+# Link to the gsidiags directory if it is populated
+# shellcheck disable=SC2312
+count_dirs=$(find "${GSIDIAGDIR}" -maxdepth 1 -type d -name "dir.*" | wc -l)
+if [[ ${count_dirs} -eq 0 ]]; then
+    export err=1
+    err_exit "No gsidiags directories found in ${GSIDIAGDIR}"
+fi
+
+# Continue if there is at least one file to process
+# Note -quit stops find after the first match
+# shellcheck disable=SC2312
+count_files=$(find "${GSIDIAGDIR}"/dir.* -maxdepth 1 -type f -printf '.' -quit | wc -c)
+if [[ ${count_files} -eq 0 ]]; then
+    echo "WARNING: No diagnostic files found to process!"
+    exit 0
 fi
 
+${NLN} "${GSIDIAGDIR}/"dir.* .
+
 # Set up lists and variables for various types of diagnostic files.
 ntype=3
 

ush/bash_utils.sh

@@ -36,7 +36,9 @@ function declare_from_tmpl() {
     #       MEMDIR='mem001' YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
     #           COMOUT_ATMOS_HISTORY:COM_ATMOS_HISTORY_TMPL
     #
-    if [[ ${DEBUG_WORKFLOW:-"NO"} == "NO" ]]; then set +x; fi
+    if [[ ${DEBUG_WORKFLOW:-"NO"} == "NO" ]]; then
+        set +x
+    fi
     local opts="-g"
     local OPTIND=1
     while getopts "rx" option; do
@@ -110,6 +112,53 @@ function wait_for_file() {
     return 1
 }
 
+# This utility is to be used to create a COM structure in the DATAROOT
+# It will replace the root path (up to $COMROOT) with $DATAROOT
+# Use realpath --relative-to to get the relative path from $COMROOT to the target file
+# and then prepend $DATAROOT to that path to get the new target path
+function dataroot_com_path() {
+    #
+    # Generate a COM path in the DATAROOT based on an existing COM path.
+    #
+    # This function takes an existing COM path and generates a corresponding
+    # path in the DATAROOT by replacing the root directory with DATAROOT.
+    #
+    # Syntax:
+    #   dataroot_com_path original_com_path
+    #
+    #   original_com_path: The original COM path to be transformed.
+    #
+    # Example:
+    #   # Declare COMOUT_ATMOS_ANALYSIS using template
+    #   YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+    #       COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL
+    #   # Get the DATAROOT version of the COM path
+    #   pCOMOUT_ATMOS_ANALYSIS=$(dataroot_com_path "${COMOUT_ATMOS_ANALYSIS}")
+    #   echo "New COM path in DATAROOT: ${pCOMOUT_ATMOS_ANALYSIS}"
+    #
+
+    set +x
+    if [[ $# -ne 1 ]]; then
+        echo "FATAL ERROR in dataroot_com_path: Incorrect number of arguments!"
+        echo "Usage: dataroot_com_path original_com_path"
+        exit 2
+    fi
+
+    local original_com_path=${1}
+
+    if [[ -z "${COMROOT:-}" || -z "${DATAROOT:-}" ]]; then
+        echo "FATAL ERROR in dataroot_com_path: COMROOT and DATAROOT must be defined!"
+        exit 2
+    fi
+
+    local relative_path
+    relative_path=$(realpath --relative-to="${COMROOT}" "${original_com_path}")
+    local new_com_path="${DATAROOT}/${relative_path}"
+
+    echo "${new_com_path}"
+    set_trace
+}
+
 # Initialize stacks for tick-tock profiling (initialize only once)
 if [[ -z ${_GW_TIMER_STACK+x} ]]; then
     declare -xa _GW_TIMER_STACK=()
@@ -198,5 +247,6 @@ tock() {
 
 declare -xf declare_from_tmpl
 declare -xf wait_for_file
+declare -xf dataroot_com_path
 declare -xf tick
 declare -xf tock