Write GSI diagnostic files directly to DATAROOT to eliminate tar/untar overhead

dev/jobs/JGLOBAL_ATMOS_ANALYSIS

@@ -38,6 +38,12 @@ MEMDIR='ensstat' RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
 
 mkdir -p "${COMOUT_ATMOS_ANALYSIS}"
 
+# Create pCOM_ATMOS_ANALYSIS in DATAROOT for direct GSI diagnostic writes
+# This allows GSI to write diagnostic files directly to a persistent location
+# that can be accessed by downstream jobs without tarring/untarring
+export pCOM_ATMOS_ANALYSIS="${DATAROOT}/${RUN}.${PDY}${cyc}/analysis/atmos"
+mkdir -p "${pCOM_ATMOS_ANALYSIS}"
+
 export ATMGES="${COMIN_ATMOS_HISTORY_PREV}/${GPREFIX}atm.f006.nc"
 if [[ ! -f "${ATMGES}" ]]; then
 

dev/jobs/JGLOBAL_ATMOS_ANALYSIS_DIAG

@@ -26,6 +26,10 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
     COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL
 mkdir -m 775 -p "${COMOUT_ATMOS_ANALYSIS}"
 
+# Create pCOM_ATMOS_ANALYSIS in DATAROOT to access GSI diagnostic files
+# This points to the same location used by the analysis job
+export pCOM_ATMOS_ANALYSIS="${DATAROOT}/${RUN}.${PDY}${cyc}/analysis/atmos"
+
 ###############################################################
 # Run relevant script
 ${ANALDIAGSH:-${SCRgfs}/exglobal_diag.sh} && true

dev/scripts/exglobal_atmos_analysis.sh

@@ -896,14 +896,22 @@ if [[ "${SENDECF}" == "YES" && "${RUN}" != "enkf" ]]; then
 fi
 
 # Diagnostic files
-# if requested, GSI diagnostic file directories for use later
+# If requested, move GSI diagnostic file directories to pCOM_ATMOS_ANALYSIS for use later
 if [[ "${GENDIAG}" == "YES" ]]; then
-    tar -cvf gsidiags.tar dir.????
-    export err=$?
-    if [[ ${err} -ne 0 ]]; then
-        err_exit "Failed to tar GSI diagnostic directories!"
+    # Create gsidiags directory in pCOM_ATMOS_ANALYSIS
+    mkdir -p "${pCOM_ATMOS_ANALYSIS}/gsidiags"
+
+    # Move dir.???? directories to pCOM_ATMOS_ANALYSIS/gsidiags
+    # This allows downstream jobs to access them directly without tarring/untarring
+    if compgen -G "dir.????" > /dev/null; then
+        mv dir.???? "${pCOM_ATMOS_ANALYSIS}/gsidiags/"
+        export err=$?
+        if [[ ${err} -ne 0 ]]; then
+            err_exit "Failed to move GSI diagnostic directories to ${pCOM_ATMOS_ANALYSIS}/gsidiags!"
+        fi
+    else
+        echo "WARNING: No GSI diagnostic directories (dir.????) found!"
     fi
-    cpfs gsidiags.tar "${COMOUT_ATMOS_ANALYSIS}/${APREFIX}gsidiags${DIAG_SUFFIX:-}.tar"
 fi
 
 echo "${rCDUMP} ${PDY}${cyc} atminc done at $(date)" > "${COMOUT_ATMOS_ANALYSIS}/${APREFIX}increment.done.txt"

dev/scripts/exglobal_diag.sh

@@ -39,7 +39,6 @@ rm -f "${RADSTAT}" "${PCPSTAT}" "${CNVSTAT}" "${OZNSTAT}"
 
 # Obs diag
 GENDIAG=${GENDIAG:-"YES"}
-GSIDIAG=${GSIDIAG:-"${COMIN_ATMOS_ANALYSIS}/${APREFIX}gsidiags${DIAG_SUFFIX:-}.tar"}
 USE_BUILD_GSINFO=${USE_BUILD_GSINFO:-"NO"}
 DIAG_COMPRESS=${DIAG_COMPRESS:-"YES"}
 if [[ "${DIAG_COMPRESS:-}" == "YES" ]]; then
@@ -56,12 +55,22 @@ if [[ "${GENDIAG}" != "YES" ]]; then
 fi
 
 ################################################################################
-# Copy gsidiags.tar file from COMIN to DATA and untar
-cpreq "${GSIDIAG}" ./gsidiags.tar
-tar -xvf gsidiags.tar
-export err=$?
-if [[ ${err} -ne 0 ]]; then
-    err_exit "Unable to unpack gsidiags.tar file!"
+# Link GSI diagnostic directories from pCOM_ATMOS_ANALYSIS to DATA
+# The diagnostic files are now in pCOM_ATMOS_ANALYSIS/gsidiags instead of a tar file
+if [[ ! -d "${pCOM_ATMOS_ANALYSIS}/gsidiags" ]]; then
+    err_exit "GSI diagnostic directory ${pCOM_ATMOS_ANALYSIS}/gsidiags does not exist!"
+fi
+
+# Link each dir.???? directory into DATA
+for diagdir in "${pCOM_ATMOS_ANALYSIS}/gsidiags"/dir.????; do
+    if [[ -d "${diagdir}" ]]; then
+        ln -sf "${diagdir}" "${DATA}/"
+    fi
+done
+
+# Verify we have diagnostic directories
+if ! compgen -G "${DATA}/dir.????" > /dev/null; then
+    err_exit "No GSI diagnostic directories found in ${pCOM_ATMOS_ANALYSIS}/gsidiags!"
 fi
 
 # Set up lists and variables for various types of diagnostic files.