Ensures any failure in piped output is captured and reported.

.shellcheckrc

@@ -17,3 +17,6 @@ disable=SC2174
 
 # Disable warning of functions in test statements
 disable=SC2310
+
+# Disable warning of read without -r
+disable=SC2312

dev/ci/scripts/run_check_ci.sh

@@ -68,7 +68,6 @@ while true; do
 
   # Get job statistics
   echo "Gather Rocoto statistics"
-  # shellcheck disable=SC2312 # We want to use the exit code of the command
   full_state=$("${HOMEgfs}/dev/ci/scripts/utils/rocotostat.py" -w "${xml}" -d "${db}" -v)
   error_stat=$?
 

dev/ci/scripts/utils/ci_utils.sh

@@ -254,7 +254,6 @@ function delete_dataroot() {
   _runtests="${1}"
   _pslot="${2}"
 
-   # shellcheck disable=SC2312
    eval "$(PDY=0 cyc=0 source "${_runtests}/EXPDIR/${_pslot}/config.base" >& /dev/null; echo _dataroot="${STMP}/RUNDIRS/${_pslot}")"
    if [[ -d "${_dataroot}" ]]; then
       echo "A previous DATAROOT exists for ${_pslot} in ${_dataroot} and is being deleted."

dev/job_cards/rocoto/prep.sh

@@ -74,11 +74,9 @@ if [[ ! -s "${COMINobsproc_PREV}/${GDUMP}.t${gcyc}z.updated.status.tm00.bufr_d"
     fi
 fi
 # exception handling to ensure no dead link
-# shellcheck disable=SC2312
 if [[ $(find "${COMOUT_OBS}" -xtype l | wc -l) -ge 1 ]]; then
     exit 9
 fi
-# shellcheck disable=SC2312
 if [[ $(find "${COMINobsproc_PREV}" -xtype l | wc -l) -ge 1 ]]; then
     exit 9
 fi

dev/jobs/JGLOBAL_ATMOS_ANALYSIS_CALC_FV3JEDI

@@ -2,7 +2,6 @@
 
 # Ignore possible spelling error (nothing is misspelled)
 # shellcheck disable=SC2153
-# shellcheck disable=SC2312
 
 source "${HOMEgfs}/ush/jjob_header.sh" -e "analcalc_fv3jedi" -c "base analcalc_fv3jedi"
 

dev/parm/config/gefs/config.resources.AZUREPW

@@ -6,7 +6,6 @@
 export is_exclusive="True"
 unset memory
 
-# shellcheck disable=SC2312
 for mem_var in $(env | grep '^memory_' | cut -d= -f1); do
   unset "${mem_var}"
 done

dev/parm/config/gefs/config.resources.GOOGLEPW

@@ -6,7 +6,6 @@
 export is_exclusive="True"
 unset memory
 
-# shellcheck disable=SC2312
 for mem_var in $(env | grep '^memory_' | cut -d= -f1); do
   unset "${mem_var}"
 done

dev/parm/config/gfs/config.ecen

@@ -13,7 +13,6 @@ source "${EXPDIR}/config.resources" ecen
 # below queries IAUFHRS_ENKF to determine NECENGRP
 export NECENGRP=1
 if [[ ${DOIAU_ENKF} = "YES" ]]; then
-   # shellcheck disable=SC2312
    ngrps=$(grep -o ',' <<<"${IAUFHRS_ENKF}" | grep -c .)
    ((ngrps++))
    export NECENGRP=${ngrps}

dev/parm/config/gfs/config.resources.AZUREPW

@@ -6,7 +6,6 @@
 export is_exclusive="True"
 unset memory
 
-# shellcheck disable=SC2312
 for mem_var in $(env | grep '^memory_' | cut -d= -f1); do
   unset "${mem_var}"
 done

dev/parm/config/gfs/config.resources.GOOGLEPW

@@ -6,7 +6,6 @@
 export is_exclusive="True"
 unset memory
 
-# shellcheck disable=SC2312
 for mem_var in $(env | grep '^memory_' | cut -d= -f1); do
   unset "${mem_var}"
 done

dev/scripts/exgdas_atmos_verfrad.sh

@@ -102,7 +102,6 @@ for type in ${SATYPE}; do
         netcdf=1
     fi
 
-    # shellcheck disable=SC2312
     if [[ $(find . -maxdepth 1 -type f -name "diag_${type}_ges.${PDY}${cyc}*.${Z}" | wc -l) -gt 0 ]]; then
         mv "diag_${type}_ges.${PDY}${cyc}"*".${Z}" "${type}.${Z}"
         ${UNCOMPRESS} "./${type}.${Z}"
@@ -111,7 +110,6 @@ for type in ${SATYPE}; do
     fi
 
     if [[ ${USE_ANL} -eq 1 ]]; then
-        # shellcheck disable=SC2312
         file_count=$(find . -maxdepth 1 -type f -name "diag_${type}_anl.${PDY}${cyc}*.${Z}" | wc -l)
         if [[ ${file_count} -gt 0 ]]; then
             mv "diag_${type}_anl.${PDY}${cyc}"*".${Z}" "${type}_anl.${Z}"

dev/scripts/exgfs_atmos_awips_20km_1p0deg.sh

@@ -82,7 +82,6 @@ export opt28=' -new_grid_interpolation budget -fi '
 cpreq "${COMIN_ATMOS_GRIB_0p25}/gfs.t${cyc}z.pres_a.0p25.f${fcsthr}.grib2" "tmpfile2${fcsthr}"
 cpreq "${COMIN_ATMOS_GRIB_0p25}/gfs.t${cyc}z.pres_b.0p25.f${fcsthr}.grib2" "tmpfile2b${fcsthr}"
 cat "tmpfile2${fcsthr}" "tmpfile2b${fcsthr}" > "tmpfile${fcsthr}"
-# shellcheck disable=SC2312
 ${WGRIB2} "tmpfile${fcsthr}" | grep -F -f "${PARMgfs}/product/gfs_awips_parmlist_g2" |
     ${WGRIB2} -i -grib masterfile "tmpfile${fcsthr}" && true
 export err=$?
@@ -91,7 +90,6 @@ if [[ ${err} -ne 0 ]]; then
 fi
 
 ${WGRIB2} masterfile -match ":PWAT:entire atmosphere" -grib gfs_pwat.grb
-# shellcheck disable=SC2312
 ${WGRIB2} masterfile | grep -v ":PWAT:entire atmosphere" | ${WGRIB2} -i -grib temp_gfs masterfile
 ##################################################################
 #  Process to change PWAT from level 200 to 10 (Entire Atmosphere)
@@ -156,7 +154,6 @@ for GRID in conus ak prico pac 003; do
     # NOTE: numparm is the total of fields in grib2_awpgfs_20km_conusf000 file
     ###########################################################################
     numparm=247
-    # shellcheck disable=SC2312
     numrec=$(${WGRIB2} "awps_file_f${fcsthr}_${GRID}" | wc -l)
 
     if [[ ${numrec} -lt ${numparm} ]]; then

dev/scripts/exgfs_atmos_fbwind.sh

@@ -38,7 +38,6 @@ for fhr3 in 006 012 024; do
     cpreq "${COMIN_ATMOS_GRIB_0p25}/gfs.${cycle}.pres_a.0p25.f${fhr3}.grib2" "tmp_pgrb2_0p25${fhr3}"
     cpreq "${COMIN_ATMOS_GRIB_0p25}/gfs.${cycle}.pres_b.0p25.f${fhr3}.grib2" "tmp_pgrb2b_0p25${fhr3}"
     cat "tmp_pgrb2_0p25${fhr3}" "tmp_pgrb2b_0p25${fhr3}" > "tmp0p25filef${fhr3}"
-    # shellcheck disable=SC2312
     ${WGRIB2} "tmp0p25filef${fhr3}" | grep -F -f "${PARMgfs}/product/gfs_fbwnd_parmlist_g2" |
         ${WGRIB2} -i -grib "tmpfilef${fhr3}" "tmp0p25filef${fhr3}"
     ${CNVGRIB} -g21 "tmpfilef${fhr3}" "gfs.t${cyc}z.grbf${fhr3}_grb1"

dev/scripts/exgfs_atmos_gempak_meta.sh

@@ -49,15 +49,13 @@ while [[ ${fhr} -le ${fhend} ]]; do
 
     if [[ ${do_all} -eq 1 ]]; then
         do_all=0
-        # shellcheck disable=SC2312
         awk '{print $1}' "${HOMEgfs}/gempak/fix/gfs_meta" | envsubst > "poescript"
     else
         #
         #    Do not try to grep out 12, it will grab the 12 from 126.
         #    This will work as long as we don't need 12 fhr metafiles
         #
         if [[ ${fhr} -ne 12 ]]; then
-            # shellcheck disable=SC2312
             grep "${fhr}" "${HOMEgfs}/gempak/fix/gfs_meta" | awk -F" [0-9]" '{print $1}' | envsubst > "poescript"
         fi
     fi

dev/scripts/exgfs_atmos_goes_nawips.sh

@@ -21,8 +21,7 @@ done
 NAGRIB_TABLE="${HOMEgfs}/gempak/fix/nagrib.tbl"
 NAGRIB="${GEMEXE}/nagrib2"
 
-# shellcheck disable=SC2312
-entry=$(grep "^${RUN2} " "${NAGRIB_TABLE}" | awk 'index($1,"#") != 1 {print $0}')
+entry=$(grep "^${RUN2} " "${NAGRIB_TABLE}" | awk 'index($1,"#") != 1 {print $0}' || echo "")
 
 if [[ "${entry}" != "" ]]; then
     cpyfil=$(echo "${entry}" | awk 'BEGIN {FS="|"} {print $2}')
@@ -61,7 +60,7 @@ startmsg
 
 ${NAGRIB} << EOF
 GBFILE   = grib${fhr3}
-INDXFL   = 
+INDXFL   =
 GDOUTF   = ${GEMGRD}
 PROJ     = ${proj}
 GRDAREA  = ${grdarea}
@@ -71,7 +70,7 @@ CPYFIL   = ${cpyfil}
 GAREA    = ${garea}
 OUTPUT   = ${output}
 GBTBLS   = ${gbtbls}
-GBDIAG   = 
+GBDIAG   =
 PDSEXT   = ${pdsext}
 l
 r

dev/scripts/exgfs_atmos_grib2_special_npoess.sh

@@ -110,7 +110,6 @@ for ((fhr = SHOUR; fhr <= FHOUR; fhr = fhr + FHINC)); do
     cpreq "${COMIN_ATMOS_GRIB_0p50}/gfs.t${cyc}z.pres_a.0p50.f${fhr3}.grib2" tmpfile2
     cpreq "${COMIN_ATMOS_GRIB_0p50}/gfs.t${cyc}z.pres_b.0p50.f${fhr3}.grib2" tmpfile2b
     cat tmpfile2 tmpfile2b > tmpfile
-    # shellcheck disable=SC2312
     ${WGRIB2} tmpfile | grep -F -f "${paramlist}" | ${WGRIB2} -i -grib pgb2file tmpfile && true
     export err=$?
     if [[ ${err} -ne 0 ]]; then

dev/scripts/exgfs_wave_post_pnt.sh

@@ -86,15 +86,13 @@ if [[ -f "${PARMgfs}/wave/wave_${NET}.buoys" ]]; then
     cpreq -f "${PARMgfs}/wave/wave_${NET}.buoys" buoy.loc.temp
     if [[ "${DOBNDPNT_WAV}" == "YES" ]]; then
         #only do boundary points
-        # shellcheck disable=SC2312
         sed -n '/^\$.*/!p' buoy.loc.temp | grep IBP > buoy.loc || {
             echo "WARNING: No boundary points found in buoy file ${PARMgfs}/wave/wave_${NET}.buoys"
             echo "         Ending job without doing anything."
             exit 0
         }
     else
         #exclude boundary points
-        # shellcheck disable=SC2312
         sed -n '/^\$.*/!p' buoy.loc.temp | grep -v IBP > buoy.loc
     fi
 fi
@@ -182,7 +180,6 @@ if [[ ${err} -ne 0 && ! -f buoy_log.ww3 ]]; then
 fi
 
 # Create new buoy_log.ww3
-# shellcheck disable=SC2312
 awk '{print $3}' buoy.loc | sed 's/'\''//g' > ibp_tags
 grep -F -f ibp_tags buoy_log.ww3 > buoy_log.tmp
 rm -f buoy_log.dat
@@ -199,7 +196,6 @@ fi
 
 # 1.f Data summary
 
-# shellcheck disable=SC2312
 cat << EOF
 
 Input files read and processed at : $(date)
@@ -219,11 +215,8 @@ EOF
 
 echo '   Making command file for wave post points '
 
-# shellcheck disable=SC2312
 grep -F -f ibp_tags buoy_log.dat | awk '{ print $2 }' > buoys
-# shellcheck disable=SC2312
 grep -F -f buoys buoy_log.ww3 | awk '{ print $1 }' > points
-# shellcheck disable=SC2312
 points=$(awk '{print $0 "\\n"}' points | tr -d '\n')
 rm -f buoys
 

dev/scripts/exgfs_wave_prdgen_bulls.sh

@@ -57,7 +57,6 @@ else
 fi
 
 # 1.b Output locations from bulletin files
-# shellcheck disable=SC2312
 Nb=$(find . -type f -name "*.cbull" -printf '.' | wc -c)
 echo ' '
 echo "   Number of bulletin files :   ${Nb}"
@@ -79,7 +78,6 @@ source awipsbull.data
 
 # 2.c Generate list of bulletins to process
 echo '   Generating buoy list ...'
-# shellcheck disable=SC2312
 bulls=$(sed -e 's/export b//g' -e 's/=/ /' awipsbull.data | grep -v "#" | awk '{print $1}')
 
 # 2.d Looping over buoys running formbul
@@ -89,7 +87,6 @@ echo
 for bull in ${bulls}; do
     fname="${RUN}.${bull}.cbull"
     oname="awipsbull.${bull}.${cycle}.${RUN}.wave"
-    # shellcheck disable=SC2312
     headr=$(grep "b${bull}=" awipsbull.data | sed 's/=/ /g' | awk '{ print $3}')
     echo "Processing ${bull} (${headr} ${oname}) ..."
 

dev/scripts/exgfs_wave_prdgen_gridded.sh

@@ -98,12 +98,9 @@ while [[ "${fhcnt}" -le "${FHMAX_WAV}" ]]; do
             echo "${nip} ${prepar} ${paridx} ${npart}"
             rm -f temp.grib2
             if [[ ${npart} -eq 0 ]]; then
-                #shellcheck disable=SC2312
                 ${WGRIB2} "${GRIBIN}" -s | grep ":${nip}" | "${WGRIB2}" -i "${GRIBIN}" -grib temp.grib2 > wgrib.out 2>&1
-                #shellcheck disable=SC2312
                 ${WGRIB2} temp.grib2 -append -grib "${GRIBOUT}"
             else
-                #shellcheck disable=SC2312
                 ${WGRIB2} "${GRIBIN}" -s | grep ":${prepar}" | grep "${paridx} in sequence" |
                     ${WGRIB2} -i "${GRIBIN}" -grib temp.grib2 > wgrib.out 2>&1
                 ${WGRIB2} temp.grib2 -append -grib "${GRIBOUT}"

dev/scripts/exglobal_atmos_analysis.sh

@@ -383,7 +383,6 @@ ${NLN} "${FIXgfs}/gsi/IASI_CLDDET.NL" IASI_CLDDET.NL
 #If using correlated error, link to the covariance files
 if [[ "${USE_CORRELATED_OBERRS}" == "YES" ]]; then
     if grep -q "Rcov" "${ANAVINFO}"; then
-        # shellcheck disable=SC2312
         mapfile -t covfile_array < <(find "${FIXgfs}/gsi/" -name "Rcov*")
         if ((${#covfile_array[@]} > 0)); then
             for covfile in "${covfile_array[@]}"; do
@@ -413,7 +412,6 @@ fi
 ##############################################################
 # CRTM Spectral and Transmittance coefficients
 mkdir -p crtm_coeffs
-# shellcheck disable=SC2312
 for file in $(awk '{if($1!~"!"){print $1}}' satinfo | sort | uniq); do
     if [[ ${file:0:4} == "hirs" ]]; then
         ${NLN} "${HIRS_FIX}/${file}.SpcCoeff.bin" "./crtm_coeffs/${file}.SpcCoeff.bin"
@@ -528,10 +526,9 @@ fi
 
 ##############################################################
 # Required bias guess files
-${NLN} "${GBIAS}" satbias_in
-${NLN} "${GBIASPC}" satbias_pc
-${NLN} "${GBIASAIR}" aircftbias_in
-${NLN} "${GRADSTAT}" radstat.gdas
+cpreq "${GBIAS}" satbias_in
+cpreq "${GBIASPC}" satbias_pc
+cpreq "${GBIASAIR}" aircftbias_in
 
 ##############################################################
 # Required model guess files
@@ -645,7 +642,6 @@ fi
 # If requested, link (and if tarred, de-tar obsinput.tar) into obs_input.* files
 if [[ "${USE_SELECT}" == "YES" ]]; then
     rm -f obs_input.*
-    # shellcheck disable=SC2312
     nl=$(file "${SELECT_OBS}" | cut -d: -f2 | grep -c tar)
     if [[ ${nl} -eq 1 ]]; then
         rm -f obsinput.tar
@@ -663,8 +659,8 @@ fi
 ##############################################################
 # If requested, copy and de-tar guess radstat file
 if [[ "${USE_RADSTAT}" == "YES" ]]; then
-    rm -f "${DATA}/unzip_radstat.sh"
-    cat > "${DATA}/unzip_radstat.sh" << EOF
+    rm -f "${DATA}/unzip_diag.sh"
+    cat > "${DATA}/unzip_diag.sh" << EOF
 #!/bin/bash
 diag_file=\$1
 diag_suffix=\$2
@@ -674,20 +670,21 @@ ${UNCOMPRESS} \$diag_file
 fnameges=\$(echo \$fname | sed 's/_ges//g')
 ${NMV} \$fname.\$fdate\$diag_suffix \$fnameges
 EOF
-    chmod 755 "${DATA}/unzip_radstat.sh"
+    chmod 755 "${DATA}/unzip_diag.sh"
 
     rm -f "${DATA}/cmdfile"
-    #shellcheck disable=SC2312
-    listdiag=$(tar -xvf radstat.gdas | cut -d' ' -f2 | grep _ges)
+    cpreq "${GRADSTAT}" radstat.tar
+    tar -xvf radstat.tar
+    listdiag=$(find ./ -path "./diag_*_ges.*" -type f)
     for type in ${listdiag}; do
-        diag_file=$(echo "${type}" | cut -d',' -f1)
-        echo "${DATA}/unzip_radstat.sh ${diag_file} ${DIAG_SUFFIX:-}.nc4" >> "${DATA}/cmdfile"
+        diag_file=$(basename "${type}")
+        echo "${DATA}/unzip_diag.sh ${diag_file} ${DIAG_SUFFIX:-}.nc4" >> "${DATA}/cmdfile"
     done
 
     "${USHgfs}/run_mpmd.sh" "${DATA}/cmdfile" && true
     export err=$?
     if [[ ${err} -ne 0 ]]; then
-        err_exit "Failed to unzip radstat.gdas file!"
+        err_exit "Failed to unzip diag file!"
     fi
 fi # if [[ $USE_RADSTAT == "YES" ]
 
@@ -860,7 +857,6 @@ cat fort.2* > "${GSISTAT}"
 
 # If requested, create obsinput tarball from obs_input.* files
 if [[ ${RUN_SELECT} == "YES" ]]; then
-    # shellcheck disable=SC2312
     echo "$(date) START tar obs_input" >&2
     if [[ -s obsinput.tar ]]; then
         rm -f obsinput.tar
@@ -871,7 +867,6 @@ if [[ ${RUN_SELECT} == "YES" ]]; then
     chmod 750 "${SELECT_OBS}"
     ${CHGRP_CMD} "${SELECT_OBS}"
     rm -f obsinput.tar
-    # shellcheck disable=SC2312
     echo "$(date) END tar obs_input" >&2
 fi
 
@@ -907,7 +902,6 @@ if [[ "${GENDIAG}" == "YES" ]]; then
     cpfs gsidiags.tar "${COMOUT_ATMOS_ANALYSIS}/${APREFIX}gsidiags${DIAG_SUFFIX:-}.tar"
 fi
 
-# shellcheck disable=SC2312
 echo "${rCDUMP} ${PDY}${cyc} atminc done at $(date)" > "${COMOUT_ATMOS_ANALYSIS}/${APREFIX}increment.done.txt"
 
 ################################################################################

dev/scripts/exglobal_atmos_analysis_calc.sh

@@ -177,7 +177,6 @@ if [[ "${DOGAUSFCANL}" == "YES" ]]; then
     fi
 fi
 
-# shellcheck disable=SC2312
 echo "${rCDUMP} ${PDY}${cyc} atmanl and sfcanl done at $(date)" > "${COMOUT_ATMOS_ANALYSIS}/${APREFIX}analysis.done.txt"
 
 ################################################################################

dev/scripts/exglobal_atmos_products.sh

@@ -58,7 +58,6 @@ for ((nset = 1; nset <= downset; nset++)); do
     paramlist="paramlist${grp}"
     parmfile="${!paramlist}"
 
-    # shellcheck disable=SC2312
     ${WGRIB2} "${MASTER_FILE}" | grep -F -f "${parmfile}" | ${WGRIB2} -i -grib "${tmpfile}" "${MASTER_FILE}" && true
     export err=$?
     if [[ ${err} -ne 0 ]]; then
@@ -68,7 +67,6 @@ for ((nset = 1; nset <= downset; nset++)); do
     # Number of processors available to process $nset
     nproc=${ntasks}
 
-    # shellcheck disable=SC2312
     ncount=$(${WGRIB2} "${tmpfile}" | wc -l)
     if [[ "${nproc}" -gt "${ncount}" ]]; then
         echo "WARNING: Total no. of available processors '${nproc}' exceeds no. of records '${ncount}' in ${tmpfile}"
@@ -90,7 +88,6 @@ for ((nset = 1; nset <= downset; nset++)); do
         # if final record is land, add next record icec
         # grep returns 1 if no match is found, so temporarily turn off exit on non-zero rc
         set +e
-        # shellcheck disable=SC2312
         ${WGRIB2} -d "${last}" "${tmpfile}" | grep -E -i "ugrd|ustm|uflx|u-gwd|land|maxuw"
         rc=$?
         set_strict