Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
9f797bf
add function to unset_strict and pipefail
aerorahul Dec 11, 2025
169825f
remove SC2312 from ush/
aerorahul Dec 11, 2025
a73ccfe
remove SC2312 from dev/scripts
aerorahul Dec 11, 2025
17a6d1f
remove SC2312 from dev gempak and sorc
aerorahul Dec 11, 2025
63e3f39
was too lazy to take Walters change and work it, until now
aerorahul Dec 12, 2025
41b4aae
Merge branch 'develop' into feature/pipedown
aerorahul Dec 12, 2025
8faea41
load grib_util modulefile on wcoss in the gw_gsi.wcoss2. RADSTAT was …
aerorahul Dec 12, 2025
e12b370
Merge branch 'develop' into feature/pipedown
aerorahul Dec 15, 2025
05f6a25
pipefail exposed an error when grepped line does not exist
aerorahul Dec 15, 2025
c1dc05a
Apply suggestions from code review
aerorahul Dec 15, 2025
9870b75
Merge branch 'develop' into feature/pipedown
DavidHuber-NOAA Dec 15, 2025
c6a4f63
revert changes to exglobal_diag.sh and add files to gitignore
aerorahul Dec 15, 2025
0c75523
Merge branch 'develop' into feature/pipedown
DavidHuber-NOAA Dec 16, 2025
f869fda
Merge branch 'develop' into feature/pipedown
aerorahul Dec 16, 2025
906deb7
fix the error from reverting
aerorahul Dec 16, 2025
70a0803
Merge branch 'develop' into feature/pipedown
aerorahul Dec 16, 2025
0d63f6d
disable archiving pres_b gcafs products since they are not being prod…
aerorahul Dec 16, 2025
8588c77
replace set +eu with unset_strict
aerorahul Dec 17, 2025
135683d
Merge branch 'develop' into feature/pipedown
aerorahul Dec 17, 2025
9bdc6a6
add missing shellcheck
aerorahul Dec 17, 2025
a6f5038
Merge branch 'develop' into feature/pipedown
aerorahul Dec 18, 2025
98543a5
ignore parm/chem/nexus -- it is being created in link_workflow.sh and…
aerorahul Dec 18, 2025
e04fa69
Merge branch 'develop' into feature/pipedown
aerorahul Dec 18, 2025
d2c0a35
revert the change to master_gcafs.yaml.j2 to develop
aerorahul Dec 18, 2025
17aa1ff
Merge branch 'develop' into feature/pipedown
aerorahul Dec 18, 2025
1514b83
Merge branch 'develop' into feature/pipedown
aerorahul Dec 19, 2025
09a6f27
comment out the archiving of pres_b gcafs files since #4347 was reverted
aerorahul Dec 19, 2025
b067f4a
Merge branch 'develop' into feature/pipedown
DavidHuber-NOAA Dec 19, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ parm/post/post_tag_gfs128
parm/post/gfs
parm/post/gefs
parm/post/sfs
parm/post/gcafs
parm/post/ocean.csv
parm/post/ocean_gefs.csv
parm/post/ocean_gfs.csv
Expand All @@ -96,9 +97,11 @@ parm/ufs/MOM6_data_table.IN
parm/ufs/ice_in.IN
parm/ufs/ufs.configure.*.IN
parm/ufs/post_itag_gfs
parm/ufs/post_itag_gcafs
parm/ufs/ww3_shel.nml.IN
parm/ufs/global_control.nml.IN
parm/regrid_sfc
parm/chem/nexus

# Ignore sorc and logs folders from externals
#--------------------------------------------
Expand Down
3 changes: 3 additions & 0 deletions .shellcheckrc
Original file line number Diff line number Diff line change
Expand Up @@ -17,3 +17,6 @@ disable=SC2174

# Disable warning of functions in test statements
disable=SC2310

# Disable masking of return codes from chained commands. Use `set -o pipefail` to trap them.
disable=SC2312
1 change: 0 additions & 1 deletion dev/ci/scripts/run_check_ci.sh
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,6 @@ while true; do

# Get job statistics
echo "Gather Rocoto statistics"
# shellcheck disable=SC2312 # We want to use the exit code of the command
full_state=$("${HOMEgfs}/dev/ci/scripts/utils/rocotostat.py" -w "${xml}" -d "${db}" -v)
error_stat=$?

Expand Down
1 change: 0 additions & 1 deletion dev/ci/scripts/utils/ci_utils.sh
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,6 @@ function delete_dataroot() {
_runtests="${1}"
_pslot="${2}"

# shellcheck disable=SC2312
eval "$(
PDY=0 cyc=0 source "${_runtests}/EXPDIR/${_pslot}/config.base" >&/dev/null
echo _dataroot="${STMP}/RUNDIRS/${_pslot}"
Expand Down
5 changes: 2 additions & 3 deletions dev/job_cards/rocoto/prep.sh
Original file line number Diff line number Diff line change
Expand Up @@ -74,11 +74,9 @@ if [[ ! -s "${COMINobsproc_PREV}/${GDUMP}.t${gcyc}z.updated.status.tm00.bufr_d"
fi
fi
# exception handling to ensure no dead link
# shellcheck disable=SC2312
if [[ $(find "${COMOUT_OBS}" -xtype l | wc -l) -ge 1 ]]; then
exit 9
fi
# shellcheck disable=SC2312
if [[ $(find "${COMINobsproc_PREV}" -xtype l | wc -l) -ge 1 ]]; then
exit 9
fi
Expand Down Expand Up @@ -155,9 +153,10 @@ export COMSP=${COMSP:-"${COMIN_OBS}/${RUN_local}.t${cyc}z."}
# Create or Copy prepbufr, prepbufr.acft_profiles, nsstbufr files
# Do not fail on external errors
if [[ ${MAKE_PREPBUFR:-"YES"} == "YES" ]]; then
set +eu
unset_strict
"${HOMEobsproc}/jobs/JOBSPROC_GLOBAL_PREP" && true
export err=$?
set_strict
if [[ ${err} -ne 0 ]]; then
err_exit "JOBSPROC_GLOBAL_PREP job failed, ABORT!"
fi
Expand Down
5 changes: 2 additions & 3 deletions dev/jobs/JGFS_ATMOS_CYCLONE_GENESIS
Original file line number Diff line number Diff line change
Expand Up @@ -34,15 +34,14 @@ export JYYYY=${PDY:0:4}
##############################################

# Do not fail on errors or unassigned variables in external code
set +eu
unset_strict
"${SCRIPTens_tracker}/exgfs_tc_genesis.sh" && true
export err=$?
set_strict
if [[ ${err} -ne 0 ]]; then
err_exit
fi

set_strict

##############################################
# Final processing
##############################################
Expand Down
7 changes: 5 additions & 2 deletions dev/jobs/JGFS_ATMOS_VERIFICATION
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,16 @@ done
if [[ "${RUN_GRID2GRID_STEP1}" == "YES" || "${RUN_GRID2OBS_STEP1}" == "YES" || "${RUN_PRECIP_STEP1}" == "YES" ]]; then
# Override the -e in VERIF_GLOBALSH's shebang and un-export SHELLOPTS
# TODO: clean up the verif-global script so it does not raise false-positive errors
set +eu
unset_strict
export -n SHELLOPTS
bash -x "${VERIF_GLOBALSH}"
err=$?
set_strict
if [[ ${err} -ne 0 ]]; then
exit "${err}"
fi
fi

if [[ ${KEEPDATA:-"NO"} == "NO" ]]; then rm -rf "${DATAROOT}"; fi # TODO: This should be $DATA
if [[ ${KEEPDATA:-"NO"} == "NO" ]]; then
rm -rf "${DATAROOT}" # TODO: This should be $DATA
fi
11 changes: 3 additions & 8 deletions dev/jobs/JGLOBAL_ARCHIVE_TARS
Original file line number Diff line number Diff line change
Expand Up @@ -80,17 +80,15 @@ fi
# Run archive script
###############################################################

set +eu
# Do not export shell opts to the bash scripts in the htar/hsi wrappers
export -n SHELLOPTS
unset_strict
${GLOBALARCHIVESH:-${SCRgfs}/exglobal_archive_tars.py}
export err=$?
set +x
set_strict
if [[ ${err} -ne 0 ]]; then
err_exit "failed to archive the COM structure"
fi
set_trace
set_strict

##############################################
# End JOB SPECIFIC work
Expand All @@ -106,10 +104,7 @@ fi
##########################################
# Remove the Temporary working directory
##########################################
cd "${DATAROOT}" || (
echo "${DATAROOT} does not exist. ABORT!"
exit 1
)
cd "${DATAROOT}" || true
if [[ "${KEEPDATA}" == "NO" ]]; then
rm -rf "${DATA}"
fi
Expand Down
8 changes: 1 addition & 7 deletions dev/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE
Original file line number Diff line number Diff line change
Expand Up @@ -45,13 +45,7 @@ fi
##########################################
# Remove the Temporary working directory
##########################################
cd "${DATAROOT}" && true
export err=$?
set +e
if [[ ${err} -ne 0 ]]; then
err_exit "${DATAROOT} does not exist, ABORT!"
fi
set_trace
cd "${DATAROOT}" || true
if [[ "${KEEPDATA}" == "NO" ]]; then
rm -rf "${DATA}"
fi
Expand Down
1 change: 0 additions & 1 deletion dev/jobs/JGLOBAL_ATMOS_ANALYSIS_CALC_FV3JEDI
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

# Ignore possible spelling error (nothing is misspelled)
# shellcheck disable=SC2153
# shellcheck disable=SC2312

source "${HOMEgfs}/ush/jjob_header.sh" -e "analcalc_fv3jedi" -c "base analcalc_fv3jedi"

Expand Down
11 changes: 3 additions & 8 deletions dev/jobs/JGLOBAL_ENKF_ARCHIVE_TARS
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,14 @@ mkdir -p "${COMIN_CONF}"
###############################################################

# Calls an external bash command; do not fail on unassigned/error
set +eu
export -n SHELLOPTS
unset_strict
"${SCRgfs}/exglobal_enkf_earc_tars.py"
export err=$?
set +x
set_strict
if [[ ${err} -ne 0 ]]; then
err_exit "Failed to archive the ensemble COM structure"
fi
set_trace
set_strict

###############################################################

Expand All @@ -50,10 +48,7 @@ fi
##########################################
# Remove the Temporary working directory
##########################################
cd "${DATAROOT}" || (
echo "${DATAROOT} does not exist. ABORT!"
exit 1
)
cd "${DATAROOT}" || true
if [[ "${KEEPDATA}" == "NO" ]]; then
rm -rf "${DATA}"
fi
Expand Down
1 change: 0 additions & 1 deletion dev/parm/config/gefs/config.resources.AZUREPW
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
export is_exclusive="True"
unset memory

# shellcheck disable=SC2312
for mem_var in $(env | grep '^memory_' | cut -d= -f1); do
unset "${mem_var}"
done
1 change: 0 additions & 1 deletion dev/parm/config/gefs/config.resources.GOOGLEPW
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
export is_exclusive="True"
unset memory

# shellcheck disable=SC2312
for mem_var in $(env | grep '^memory_' | cut -d= -f1); do
unset "${mem_var}"
done
1 change: 0 additions & 1 deletion dev/parm/config/gfs/config.ecen
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ source "${EXPDIR}/config.resources" ecen
# below queries IAUFHRS_ENKF to determine NECENGRP
export NECENGRP=1
if [[ ${DOIAU_ENKF} = "YES" ]]; then
# shellcheck disable=SC2312
ngrps=$(grep -o ',' <<<"${IAUFHRS_ENKF}" | grep -c .)
((ngrps++))
export NECENGRP=${ngrps}
Expand Down
1 change: 0 additions & 1 deletion dev/parm/config/gfs/config.resources.AZUREPW
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
export is_exclusive="True"
unset memory

# shellcheck disable=SC2312
for mem_var in $(env | grep '^memory_' | cut -d= -f1); do
unset "${mem_var}"
done
1 change: 0 additions & 1 deletion dev/parm/config/gfs/config.resources.GOOGLEPW
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
export is_exclusive="True"
unset memory

# shellcheck disable=SC2312
for mem_var in $(env | grep '^memory_' | cut -d= -f1); do
unset "${mem_var}"
done
4 changes: 1 addition & 3 deletions dev/scripts/exgdas_atmos_verfrad.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
################################################################################

# Do not exit on errors so that restricted data can be protected
set +eu
unset_strict

if [[ ! -s "${radstat}" || ! -s "${biascr}" ]]; then
export err=1
Expand Down Expand Up @@ -102,7 +102,6 @@ for type in ${SATYPE}; do
netcdf=1
fi

# shellcheck disable=SC2312
if [[ $(find . -maxdepth 1 -type f -name "diag_${type}_ges.${PDY}${cyc}*.${Z}" | wc -l) -gt 0 ]]; then
mv "diag_${type}_ges.${PDY}${cyc}"*".${Z}" "${type}.${Z}"
${UNCOMPRESS} "./${type}.${Z}"
Expand All @@ -111,7 +110,6 @@ for type in ${SATYPE}; do
fi

if [[ ${USE_ANL} -eq 1 ]]; then
# shellcheck disable=SC2312
file_count=$(find . -maxdepth 1 -type f -name "diag_${type}_anl.${PDY}${cyc}*.${Z}" | wc -l)
if [[ ${file_count} -gt 0 ]]; then
mv "diag_${type}_anl.${PDY}${cyc}"*".${Z}" "${type}_anl.${Z}"
Expand Down
3 changes: 0 additions & 3 deletions dev/scripts/exgfs_atmos_awips_20km_1p0deg.sh
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,6 @@ export opt28=' -new_grid_interpolation budget -fi '
cpreq "${COMIN_ATMOS_GRIB_0p25}/gfs.t${cyc}z.pres_a.0p25.f${fcsthr}.grib2" "tmpfile2${fcsthr}"
cpreq "${COMIN_ATMOS_GRIB_0p25}/gfs.t${cyc}z.pres_b.0p25.f${fcsthr}.grib2" "tmpfile2b${fcsthr}"
cat "tmpfile2${fcsthr}" "tmpfile2b${fcsthr}" > "tmpfile${fcsthr}"
# shellcheck disable=SC2312
${WGRIB2} "tmpfile${fcsthr}" | grep -F -f "${PARMgfs}/product/gfs_awips_parmlist_g2" |
${WGRIB2} -i -grib masterfile "tmpfile${fcsthr}" && true
export err=$?
Expand All @@ -91,7 +90,6 @@ if [[ ${err} -ne 0 ]]; then
fi

${WGRIB2} masterfile -match ":PWAT:entire atmosphere" -grib gfs_pwat.grb
# shellcheck disable=SC2312
${WGRIB2} masterfile | grep -v ":PWAT:entire atmosphere" | ${WGRIB2} -i -grib temp_gfs masterfile
##################################################################
# Process to change PWAT from level 200 to 10 (Entire Atmosphere)
Expand Down Expand Up @@ -156,7 +154,6 @@ for GRID in conus ak prico pac 003; do
# NOTE: numparm is the total of fields in grib2_awpgfs_20km_conusf000 file
###########################################################################
numparm=247
# shellcheck disable=SC2312
numrec=$(${WGRIB2} "awps_file_f${fcsthr}_${GRID}" | wc -l)

if [[ ${numrec} -lt ${numparm} ]]; then
Expand Down
1 change: 0 additions & 1 deletion dev/scripts/exgfs_atmos_fbwind.sh
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@ for fhr3 in 006 012 024; do
cpreq "${COMIN_ATMOS_GRIB_0p25}/gfs.${cycle}.pres_a.0p25.f${fhr3}.grib2" "tmp_pgrb2_0p25${fhr3}"
cpreq "${COMIN_ATMOS_GRIB_0p25}/gfs.${cycle}.pres_b.0p25.f${fhr3}.grib2" "tmp_pgrb2b_0p25${fhr3}"
cat "tmp_pgrb2_0p25${fhr3}" "tmp_pgrb2b_0p25${fhr3}" > "tmp0p25filef${fhr3}"
# shellcheck disable=SC2312
${WGRIB2} "tmp0p25filef${fhr3}" | grep -F -f "${PARMgfs}/product/gfs_fbwnd_parmlist_g2" |
${WGRIB2} -i -grib "tmpfilef${fhr3}" "tmp0p25filef${fhr3}"
${CNVGRIB} -g21 "tmpfilef${fhr3}" "gfs.t${cyc}z.grbf${fhr3}_grb1"
Expand Down
2 changes: 0 additions & 2 deletions dev/scripts/exgfs_atmos_gempak_meta.sh
Original file line number Diff line number Diff line change
Expand Up @@ -49,15 +49,13 @@ while [[ ${fhr} -le ${fhend} ]]; do

if [[ ${do_all} -eq 1 ]]; then
do_all=0
# shellcheck disable=SC2312
awk '{print $1}' "${HOMEgfs}/gempak/fix/gfs_meta" | envsubst > "poescript"
else
#
# Do not try to grep out 12, it will grab the 12 from 126.
# This will work as long as we don't need 12 fhr metafiles
#
if [[ ${fhr} -ne 12 ]]; then
# shellcheck disable=SC2312
grep "${fhr}" "${HOMEgfs}/gempak/fix/gfs_meta" | awk -F" [0-9]" '{print $1}' | envsubst > "poescript"
fi
fi
Expand Down
7 changes: 3 additions & 4 deletions dev/scripts/exgfs_atmos_goes_nawips.sh
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,7 @@ done
NAGRIB_TABLE="${HOMEgfs}/gempak/fix/nagrib.tbl"
NAGRIB="${GEMEXE}/nagrib2"

# shellcheck disable=SC2312
entry=$(grep "^${RUN2} " "${NAGRIB_TABLE}" | awk 'index($1,"#") != 1 {print $0}')
entry=$(grep "^${RUN2} " "${NAGRIB_TABLE}" | awk 'index($1,"#") != 1 {print $0}' || echo "")

if [[ "${entry}" != "" ]]; then
cpyfil=$(echo "${entry}" | awk 'BEGIN {FS="|"} {print $2}')
Expand Down Expand Up @@ -61,7 +60,7 @@ startmsg

${NAGRIB} << EOF
GBFILE = grib${fhr3}
INDXFL =
INDXFL =
GDOUTF = ${GEMGRD}
PROJ = ${proj}
GRDAREA = ${grdarea}
Expand All @@ -71,7 +70,7 @@ CPYFIL = ${cpyfil}
GAREA = ${garea}
OUTPUT = ${output}
GBTBLS = ${gbtbls}
GBDIAG =
GBDIAG =
PDSEXT = ${pdsext}
l
r
Expand Down
1 change: 0 additions & 1 deletion dev/scripts/exgfs_atmos_grib2_special_npoess.sh
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,6 @@ for ((fhr = SHOUR; fhr <= FHOUR; fhr = fhr + FHINC)); do
cpreq "${COMIN_ATMOS_GRIB_0p50}/gfs.t${cyc}z.pres_a.0p50.f${fhr3}.grib2" tmpfile2
cpreq "${COMIN_ATMOS_GRIB_0p50}/gfs.t${cyc}z.pres_b.0p50.f${fhr3}.grib2" tmpfile2b
cat tmpfile2 tmpfile2b > tmpfile
# shellcheck disable=SC2312
${WGRIB2} tmpfile | grep -F -f "${paramlist}" | ${WGRIB2} -i -grib pgb2file tmpfile && true
export err=$?
if [[ ${err} -ne 0 ]]; then
Expand Down
7 changes: 0 additions & 7 deletions dev/scripts/exgfs_wave_post_pnt.sh
Original file line number Diff line number Diff line change
Expand Up @@ -86,15 +86,13 @@ if [[ -f "${PARMgfs}/wave/wave_${NET}.buoys" ]]; then
cpreq -f "${PARMgfs}/wave/wave_${NET}.buoys" buoy.loc.temp
if [[ "${DOBNDPNT_WAV}" == "YES" ]]; then
#only do boundary points
# shellcheck disable=SC2312
sed -n '/^\$.*/!p' buoy.loc.temp | grep IBP > buoy.loc || {
echo "WARNING: No boundary points found in buoy file ${PARMgfs}/wave/wave_${NET}.buoys"
echo " Ending job without doing anything."
exit 0
}
else
#exclude boundary points
# shellcheck disable=SC2312
sed -n '/^\$.*/!p' buoy.loc.temp | grep -v IBP > buoy.loc
fi
fi
Expand Down Expand Up @@ -182,7 +180,6 @@ if [[ ${err} -ne 0 && ! -f buoy_log.ww3 ]]; then
fi

# Create new buoy_log.ww3
# shellcheck disable=SC2312
awk '{print $3}' buoy.loc | sed 's/'\''//g' > ibp_tags
grep -F -f ibp_tags buoy_log.ww3 > buoy_log.tmp
rm -f buoy_log.dat
Expand All @@ -199,7 +196,6 @@ fi

# 1.f Data summary

# shellcheck disable=SC2312
cat << EOF

Input files read and processed at : $(date)
Expand All @@ -219,11 +215,8 @@ EOF

echo ' Making command file for wave post points '

# shellcheck disable=SC2312
grep -F -f ibp_tags buoy_log.dat | awk '{ print $2 }' > buoys
# shellcheck disable=SC2312
grep -F -f buoys buoy_log.ww3 | awk '{ print $1 }' > points
# shellcheck disable=SC2312
points=$(awk '{print $0 "\\n"}' points | tr -d '\n')
rm -f buoys

Expand Down
Loading
Loading