From 1b679ec816db98a324932f078b09bfdbac59f159 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Wed, 12 Nov 2025 16:00:51 -0500 Subject: [PATCH 01/15] save staging yaml to DATA for looking back --- ush/python/pygfs/task/stage_ic.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ush/python/pygfs/task/stage_ic.py b/ush/python/pygfs/task/stage_ic.py index d1e95b8d120..9b4082c4422 100644 --- a/ush/python/pygfs/task/stage_ic.py +++ b/ush/python/pygfs/task/stage_ic.py @@ -52,6 +52,11 @@ def execute_stage(self, stage_dict: Dict[str, Any]) -> None: # Parse staging yaml to get list of files to stage stage_set = parse_j2yaml(self.task_config.STAGE_IC_YAML_TMPL, stage_dict, allow_missing=False) + logger.info(stage_set) + + # Write stage_set to a file in DATA for debugging + stage_set.save("./stage.yaml") + logger.debug("Staging yaml written to ./stage.yaml for debugging purposes.") # stage files to ROTDIR for key in stage_set.keys(): From aa1e683731fa7edf47c8b1df545d4506f0c8c8b9 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 20 Nov 2025 15:30:08 -0500 Subject: [PATCH 02/15] wip --- dev/parm/config/gfs/config.resources | 23 ++-- env/GAEAC6.env | 6 +- jobs/JGDAS_ATMOS_ANALYSIS_DIAG | 21 +-- jobs/JGDAS_ENKF_DIAG | 99 +++++++------- jobs/JGDAS_ENKF_SELECT_OBS | 10 +- scripts/exgdas_enkf_update.sh | 136 +++++-------------- scripts/exglobal_atmos_analysis.sh | 129 +++++------------- scripts/exglobal_diag.sh | 191 ++++++++++----------------- 8 files changed, 218 insertions(+), 397 deletions(-) diff --git a/dev/parm/config/gfs/config.resources b/dev/parm/config/gfs/config.resources index 2f3829622eb..c5ca9e38366 100644 --- a/dev/parm/config/gfs/config.resources +++ b/dev/parm/config/gfs/config.resources @@ -773,12 +773,19 @@ case ${step} in fi ;; - "analdiag") + "analdiag" | "ediag") walltime="00:15:00" ntasks=96 # Should be at least twice ediag's tasks + memory="48GB" + #if [[ "${step}" == "ediag" ]]; then + # ntasks=48 + #fi threads_per_task=1 + if [[ ${ntasks} -gt ${max_tasks_per_node} ]]; then + ntasks=${max_tasks_per_node} + export is_exclusive=True + fi tasks_per_node=$(( max_tasks_per_node / threads_per_task )) - memory="48GB" ;; "anlstat") @@ -868,7 +875,7 @@ case ${step} in MEDTHREADS=${nthreads_mediator:-1} MEDPETS=${MEDPETS:-${FV3PETS}} (( MEDPETS = FV3PETS )) - (( MEDTHREADS = FV3THREADS )) + (( MEDTHREADS = FV3THREADS )) (( "${MEDPETS}" > 4800 )) && MEDPETS=4800 export MEDPETS MEDTHREADS echo "MEDIATOR using (threads, PETS) = (${MEDTHREADS}, ${MEDPETS})" @@ -994,7 +1001,7 @@ case ${step} in ;; "upp") - #ntasks cannot be greater than $CASE_HIST + #ntasks cannot be greater than $CASE_HIST #define resources based on $CASE_HIST for upp case "${CASE_HIST}" in "C48" | "C96") @@ -1270,14 +1277,6 @@ case ${step} in fi ;; - "ediag") - walltime="00:15:00" - ntasks=48 - threads_per_task=1 - tasks_per_node=$(( max_tasks_per_node / threads_per_task )) - memory="30GB" - ;; - "eupd") walltime="00:45:00" case "${CASE}" in diff --git a/env/GAEAC6.env b/env/GAEAC6.env index b37572ba7cf..0e9913a01d2 100755 --- a/env/GAEAC6.env +++ b/env/GAEAC6.env @@ -203,7 +203,11 @@ case ${step} in ntasks_gausfcanl=${ntasks_gausfcanl:-1} export APRUN_GAUSFCANL="${launcher} -n ${ntasks_gausfcanl} --cpus-per-task=${NTHREADS_GAUSFCANL}" ;; - "offlineanl") + + "analdiag" | "ediag") + export USE_CFP="YES" + ;; + "offlineanl") export NTHREADS_CHGRES=${threads_per_task_chgres:-12} if [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]]; then diff --git a/jobs/JGDAS_ATMOS_ANALYSIS_DIAG b/jobs/JGDAS_ATMOS_ANALYSIS_DIAG index ad83614bff8..553a58e1160 100755 --- a/jobs/JGDAS_ATMOS_ANALYSIS_DIAG +++ b/jobs/JGDAS_ATMOS_ANALYSIS_DIAG @@ -1,27 +1,28 @@ #! /usr/bin/env bash -source "${HOMEgfs}/ush/jjob_header.sh" -e "anal" -c "base anal analdiag" +source "${HOMEgfs}/ush/jjob_header.sh" -e "analdiag" -c "base anal analdiag" ############################################## # Set variables used in the script ############################################## -export DO_CALC_ANALYSIS=${DO_CALC_ANALYSIS:-"YES"} +#export DO_CALC_ANALYSIS=${DO_CALC_ANALYSIS:-"YES"} ############################################## # Begin JOB SPECIFIC work ############################################## # shellcheck disable=SC2153 -GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") -export gPDY=${GDATE:0:8} -export gcyc=${GDATE:8:2} -export GDUMP="gdas" -export GDUMP_ENS="enkf${GDUMP}" +#GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") +#export gPDY=${GDATE:0:8} +#export gcyc=${GDATE:8:2} +#export GDUMP="gdas" +#export GDUMP_ENS="enkf${GDUMP}" -export OPREFIX="${RUN/enkf/}.t${cyc}z." -export GPREFIX="${GDUMP}.t${gcyc}z." -export APREFIX="${RUN}.t${cyc}z." +#export OPREFIX="${RUN/enkf/}.t${cyc}z." +#export GPREFIX="${GDUMP}.t${gcyc}z." +#export APREFIX="${RUN}.t${cyc}z." YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \ COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL mkdir -m 775 -p "${COMOUT_ATMOS_ANALYSIS}" diff --git a/jobs/JGDAS_ENKF_DIAG b/jobs/JGDAS_ENKF_DIAG index 70567283f2f..fa494dfb847 100755 --- a/jobs/JGDAS_ENKF_DIAG +++ b/jobs/JGDAS_ENKF_DIAG @@ -1,6 +1,6 @@ #! /usr/bin/env bash -source "${HOMEgfs}/ush/jjob_header.sh" -e "eobs" -c "base anal eobs analdiag ediag" +source "${HOMEgfs}/ush/jjob_header.sh" -e "ediag" -c "base anal eobs analdiag ediag" ############################################## # Set variables used in the script @@ -10,83 +10,76 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "eobs" -c "base anal eobs analdiag edi # Begin JOB SPECIFIC work ############################################## # shellcheck disable=SC2153 -GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") -export gPDY=${GDATE:0:8} -export gcyc=${GDATE:8:2} -export GDUMP="gdas" -export GDUMP_ENS="enkf${GDUMP}" +#GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") +#export gPDY=${GDATE:0:8} +#export gcyc=${GDATE:8:2} +#export GDUMP="gdas" +#export GDUMP_ENS="enkf${GDUMP}" export CASE=${CASE_ENS} -export OPREFIX="${RUN/enkf/}.t${cyc}z." +#export OPREFIX="${RUN/enkf/}.t${cyc}z." export APREFIX="${RUN}.t${cyc}z." -export GPREFIX="${GDUMP_ENS}.t${gcyc}z." -GPREFIX_DET="${GDUMP}.t${gcyc}z." +#export GPREFIX="${GDUMP_ENS}.t${gcyc}z." +#GPREFIX_DET="${GDUMP}.t${gcyc}z." -RUN=${RUN/enkf/} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ - COMIN_OBS:COM_OBS_TMPL +#RUN=${RUN/enkf/} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ +# COMIN_OBS:COM_OBS_TMPL MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \ COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL -RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ - COMIN_OBS_PREV:COM_OBS_TMPL \ - COMIN_ATMOS_ANALYSIS_DET_PREV:COM_ATMOS_ANALYSIS_TMPL +#RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ +# COMIN_OBS_PREV:COM_OBS_TMPL \ +# COMIN_ATMOS_ANALYSIS_DET_PREV:COM_ATMOS_ANALYSIS_TMPL -MEMDIR="ensstat" RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ - COMIN_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL +#MEMDIR="ensstat" RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ +# COMIN_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL -export ATMGES_ENSMEAN="${COMIN_ATMOS_HISTORY_PREV}/${GPREFIX}ensmean.atm.f006.nc" -if [[ ! -f "${ATMGES_ENSMEAN}" ]]; then - export err=1 - err_exit "FILE MISSING: ATMGES_ENSMEAN == ${ATMGES_ENSMEAN}" -fi +#export ATMGES_ENSMEAN="${COMIN_ATMOS_HISTORY_PREV}/${GPREFIX}ensmean.atm.f006.nc" +#if [[ ! -f "${ATMGES_ENSMEAN}" ]]; then +# export err=1 +# err_exit "FILE MISSING: ATMGES_ENSMEAN == ${ATMGES_ENSMEAN}" +#fi # Link observational data -export PREPQC="${COMIN_OBS}/${OPREFIX}prepbufr" -if [[ ! -f ${PREPQC} ]]; then - echo "WARNING: Global PREPBUFR FILE ${PREPQC} MISSING" -fi -export TCVITL="${COMIN_OBS}/${OPREFIX}syndata.tcvitals.tm00" -if [[ ${DONST} == "YES" ]]; then - export NSSTBF="${COMIN_OBS}/${OPREFIX}nsstbufr" -fi -export PREPQCPF="${COMIN_OBS}/${OPREFIX}prepbufr.acft_profiles" +#export PREPQC="${COMIN_OBS}/${OPREFIX}prepbufr" +#if [[ ! -f ${PREPQC} ]]; then +# echo "WARNING: Global PREPBUFR FILE ${PREPQC} MISSING" +#fi +#export TCVITL="${COMIN_OBS}/${OPREFIX}syndata.tcvitals.tm00" +#if [[ ${DONST} == "YES" ]]; then +# export NSSTBF="${COMIN_OBS}/${OPREFIX}nsstbufr" +#fi +#export PREPQCPF="${COMIN_OBS}/${OPREFIX}prepbufr.acft_profiles" # Guess Bias correction coefficients related to control -export GBIAS=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias.txt -export GBIASPC=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_pc.txt -export GBIASAIR=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_air.txt -export GRADSTAT=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}radstat.tar +#export GBIAS=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias.txt +#export GBIASPC=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_pc.txt +#export GBIASAIR=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_air.txt +#export GRADSTAT=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}radstat.tar # Bias correction coefficients related to ensemble mean -export ABIAS="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias.txt" -export ABIASPC="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_pc.txt" -export ABIASAIR="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_air.txt" -export ABIASe="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_int.txt" +#export ABIAS="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias.txt" +#export ABIASPC="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_pc.txt" +#export ABIASAIR="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_air.txt" +#export ABIASe="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_int.txt" # Diagnostics related to ensemble mean -export GSISTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}gsistat.ensmean.tar" -export CNVSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}cnvstat.ensmean.tar" -export OZNSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}oznstat.ensmean.tar" -export RADSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}radstat.ensmean.tar" +export CNVSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}cnvstat_ensmean.tar" +export OZNSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}oznstat_ensmean.tar" +export RADSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}radstat_ensmean.tar" # Select observations based on ensemble mean -export RUN_SELECT="YES" -export USE_SELECT="NO" -export SELECT_OBS="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}obsinput.ensmean" +#export RUN_SELECT="YES" +#export USE_SELECT="NO" +#export SELECT_OBS="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}obsinput.ensmean" export DIAG_SUFFIX="_ensmean" export DIAG_COMPRESS="NO" # GSI namelist options specific to eobs -export SETUP_INVOBS="passive_bc=.false.,${SETUP_INVOBS}" - -# Ensure clean stat tarballs for ensemble mean -for fstat in ${CNVSTAT} ${OZNSTAT} ${RADSTAT}; do - if [[ -f "${fstat}" ]]; then - rm -f "${fstat}" - fi -done +#export SETUP_INVOBS="passive_bc=.false.,${SETUP_INVOBS}" ############################################################### # Run relevant script diff --git a/jobs/JGDAS_ENKF_SELECT_OBS b/jobs/JGDAS_ENKF_SELECT_OBS index ab3a426808e..2f2a8c555ea 100755 --- a/jobs/JGDAS_ENKF_SELECT_OBS +++ b/jobs/JGDAS_ENKF_SELECT_OBS @@ -84,15 +84,15 @@ export ABIASAIR="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_air.txt" export ABIASe="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_int.txt" # Diagnostics related to ensemble mean -export GSISTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}gsistat.ensmean.tar" -export CNVSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}cnvstat.ensmean.tar" -export OZNSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}oznstat.ensmean.tar" -export RADSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}radstat.ensmean.tar" +export GSISTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}gsistat_ensmean.tar" +export CNVSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}cnvstat_ensmean.tar" +export OZNSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}oznstat_ensmean.tar" +export RADSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}radstat_ensmean.tar" # Select observations based on ensemble mean export RUN_SELECT="YES" export USE_SELECT="NO" -export SELECT_OBS="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}obsinput.ensmean" +export SELECT_OBS="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}obsinput_ensmean.tar" export DIAG_SUFFIX="_ensmean" diff --git a/scripts/exgdas_enkf_update.sh b/scripts/exgdas_enkf_update.sh index a856e9c8f66..03ec4d43a54 100755 --- a/scripts/exgdas_enkf_update.sh +++ b/scripts/exgdas_enkf_update.sh @@ -18,17 +18,10 @@ ################################################################################ # Directories. -pwd=$(pwd) +cd "${DATA}" || exit 1 # Utilities NCLEN=${NCLEN:-${USHgfs}/getncdimlen} -USE_CFP=${USE_CFP:-"NO"} -CFP_MP=${CFP_MP:-"NO"} -nm="" -if [[ "${CFP_MP}" == "YES" ]]; then - nm=0 -fi -APRUNCFP=${APRUNCFP:-""} APRUN_ENKF=${APRUN_ENKF:-${APRUN:-""}} NTHREADS_ENKF=${NTHREADS_ENKF:-${NTHREADS:-1}} @@ -39,39 +32,20 @@ ENKFEXEC=${ENKFEXEC:-${EXECgfs}/enkf.x} GPREFIX=${GPREFIX:-""} APREFIX=${APREFIX:-""} -SMOOTH_ENKF=${SMOOTH_ENKF:-"YES"} - GBIASe=${GBIASe:-${APREFIX}abias_int.ensmean.txt} -CNVSTAT=${CNVSTAT:-${APREFIX}cnvstat.tar} -OZNSTAT=${OZNSTAT:-${APREFIX}oznstat.tar} -RADSTAT=${RADSTAT:-${APREFIX}radstat.tar} +CNVSTAT=${CNVSTAT:-${APREFIX}cnvstat_ensmean.tar} +OZNSTAT=${OZNSTAT:-${APREFIX}oznstat_ensmean.tar} +RADSTAT=${RADSTAT:-${APREFIX}radstat_ensmean.tar} ENKFSTAT=${ENKFSTAT:-${APREFIX}enkfstat.txt} # Namelist parameters -USE_CORRELATED_OBERRS=${USE_CORRELATED_OBERRS:-"NO"} -NAM_ENKF=${NAM_ENKF:-""} -SATOBS_ENKF=${SATOBS_ENKF:-""} -OZOBS_ENKF=${OZOBS_ENKF:-""} -use_correlated_oberrs=${use_correlated_oberrs:-".false."} -if [[ "${USE_CORRELATED_OBERRS}" == "YES" ]]; then +if [[ "${USE_CORRELATED_OBERRS:-}" == "YES" ]]; then use_correlated_oberrs=".true." fi -imp_physics=${imp_physics:-"99"} -lupp=${lupp:-".true."} corrlength=${corrlength:-1250} lnsigcutoff=${lnsigcutoff:-2.5} analpertwt=${analpertwt:-0.85} -readin_localization_enkf=${readin_localization_enkf:-".true."} -reducedgrid=${reducedgrid:-".false."} -letkf_flag=${letkf_flag:-".false."} -getkf=${getkf:-".false."} -denkf=${denkf:-".false."} -nobsl_max=${nobsl_max:-10000} -write_spread_diag=${write_spread_diag:-".false."} cnvw_option=${cnvw_option:-".false."} -netcdf_diag=${netcdf_diag:-".true."} -modelspace_vloc=${modelspace_vloc:-".false."} # if true, 'vlocal_eig.dat' is needed -taperanalperts=${taperanalperts:-".false."} IAUFHRS_ENKF=${IAUFHRS_ENKF:-"6,"} NMEM_ENS_MAX=${NMEM_ENS:-80} if [[ "${RUN}" == "enkfgfs" ]]; then @@ -94,9 +68,6 @@ ATMGES_ENSMEAN="${COMIN_ATMOS_HISTORY_STAT_PREV}/${GPREFIX}ensmean.atm.f006.nc" LONB_ENKF=${LONB_ENKF:-$(${NCLEN} "${ATMGES_ENSMEAN}" grid_xt)} # get LONB_ENKF LATB_ENKF=${LATB_ENKF:-$(${NCLEN} "${ATMGES_ENSMEAN}" grid_yt)} # get LATB_ENFK LEVS_ENKF=${LEVS_ENKF:-$(${NCLEN} "${ATMGES_ENSMEAN}" pfull)} # get LEVS_ENFK -use_gfs_ncio=".true." -use_gfs_nemsio=".false." -paranc=${paranc:-".true."} WRITE_INCR_ZERO="incvars_to_zero= ${INCREMENTS_TO_ZERO}," if [[ "${DO_CALC_INCREMENT}" == "YES" ]]; then write_fv3_incr=".false." @@ -114,60 +85,33 @@ HYBENSINFO=${HYBENSINFO:-${FIXgfs}/gsi/global_hybens_info.l${LEVS_ENKF}.txt} ANAVINFO=${ANAVINFO:-${FIXgfs}/gsi/global_anavinfo.l${LEVS_ENKF}.txt} VLOCALEIG=${VLOCALEIG:-${FIXgfs}/gsi/vlocal_eig_l${LEVS_ENKF}.dat} ENKF_SUFFIX="s" -if [[ "${SMOOTH_ENKF}" == "NO" ]]; then +if [[ "${SMOOTH_ENKF:-YES}" == "NO" ]]; then ENKF_SUFFIX="" fi ################################################################################ # Fixed files -${NLN} "${SATANGL}" satbias_angle -${NLN} "${SATINFO}" satinfo -${NLN} "${SCANINFO}" scaninfo -${NLN} "${CONVINFO}" convinfo -${NLN} "${OZINFO}" ozinfo -${NLN} "${HYBENSINFO}" hybens_info -${NLN} "${ANAVINFO}" anavinfo -${NLN} "${VLOCALEIG}" vlocal_eig.dat +cpreq "${SATANGL}" satbias_angle +cpreq "${SATINFO}" satinfo +cpreq "${SCANINFO}" scaninfo +cpreq "${CONVINFO}" convinfo +cpreq "${OZINFO}" ozinfo +cpreq "${HYBENSINFO}" hybens_info +cpreq "${ANAVINFO}" anavinfo +cpreq "${VLOCALEIG}" vlocal_eig.dat # Bias correction coefficients based on the ensemble mean -${NLN} "${COMIN_ATMOS_ANALYSIS_STAT}/${GBIASe}" "satbias_in" +cpreq "${COMIN_ATMOS_ANALYSIS_STAT}/${GBIASe}" "satbias_in" ################################################################################ +# Ensemble guess, observational data and analyses/increments -if [[ "${USE_CFP}" == "YES" ]]; then - rm -f "${DATA}/untar.sh" "${DATA}/mp_untar.sh" - cat > "${DATA}/untar.sh" << EOFuntar -#!/bin/sh -memchar=\$1 -COMOUT_ATMOS_ANALYSIS=\$2 flist="${CNVSTAT} ${OZNSTAT} ${RADSTAT}" -for ftype in \$flist; do - if [[ "\${memchar}" == "ensmean" ]]; then - fname=\${COMOUT_ATMOS_ANALYSIS}/\${ftype%.tar}.ensmean.tar - else - fname=\${COMOUT_ATMOS_ANALYSIS}/\${ftype} - fi - tar -xvf \$fname +for ftype in ${flist}; do + fname="${COMIN_ATMOS_ANALYSIS_STAT}/${ftype}" + tar -xvf "${fname}" done -EOFuntar - chmod 755 "${DATA}/untar.sh" -fi -################################################################################ -# Ensemble guess, observational data and analyses/increments - -flist="${CNVSTAT} ${OZNSTAT} ${RADSTAT}" -if [[ "${USE_CFP}" == "YES" ]]; then - echo "${nm} ${DATA}/untar.sh ensmean ${COMIN_ATMOS_ANALYSIS_STAT}" | tee -a "${DATA}/mp_untar.sh" - if [[ "${CFP_MP:-NO}" == "YES" ]]; then - nm=$((nm+1)) - fi -else - for ftype in ${flist}; do - fname="${COMIN_ATMOS_ANALYSIS_STAT}/${ftype}.tar" - tar -xvf "${fname}" - done -fi nfhrs=$(echo "${IAUFHRS_ENKF}" | sed 's/,/ /g') for imem in $(seq 1 ${NMEM_ENS}); do smem=$((imem + mem_offset)) @@ -227,20 +171,6 @@ for FHR in ${nfhrs}; do fi done -if [[ "${USE_CFP}" == "YES" ]]; then - chmod 755 "${DATA}/mp_untar.sh" - ncmd=$(wc -l < "${DATA}/mp_untar.sh") - if [[ ${ncmd} -gt 0 ]]; then - ncmd_max=$((ncmd < max_tasks_per_node ? ncmd : max_tasks_per_node)) - APRUNCFP=$(eval echo "${APRUNCFP}") - ${APRUNCFP} "${DATA}/mp_untar.sh" && true - export err=$? - if [[ ${err} -ne 0 ]]; then - err_exit "Failed to untar input data!" - fi - fi -fi - ################################################################################ # Create global_enkf namelist cat > enkf.nml << EOFnml @@ -257,19 +187,19 @@ cat > enkf.nml << EOFnml sprd_tol=1.e30,paoverpb_thresh=0.98, nlons=${LONA_ENKF},nlats=${LATA_ENKF},nlevs=${LEVS_ENKF},nanals=${NMEM_ENS}, deterministic=.true.,sortinc=.true.,lupd_satbiasc=.false., - reducedgrid=${reducedgrid},readin_localization=${readin_localization_enkf}., - use_gfs_nemsio=${use_gfs_nemsio},use_gfs_ncio=${use_gfs_ncio},imp_physics=${imp_physics},lupp=${lupp}, + reducedgrid=${reducedgrid:-.false.},readin_localization=${readin_localization_enkf:-.true.}, + use_gfs_nemsio=.false.,use_gfs_ncio=.true.,imp_physics=${imp_physics:-99},lupp=${lupp:-.true.}, univaroz=.false.,adp_anglebc=.true.,angord=4,use_edges=.false.,emiss_bc=.true., - letkf_flag=${letkf_flag},nobsl_max=${nobsl_max},denkf=${denkf},getkf=${getkf}., + letkf_flag=${letkf_flag:-.false.},nobsl_max=${nobsl_max:-10000},denkf=${denkf:-.false.},getkf=${getkf:-.false.}, nhr_anal=${IAUFHRS_ENKF},nhr_state=${IAUFHRS_ENKF}, - lobsdiag_forenkf=.true.,taperanalperts=${taperanalperts}, - write_spread_diag=${write_spread_diag}, - modelspace_vloc=${modelspace_vloc}, - use_correlated_oberrs=${use_correlated_oberrs}, - netcdf_diag=${netcdf_diag},cnvw_option=${cnvw_option}, - paranc=${paranc},write_fv3_incr=${write_fv3_incr}, + lobsdiag_forenkf=.true.,taperanalperts=${taperanalperts:-.false.}, + write_spread_diag=${write_spread_diag:-".false."}, + modelspace_vloc=${modelspace_vloc:-.false.}, + use_correlated_oberrs=${use_correlated_oberrs:-.false.}, + netcdf_diag=.true.,cnvw_option=${cnvw_option}, + paranc=${paranc:-.true.},write_fv3_incr=${write_fv3_incr}, ${WRITE_INCR_ZERO} - ${NAM_ENKF} + ${NAM_ENKF:-} / &satobs_enkf sattypes_rad(1) = 'amsua_n15', dsis(1) = 'amsua_n15', @@ -352,7 +282,7 @@ cat > enkf.nml << EOFnml sattypes_rad(78)= 'atms_n21', dsis(78)= 'atms_n21', sattypes_rad(79)= 'cris-fsr_n21', dsis(79)= 'cris-fsr_n21', sattypes_rad(80)= 'abi_g19', dsis(80)= 'abi_g19', - ${SATOBS_ENKF} + ${SATOBS_ENKF:-} / &ozobs_enkf sattypes_oz(1) = 'sbuv2_n16', @@ -371,7 +301,7 @@ cat > enkf.nml << EOFnml sattypes_oz(14) = 'ompstc8_n21', sattypes_oz(15) = 'ompsnp_n21', sattypes_oz(16) = 'gome_metop-c', - ${OZOBS_ENKF} + ${OZOBS_ENKF:-} / EOFnml @@ -390,10 +320,10 @@ if [[ ${err} -ne 0 ]]; then fi # Cat runtime output files. -cat stdout stderr > "${COMOUT_ATMOS_ANALYSIS_STAT}/${ENKFSTAT}" +cat stdout stderr > enkfstat.txt +cpfs enkfstat.txt "${COMOUT_ATMOS_ANALYSIS_STAT}/${ENKFSTAT}" ################################################################################ # Postprocessing -cd "${pwd}" || exit 1 exit "${err}" diff --git a/scripts/exglobal_atmos_analysis.sh b/scripts/exglobal_atmos_analysis.sh index 97a02f7f16b..d738b76c643 100755 --- a/scripts/exglobal_atmos_analysis.sh +++ b/scripts/exglobal_atmos_analysis.sh @@ -38,7 +38,6 @@ export CHGRP_CMD=${CHGRP_CMD:-"chgrp ${group_name:-rstprod}"} export NCLEN=${NCLEN:-${USHgfs}/getncdimlen} COMPRESS=${COMPRESS:-gzip} UNCOMPRESS=${UNCOMPRESS:-gunzip} -APRUNCFP=${APRUNCFP:-""} APRUN_GSI=${APRUN_GSI:-${APRUN:-""}} NTHREADS_GSI=${NTHREADS_GSI:-${NTHREADS:-1}} @@ -56,8 +55,6 @@ IASI_CADS=${IASI_CADS:-".false."} CRIS_CADS=${CRIS_CADS:-".false."} # Diagnostic files options -netcdf_diag=${netcdf_diag:-".true."} -binary_diag=${binary_diag:-".false."} lobsdiag_forenkf=${lobsdiag_forenkf:-".false."} # IAU @@ -185,11 +182,7 @@ ABIAS=${ABIAS:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias.txt} ABIASPC=${ABIASPC:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_pc.txt} ABIASAIR=${ABIASAIR:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_air.txt} ABIASe=${ABIASe:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_int.txt} -RADSTAT=${RADSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}radstat.tar} GSISTAT=${GSISTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}gsistat.txt} -PCPSTAT=${PCPSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}pcpstat} -CNVSTAT=${CNVSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}cnvstat.tar} -OZNSTAT=${OZNSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}oznstat.tar} # Increment files ATMINC=${ATMINC:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}increment.atm.i006.nc} @@ -198,21 +191,8 @@ ATMINC=${ATMINC:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}increment.atm.i006.nc} RUN_SELECT=${RUN_SELECT:-"NO"} USE_SELECT=${USE_SELECT:-"NO"} USE_RADSTAT=${USE_RADSTAT:-"YES"} -SELECT_OBS=${SELECT_OBS:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}obsinput} +SELECT_OBS=${SELECT_OBS:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}obsinput.tar} GENDIAG=${GENDIAG:-"YES"} -DIAG_SUFFIX=${DIAG_SUFFIX:-""} -if [[ ${netcdf_diag} == ".true." ]] ; then - DIAG_SUFFIX="${DIAG_SUFFIX}.nc4" -fi -DIAG_COMPRESS=${DIAG_COMPRESS:-"YES"} -DIAG_TARBALL=${DIAG_TARBALL:-"YES"} -USE_CFP=${USE_CFP:-"NO"} -CFP_MP=${CFP_MP:-"NO"} -nm="" -if [[ ${CFP_MP} == "YES" ]]; then - nm=0 -fi -DIAG_DIR=${DIAG_DIR:-${COMOUT_ATMOS_ANALYSIS}/gsidiags} # Set script / GSI control parameters DOHYBVAR=${DOHYBVAR:-"NO"} @@ -335,7 +315,6 @@ CHEM=${CHEM:-""} NST=${NST:-""} #uGSI Namelist parameters -lrun_subdirs=${lrun_subdirs:-".true."} if [[ ${DOHYBVAR} == "YES" ]]; then l_hyb_ens=.true. export l4densvar=${l4densvar:-".false."} @@ -629,25 +608,6 @@ if [[ ${JCAP} -ne ${JCAP_A} ]]; then fi fi -############################################################## -# Diagnostic files -# if requested, link GSI diagnostic file directories for use later -if [[ ${GENDIAG} == "YES" ]] ; then - if [[ ${lrun_subdirs} == ".true." ]] ; then - if [[ -d ${DIAG_DIR} ]]; then - rm -rf "${DIAG_DIR}" - fi - ntasks_m1="$((ntasks-1))" - for pe in $(seq 0 ${ntasks_m1}); do - pedir="dir."$(printf %04i ${pe}) - mkdir -p "${DIAG_DIR}/${pedir}" - ${NLN} "${DIAG_DIR}/${pedir}" "${pedir}" - done - else - err_exit "lrun_subdirs must be true. lrun_subdirs=${lrun_subdirs}" - fi -fi - ############################################################## # Output files ${NLN} ${ATMANL} siganl @@ -694,60 +654,31 @@ fi ############################################################## # If requested, copy and de-tar guess radstat file if [[ "${USE_RADSTAT}" == "YES" ]]; then - if [[ "${USE_CFP}" == "YES" ]]; then - if [[ -f "${DATA}/unzip.sh" ]]; then - rm -f "${DATA}/unzip.sh" - fi - if [[ -f "${DATA}/mp_unzip.sh" ]]; then - rm -f "${DATA}/mp_unzip.sh" - fi - cat > "${DATA}/unzip.sh" << EOFunzip -#!/bin/sh - diag_file=\$1 - diag_suffix=\$2 - fname=\$(echo \$diag_file | cut -d'.' -f1) - fdate=\$(echo \$diag_file | cut -d'.' -f2) - ${UNCOMPRESS} \$diag_file - fnameges=\$(echo \$fname | sed 's/_ges//g') - ${NMV} \$fname.\$fdate\$diag_suffix \$fnameges -EOFunzip - chmod 755 ${DATA}/unzip.sh - fi + rm -f "${DATA}/unzip_radstat.sh" + cat > "${DATA}/unzip_radstat.sh" << EOF +#!/bin/bash +diag_file=\$1 +diag_suffix=\$2 +fname=\$(echo \$diag_file | cut -d'.' -f1) +fdate=\$(echo \$diag_file | cut -d'.' -f2) +${UNCOMPRESS} \$diag_file +fnameges=\$(echo \$fname | sed 's/_ges//g') +${NMV} \$fname.\$fdate\$diag_suffix \$fnameges +EOF + chmod 755 ${DATA}/unzip_radstat.sh - listdiag=$(tar xvf radstat.gdas | cut -d' ' -f2 | grep _ges) + rm -f "${DATA}/cmdfile" + listdiag=$(tar -xvf radstat.gdas | cut -d' ' -f2 | grep _ges) for type in ${listdiag}; do diag_file=$(echo ${type} | cut -d',' -f1) - if [[ ${USE_CFP} == "YES" ]] ; then - echo "${nm} ${DATA}/unzip.sh ${diag_file} ${DIAG_SUFFIX}" | tee -a "${DATA}/mp_unzip.sh" - if [[ ${CFP_MP:-"NO"} == "YES" ]]; then - nm=$((nm+1)) - fi - else - fname=$(echo "${diag_file}" | cut -d'.' -f1) - date=$(echo "${diag_file}" | cut -d'.' -f2) - ${UNCOMPRESS} "${diag_file}" - fnameges=$(echo "${fname}"|sed 's/_ges//g') - ${NMV} "${fname}.${date}${DIAG_SUFFIX}" "${fnameges}" - fi + echo "${DATA}/unzip_radstat.sh ${diag_file} ${DIAG_SUFFIX:-}.nc4" >> "${DATA}/cmdfile" done - if [[ "${USE_CFP}" == "YES" ]] ; then - chmod 755 "${DATA}/mp_unzip.sh" - ncmd=$(wc -l < "${DATA}/mp_unzip.sh") - if [[ ${ncmd} -gt 0 ]]; then - if [[ ${ncmd} -lt ${max_tasks_per_node} ]]; then - ncmd_max=${ncmd} - else - ncmd_max=${max_tasks_per_node} - fi - APRUNCFP_UNZIP=$(eval echo "${APRUNCFP}") - ${APRUNCFP_UNZIP} "${DATA}/mp_unzip.sh" - export err=$? - if [[ ${err} -ne 0 ]]; then - err_exit "Failed to unzip input data files!" - fi - fi - fi + "${USHgfs}/run_mpmd.sh" "${DATA}/cmdfile" && true + export err=$? + if [[ ${err} -ne 0 ]]; then + err_exit "Failed to unzip radstat.gdas file!" + fi fi # if [[ $USE_RADSTAT == "YES" ] ############################################################## @@ -785,12 +716,12 @@ cat > gsiparm.anl << EOF use_pbl=.false.,use_compress=.true.,nsig_ext=45,gpstop=50.,commgpstop=45.,commgpserrinf=1.0, use_gfs_nemsio=.false.,use_gfs_ncio=.true.,sfcnst_comb=.true., use_readin_anl_sfcmask=${USE_READIN_ANL_SFCMASK}, - lrun_subdirs=${lrun_subdirs}, + lrun_subdirs=.true., crtm_coeffs_path='./crtm_coeffs/', newpc4pred=.true.,adp_anglebc=.true.,angord=4,passive_bc=.true.,use_edges=.false., diag_precon=.true.,step_start=1.e-3,emiss_bc=.true.,nhr_obsbin=${nhr_obsbin:-3}, cwoption=3,imp_physics=${imp_physics},lupp=${lupp},cnvw_option=${cnvw_option},cao_check=${cao_check}, - netcdf_diag=${netcdf_diag},binary_diag=${binary_diag}, + netcdf_diag=.true.,binary_diag=.false., lobsdiag_forenkf=${lobsdiag_forenkf}, write_fv3_incr=${write_fv3_increment}, nhr_anal=${IAUFHRS}, @@ -944,7 +875,7 @@ fi ################################################################################ # Postprocessing -cd "${pwd}" || exit 1 +cd "${DATA}" || exit 1 ############################################################## # Add this statement to release the forecast job once the @@ -954,6 +885,18 @@ cd "${pwd}" || exit 1 if [[ ${SENDECF} == "YES" && "${RUN}" != "enkf" ]]; then ecflow_client --event release_fcst fi + +# Diagnostic files +# if requested, GSI diagnostic file directories for use later +if [[ "${GENDIAG}" == "YES" ]] ; then + tar -cvf gsidiags.tar dir.???? + export err=$? + if [[ ${err} -ne 0 ]]; then + err_exit "Failed to tar GSI diagnostic directories!" + fi + cpfs gsidiags.tar "${COMOUT_ATMOS_ANALYSIS}/${APREFIX}gsidiags${DIAG_SUFFIX:-}.tar" +fi + echo "${rCDUMP} ${PDY}${cyc} atminc done at $(date)" > "${COMOUT_ATMOS_ANALYSIS}/${APREFIX}analysis.done.txt" ################################################################################ diff --git a/scripts/exglobal_diag.sh b/scripts/exglobal_diag.sh index 8957e6201ca..d0fba8550c8 100755 --- a/scripts/exglobal_diag.sh +++ b/scripts/exglobal_diag.sh @@ -18,24 +18,19 @@ ################################################################################ # Set environment. +cd "${DATA}" || exit 8 # Base variables -GDUMP=${GDUMP:-"gdas"} # Utilities -export CHGRP_CMD=${CHGRP_CMD:-"chgrp ${group_name:-rstprod}"} -export NCLEN=${NCLEN:-${USHgfs}/getncdimlen} -export CATEXEC=${CATEXEC:-${ncdiag_ROOT:-${gsi_ncdiag_ROOT}}/bin/ncdiag_cat_serial.x} +CHGRP_CMD=${CHGRP_CMD:-"chgrp ${group_name:-rstprod}"} +CATEXEC=${CATEXEC:-${ncdiag_ROOT:-${gsi_ncdiag_ROOT}}/bin/ncdiag_cat_serial.x} COMPRESS=${COMPRESS:-gzip} UNCOMPRESS=${UNCOMPRESS:-gunzip} APRUNCFP=${APRUNCFP:-""} -# Diagnostic files options -netcdf_diag=${netcdf_diag:-".true."} -binary_diag=${binary_diag:-".false."} - # Analysis files -export APREFIX=${APREFIX:-""} +APREFIX=${APREFIX:-"${RUN}.t${cyc}z."} RADSTAT=${RADSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}radstat.tar} PCPSTAT=${PCPSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}pcpstat} CNVSTAT=${CNVSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}cnvstat.tar} @@ -46,36 +41,27 @@ rm -f "${RADSTAT}" "${PCPSTAT}" "${CNVSTAT}" "${OZNSTAT}" # Obs diag GENDIAG=${GENDIAG:-"YES"} -DIAG_SUFFIX=${DIAG_SUFFIX:-""} -if [[ "${netcdf_diag}" == ".true." ]] ; then - DIAG_SUFFIX="${DIAG_SUFFIX}.nc4" -fi +GSIDIAG=${GSIDIAG:-"${COMIN_ATMOS_ANALYSIS}/${APREFIX}gsidiags${DIAG_SUFFIX:-}.tar"} +USE_BUILD_GSINFO=${USE_BUILD_GSINFO:-"NO"} DIAG_COMPRESS=${DIAG_COMPRESS:-"YES"} DIAG_TARBALL=${DIAG_TARBALL:-"YES"} -USE_CFP=${USE_CFP:-"NO"} -CFP_MP=${CFP_MP:-"NO"} -nm="" -if [[ "${CFP_MP}" == "YES" ]]; then - nm=0 -fi -DIAG_DIR=${DIAG_DIR:-${COMOUT_ATMOS_ANALYSIS}/gsidiags} -REMOVE_DIAG_DIR=${REMOVE_DIAG_DIR:-"NO"} # Set script / GSI control parameters -lrun_subdirs=${lrun_subdirs:-".true."} +################################################################################ +if [[ "${GENDIAG}" != "YES" ]] ; then + echo "INFO: GENDIAG set to NO. Skipping diagnostic file generation." + exit 0 +fi ################################################################################ -# If requested, generate diagnostic files -if [[ "${GENDIAG}" == "YES" ]] ; then - if [[ "${lrun_subdirs}" == ".true." ]] ; then - for pe in ${DIAG_DIR}/dir.*; do - pedir="$(basename -- "${pe}")" - ${NLN} "${pe}" "${DATA}/${pedir}" - done - else - err_exit "lrun_subdirs must be true. Abort job" - fi +# Copy gsidiags.tar file from COMIN to DATA and untar +cpreq "${GSIDIAG}" ./gsidiags.tar +tar -xvf gsidiags.tar +export err=$? +if [[ ${err} -ne 0 ]]; then + err_exit "Unable to unpack gsidiags.tar file!" +fi # Set up lists and variables for various types of diagnostic files. ntype=3 @@ -106,43 +92,40 @@ if [[ "${GENDIAG}" == "YES" ]] ; then numfile[2]=0 numfile[3]=0 - # Set diagnostic file prefix based on lrun_subdirs variable - if [[ "${lrun_subdirs}" == ".true." ]]; then - prefix=" dir.*/" - else - prefix="pe*" - fi + prefix="dir.*/" - if [[ "${USE_CFP}" == "YES" ]]; then - rm -f "${DATA}/diag.sh" "${DATA}/mp_diag.sh" - cat > "${DATA}/diag.sh" << EOFdiag -#!/bin/sh -lrun_subdirs=\$1 -binary_diag=\$2 -type=\$3 -loop=\$4 -string=\$5 -PDY=\$6 -cyc=\$7 -DIAG_COMPRESS=\$8 -DIAG_SUFFIX=\$9 -if [[ "\${lrun_subdirs}" == ".true." ]]; then - prefix=" dir.*/" -else - prefix="pe*" -fi -file=diag_\${type}_\${string}.\${PDY}\${cyc}\${DIAG_SUFFIX} -if [[ "\${binary_diag}" == ".true." ]]; then - cat \${prefix}\${type}_\${loop}* > \$file + rm -f "${DATA}/diag.sh" + cat > "${DATA}/diag.sh" << EOF +#!/bin/bash +set -x + +type=\$1 +loop=\$2 +string=\$3 +count=\$4 +suffix=\$5 + +# Match files with this prefix +diag_files=dir.*/\${type}_\${loop} + +# Name of combined diagnostic file from matched files +out_diag_file=diag_\${type}_\${string}.${PDY}${cyc}\${suffix} + +# Combine diagnostic files +if [[ \${count} -gt 1 ]]; then + ${CATEXEC} -o \${out_diag_file} \${diag_files}* else - ${CATEXEC} -o \$file \${prefix}\${type}_\${loop}* + cat \${diag_files}* > "\${out_diag_file}" fi -if [[ "\${DIAG_COMPRESS}" == "YES" ]]; then - ${COMPRESS} "\${file}" + +# Compress diagnostic file if requested +if [[ "${DIAG_COMPRESS:-}" == "YES" ]]; then + ${COMPRESS} "\${out_diag_file}" fi -EOFdiag - chmod 755 "${DATA}/diag.sh" - fi + +exit 0 +EOF + chmod 755 "${DATA}/diag.sh" # Collect diagnostic files as a function of loop and type. # Loop over first and last outer loops to generate innovation @@ -155,6 +138,8 @@ EOFdiag # write_diag(1)=.true. turns on creation of o-g # innovation files. + rm -f cmdfile + touch cmdfile loops="01 03" for loop in ${loops}; do case ${loop} in @@ -168,64 +153,35 @@ EOFdiag n=$(( n + 1 )) for type in $(echo "${diagtype[n]}"); do count=$(ls ${prefix}${type}_${loop}* 2>/dev/null | wc -l) - if [[ ${count} -gt 1 ]]; then - if [[ "${USE_CFP}" == "YES" ]]; then - echo "${nm} ${DATA}/diag.sh ${lrun_subdirs} ${binary_diag} ${type} ${loop} ${string} ${PDY} ${cyc} ${DIAG_COMPRESS} ${DIAG_SUFFIX}" | tee -a "${DATA}/mp_diag.sh" - if [[ "${CFP_MP:-"NO"}" == "YES" ]]; then - nm=$((nm+1)) - fi - else - if [[ "${binary_diag}" == ".true." ]]; then - cat ${prefix}${type}_${loop}* > "diag_${type}_${string}.${PDY}${cyc}${DIAG_SUFFIX}" - else - ${CATEXEC} -o "diag_${type}_${string}.${PDY}${cyc}${DIAG_SUFFIX}" "${prefix}${type}_${loop}"* - fi - fi - echo "diag_${type}_${string}.${PDY}${cyc}*" >> "${diaglist[n]}" - numfile[n]=$(expr ${numfile[n]} + 1) - elif [[ ${count} -eq 1 ]]; then - cat ${prefix}${type}_${loop}* > "diag_${type}_${string}.${PDY}${cyc}${DIAG_SUFFIX}" - if [[ "${DIAG_COMPRESS}" == "YES" ]]; then - ${COMPRESS} "diag_${type}_${string}.${PDY}${cyc}${DIAG_SUFFIX}" - fi - echo "diag_${type}_${string}.${PDY}${cyc}*" >> "${diaglist[n]}" - numfile[n]=$(expr "${numfile[n]}" + 1) + if [[ ${count} -eq 0 ]]; then + continue fi + echo "${DATA}/diag.sh ${type} ${loop} ${string} ${count} ${DIAG_SUFFIX:-}.nc4" >> cmdfile + echo "diag_${type}_${string}.${PDY}${cyc}*" >> "${diaglist[n]}" + numfile[n]=$(expr ${numfile[n]} + 1) done done echo $(date) END loop "${string}" >&2 done - # We should already be in $DATA, but extra cd to be sure. - cd "${DATA}" || exit - - # If requested, compress diagnostic files - if [[ "${DIAG_COMPRESS}" == "YES" && "${USE_CFP}" == "NO" ]]; then - echo $(date) START "${COMPRESS}" diagnostic files >&2 - # shellcheck disable=SC2086 - for file in "diag_"*"${PDY}${cyc}${DIAG_SUFFIX}"; do - ${COMPRESS} "${file}" - done - echo "$(date) END ${COMPRESS} diagnostic files" >&2 + ncmd=$(wc -l < cmdfile) + if [[ ${ncmd} -eq 0 ]]; then + echo "WARNING: No diagnostic files found to process!" + exit 0 fi - if [[ "${USE_CFP}" == "YES" ]] ; then - chmod 755 "${DATA}/mp_diag.sh" - ncmd=$(wc -l < "${DATA}/mp_diag.sh") - if [[ ${ncmd} -gt 0 ]]; then - if [[ ${ncmd} -lt ${max_tasks_per_node} ]]; then - ncmd_max=${ncmd} - else - ncmd_max=${max_tasks_per_node} - fi - APRUNCFP_DIAG=$(eval echo "${APRUNCFP}") - ${APRUNCFP_DIAG} "${DATA}/mp_diag.sh" - export err=$? - if [[ ${err} -ne 0 ]]; then - err_exit "Failed to compress one or more observation diagnostic files!" - fi - fi - fi + # MPMD can only be executed on a single node, + # so break up cmdfile into parts of tasks_per_node size files + # and run them sequentially + split -l ${tasks_per_node} ./cmdfile cmdfile_part_ + cmdfile_parts=$(ls cmdfile_part_*) + for partfile in ${cmdfile_parts}; do + "${USHgfs}/run_mpmd.sh" "${partfile}" && true + export err=$? + if [[ ${err} -ne 0 ]]; then + err_exit "Failed to create one or more observation diagnostic files for ${partfile}!" + fi + done # Restrict diagnostic files containing rstprod data if [[ "${CHGRP_RSTPROD}" == "YES" ]]; then @@ -272,13 +228,8 @@ EOFdiag echo "$(date) END tar diagnostic files" >&2 fi -fi # End diagnostic file generation block - if [[ "${GENDIAG}" == "YES" ]] ################################################################################ # Postprocessing -# Remove $DIAG_DIR -if [[ "${REMOVE_DIAG_DIR}" == "YES" ]]; then - rm -rf "${DIAG_DIR}" -fi exit 0 From 319f6cf4406bee96b664a15787ff3ce4273535d8 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 21 Nov 2025 09:38:26 -0500 Subject: [PATCH 03/15] remove undeclared function name usage from these build scripts --- sorc/build_gdas.sh | 2 -- sorc/build_gsi_enkf.sh | 3 --- sorc/build_gsi_monitor.sh | 2 -- sorc/build_ufs_utils.sh | 3 --- sorc/build_ww3prepost.sh | 2 -- 5 files changed, 12 deletions(-) diff --git a/sorc/build_gdas.sh b/sorc/build_gdas.sh index 32213153423..e49cc00429c 100755 --- a/sorc/build_gdas.sh +++ b/sorc/build_gdas.sh @@ -13,11 +13,9 @@ while getopts ":j:dv" option; do v) _opts+="-v ";; :) echo "[${BASH_SOURCE[0]}]: ${option} requires an argument" - usage ;; *) echo "[${BASH_SOURCE[0]}]: Unrecognized option: ${option}" - usage ;; esac done diff --git a/sorc/build_gsi_enkf.sh b/sorc/build_gsi_enkf.sh index 09cf250acf2..badba6c332d 100755 --- a/sorc/build_gsi_enkf.sh +++ b/sorc/build_gsi_enkf.sh @@ -9,11 +9,9 @@ while getopts ":j:dv" option; do v) BUILD_VERBOSE="YES";; :) echo "[${BASH_SOURCE[0]}]: ${option} requires an argument" - usage ;; *) echo "[${BASH_SOURCE[0]}]: Unrecognized option: ${option}" - usage ;; esac done @@ -33,4 +31,3 @@ REGRESSION_TESTS=NO \ "${HOMEgfs_}/sorc/gsi_enkf.fd/ush/build.sh" exit - diff --git a/sorc/build_gsi_monitor.sh b/sorc/build_gsi_monitor.sh index d355067f127..e982f14f703 100755 --- a/sorc/build_gsi_monitor.sh +++ b/sorc/build_gsi_monitor.sh @@ -12,11 +12,9 @@ while getopts ":j:dv" option; do v) BUILD_VERBOSE="YES";; :) echo "[${BASH_SOURCE[0]}]: ${option} requires an argument" - usage ;; *) echo "[${BASH_SOURCE[0]}]: Unrecognized option: ${option}" - usage ;; esac done diff --git a/sorc/build_ufs_utils.sh b/sorc/build_ufs_utils.sh index 2284b151343..e6698bb5008 100755 --- a/sorc/build_ufs_utils.sh +++ b/sorc/build_ufs_utils.sh @@ -12,11 +12,9 @@ while getopts ":j:dv" option; do v) BUILD_VERBOSE="YES";; :) echo "[${BASH_SOURCE[0]}]: ${option} requires an argument" - usage ;; *) echo "[${BASH_SOURCE[0]}]: Unrecognized option: ${option}" - usage ;; esac done @@ -31,4 +29,3 @@ BUILD_VERBOSE=${BUILD_VERBOSE:-} \ "${HOMEgfs_}/sorc/ufs_utils.fd/build_all.sh" exit - diff --git a/sorc/build_ww3prepost.sh b/sorc/build_ww3prepost.sh index 19133b12972..5408a5c390c 100755 --- a/sorc/build_ww3prepost.sh +++ b/sorc/build_ww3prepost.sh @@ -16,11 +16,9 @@ while getopts ":j:a:dvw" option; do w) PDLIB="OFF" ;; :) echo "[${BASH_SOURCE[0]}]: ${option} requires an argument" - usage ;; *) echo "[${BASH_SOURCE[0]}]: Unrecognized option: ${option}" - usage ;; esac done From ffad4c7d4105b68806ad2315bb0b86a074060e1b Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 21 Nov 2025 17:48:21 -0500 Subject: [PATCH 04/15] made it past eupd and gdas_sfcanl --- jobs/JGDAS_ENKF_UPDATE | 12 +- scripts/exgdas_enkf_update.sh | 23 ++- ush/regrid_gsiSfcIncr_to_tile.sh | 264 ++++++++++++++++--------------- 3 files changed, 149 insertions(+), 150 deletions(-) diff --git a/jobs/JGDAS_ENKF_UPDATE b/jobs/JGDAS_ENKF_UPDATE index 96d5dff079a..15ab2512c39 100755 --- a/jobs/JGDAS_ENKF_UPDATE +++ b/jobs/JGDAS_ENKF_UPDATE @@ -9,15 +9,11 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "eupd" -c "base anal eupd" ############################################## # Begin JOB SPECIFIC work ############################################## -# shellcheck disable=SC2153 -GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") -export gPDY=${GDATE:0:8} -export gcyc=${GDATE:8:2} -export GDUMP="gdas" -export GDUMP_ENS="enkf${GDUMP}" +export GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") +export GDUMP="enkfgdas" # EnKF ensemble always uses RUN=enkfgdas for prior members export APREFIX="${RUN}.t${cyc}z." -export GPREFIX="${GDUMP_ENS}.t${gcyc}z." +export GPREFIX="${GDUMP}.t${GDATE:8:2}z." MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMIN_ATMOS_ANALYSIS_STAT:COM_ATMOS_ANALYSIS_TMPL @@ -25,7 +21,7 @@ MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMOUT_ATMOS_ANALYSIS_STAT:COM_ATMOS_ANALYSIS_TMPL -MEMDIR="ensstat" RUN="enkfgdas" YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ +MEMDIR="ensstat" RUN="enkfgdas" YMD=${GDATE:0:8} HH=${GDATE:8:2} declare_from_tmpl -rx \ COMIN_ATMOS_HISTORY_STAT_PREV:COM_ATMOS_HISTORY_TMPL ############################################################### diff --git a/scripts/exgdas_enkf_update.sh b/scripts/exgdas_enkf_update.sh index d4829132eb2..9253c69a110 100755 --- a/scripts/exgdas_enkf_update.sh +++ b/scripts/exgdas_enkf_update.sh @@ -28,15 +28,13 @@ NTHREADS_ENKF=${NTHREADS_ENKF:-${NTHREADS:-1}} # Executables ENKFEXEC=${ENKFEXEC:-${EXECgfs}/enkf.x} -# Filenames. -GPREFIX=${GPREFIX:-""} -APREFIX=${APREFIX:-""} +APREFIX=${APREFIX:-${RUN}.t${cyc}z.} +GPREFIX=${GPREFIX:-${RUN}.t${GDATE:8:2}z.} -GBIASe=${GBIASe:-${APREFIX}abias_int.ensmean.txt} -CNVSTAT=${CNVSTAT:-${APREFIX}cnvstat_ensmean.tar} -OZNSTAT=${OZNSTAT:-${APREFIX}oznstat_ensmean.tar} -RADSTAT=${RADSTAT:-${APREFIX}radstat_ensmean.tar} -ENKFSTAT=${ENKFSTAT:-${APREFIX}enkfstat.txt} +# GBIASe=${GBIASe:-${APREFIX}abias_int.ensmean.txt} # TODO: remove (see comment and TODO below) Also, this is not a "G"BIAS (See the name, it has APREFIX in its name) +CNVSTAT="${APREFIX}cnvstat_ensmean.tar" +OZNSTAT="${APREFIX}oznstat_ensmean.tar" +RADSTAT="${APREFIX}radstat_ensmean.tar" # Namelist parameters if [[ "${USE_CORRELATED_OBERRS:-}" == "YES" ]]; then @@ -113,7 +111,7 @@ else fi # Bias correction coefficients based on the ensemble mean -${NLN} "${COMIN_ATMOS_ANALYSIS_STAT}/${GBIASe}" "satbias_in" +#${NLN} "${COMIN_ATMOS_ANALYSIS_STAT}/${GBIASe}" "satbias_in" # This file does not exist when test was run # TODO: remove ################################################################################ # Ensemble guess, observational data and analyses/increments @@ -133,7 +131,7 @@ for imem in $(seq 1 ${NMEM_ENS}); do gmemchar="mem"$(printf "%03i" "${smem}") memchar="mem"$(printf "%03i" "${imem}") - MEMDIR=${gmemchar} RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -x \ + MEMDIR=${gmemchar} RUN=${GDUMP} YMD=${GDATE:0:8} HH=${GDATE:8:2} declare_from_tmpl -x \ COMIN_ATMOS_HISTORY_MEM_PREV:COM_ATMOS_HISTORY_TMPL MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl -x \ @@ -178,9 +176,10 @@ for FHR in ${nfhrs}; do if [[ "${DO_GSISOILDA}" == "YES" ]]; then ${NLN} "${COMIN_ATMOS_HISTORY_STAT_PREV}/${GPREFIX}ensmean.sfc.f00${FHR}.nc" \ "bfg_${PDY}${cyc}_fhr0${FHR}_ensmean" - ${NLN} "${COMIN_ATMOS_ANALYSIS_STAT}/${APREFIX}increment.sfc.i00${FHR}.nc" \ + ${NLN} "${COMOUT_ATMOS_ANALYSIS_STAT}/${APREFIX}ensmean_increment.sfc.i00${FHR}.nc" "sfcincr_${PDY}${cyc}_fhr0${FHR}_ensmean" fi + done ################################################################################ @@ -333,7 +332,7 @@ fi # Cat runtime output files. cat stdout stderr > enkfstat.txt -cpfs enkfstat.txt "${COMOUT_ATMOS_ANALYSIS_STAT}/${ENKFSTAT}" +cpfs enkfstat.txt "${COMOUT_ATMOS_ANALYSIS_STAT}/${APREFIX}enkfstat.txt" ################################################################################ # Postprocessing diff --git a/ush/regrid_gsiSfcIncr_to_tile.sh b/ush/regrid_gsiSfcIncr_to_tile.sh index aa922229f6c..67f875217f0 100755 --- a/ush/regrid_gsiSfcIncr_to_tile.sh +++ b/ush/regrid_gsiSfcIncr_to_tile.sh @@ -9,12 +9,6 @@ source "${HOMEgfs}/ush/atparse.bash" # David New, Nov 2025 (parallelization updates) #------------------------------------------------------------------------------------------------- -export PGMOUT=${PGMOUT:-${pgmout:-'&1'}} -export PGMERR=${PGMERR:-${pgmerr:-'&2'}} - -export PGM=${REGRID_EXEC} -export pgm=${PGM} - NMEM_REGRID=${NMEM_REGRID:-1} CASE_IN=${CASE_IN:-${CASE_ENS}} LFHR=${LFHR:-6} @@ -31,13 +25,9 @@ LSOIL_INCR=${LSOIL_INCR:-2} export n_vars=$(( LSOIL_INCR*2 )) -soil_incr_vars="" -for vi in $( seq 1 "${LSOIL_INCR}" ); do - soil_incr_vars=${soil_incr_vars}'"soilt'${vi}'_inc"', -done -for vi in $( seq 1 "${LSOIL_INCR}" ); do - soil_incr_vars=${soil_incr_vars}'"slc'${vi}'_inc"', -done +soilt_incr_vars=$(seq -s ',' -f '"soilt%g_inc"' 1 "${LSOIL_INCR}") +slc_incr_vars=$(seq -s ',' -f '"slc%g_inc"' 1 "${LSOIL_INCR}") +export soil_incr_vars="${soilt_incr_vars},${slc_incr_vars}" if [[ "${DO_LAND_IAU}" = ".true." ]]; then IFS=',' read -ra landifhrs <<< "${IAUFHRS}" @@ -52,9 +42,9 @@ export jres=${LATB_CASE_IN} export ireso=${CASE_OUT:1} export jreso=${CASE_OUT:1} -regrid_nml_tmpl="${PARMgfs}/regrid_sfc/regrid.nml_tmpl" +regrid_nml_tmpl="${PARMgfs}/regrid_sfc/regrid.nml_tmpl" -if (( LFHR >= 0 )); then +if [[ "${LFHR}" -ge 0 ]]; then soilinc_fhrs=("${LFHR}") else # construct restart times for deterministic member soilinc_fhrs=("${assim_freq}") # increment file at middle of window @@ -64,14 +54,14 @@ else # construct restart times for deterministic member fi fi -# # Stage input files -# + +cd "${DATA}" || exit 1 # Create MDMD command file for fixed files -rm -f cmdfile.0 -touch cmdfile.0 -chmod 755 cmdfile.0 +rm -f cmdfile_in.0 +touch cmdfile_in.0 +chmod 755 cmdfile_in.0 # Append fixed files command file to master command file { @@ -91,80 +81,149 @@ for n in $(seq 1 "${ntiles}"); do echo "cpreq ${FIXorog}/${CASE_OUT}/${CASE_OUT}_grid.tile${n}.nc \ ${DATA}/${CASE_OUT}_grid.tile${n}.nc" done -} > cmdfile.0 +} >> cmdfile_in.0 -for imem in $(seq 1 "${NMEM_REGRID}"); do - cmem=$(printf %03i "${imem}") - memchar="mem${cmem}" +if [[ "${NMEM_REGRID}" -gt 1 ]]; then - # If deterministic job, COMOUT_ATMOS_ANALYSIS_MEM is just COMOUT_ATMOS_ANALYSIS - if (( NMEM_REGRID > 1 )); then - MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \ + echo "INFO: Preparing to regrid surface increments for ${NMEM_REGRID} ensemble members." + for imem in $(seq 1 "${NMEM_REGRID}"); do + + memdir=$(printf %03i "mem${imem}") + + MEMDIR=${memdir} YMD=${PDY} HH=${cyc} declare_from_tmpl \ COMIN_SOIL_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL + MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \ + COMOUT_ATMOS_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL + + # Create MPMD command file for this member + rm -f "cmdfile_in.${imem}" "cmdfile_out.${imem}" + touch "cmdfile_in.${imem}" "cmdfile_out.${imem}" + chmod 755 "cmdfile_in.${imem}" "cmdfile_out.${imem}" - memdir="${DATA}/${memchar}" - mkdir -p "${memdir}" + # Create commands to stage input files + { + echo "#!/bin/bash" - if [[ "${imem}" -gt 1 ]]; then - in_dir+=", " + echo "mkdir -p ${DATA}/${memdir}" + + for FHR in "${soilinc_fhrs[@]}"; do + echo "cpreq ${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}increment.sfc.i00${FHR}.nc \ + ${DATA}/${memdir}/sfci00${FHR}.nc" + done + + if [[ "${DO_LAND_IAU}" = ".true." ]]; then + for FHI in "${landifhrs[@]}"; do + echo "cpreq ${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}increment.sfc.i00${FHI}.nc \ + ${DATA}/${memdir}/sfci00${FHI}.nc" + done + fi + } >> "cmdfile_in.${imem}" + + # Create commands to copy output files + { + echo "#!/bin/bash" + + if [[ "${DO_LAND_IAU}" = ".false." || "${RUN}" == "gdas" || "${RUN}" == "gfs" ]]; then + for FHR in "${soilinc_fhrs[@]}"; do + for n in $(seq 1 "${ntiles}"); do + echo "cpfs ${DATA}/${memdir}/sfci00${FHR}.mem${imem}.tile${n}.nc \ + ${COMOUT_ATMOS_ANALYSIS_MEM}/increment.sfc.i00${FHR}.tile${n}.nc" + done + done fi - in_dir+="\"./${memchar}/\"" - else - # If deterministic job, memdir is just DATA - memdir="${DATA}" - in_dir="'./'" - fi + if [[ "${DO_LAND_IAU}" = ".true." ]]; then + for n in $(seq 1 "${ntiles}"); do + echo "cpfs ${DATA}/${memdir}/sfci.mem${imem}.tile${n}.nc \ + ${COMOUT_ATMOS_ANALYSIS_MEM}/increment.sfc.tile${n}.nc" + done + fi + } >> "cmdfile_out.${imem}" - # Create MPMD command file for this member - rm -f "cmdfile.${imem}" - touch "cmdfile.${imem}" - chmod 755 "cmdfile.${imem}" + done # for imem in $(seq 1 "${NMEM_REGRID}"); do + in_dir=$(seq -s ", " -f "'./mem%03g/'" 1 ${NMEM_REGRID}) - # Create commands to stage input files - { - echo "#!/bin/bash" +else # deterministic member only (NMEM_REGRID=1) + + echo "INFO: Preparing to regrid surface increments for deterministic member." + # Create commands to stage input files and append to the cmdfile.0 + { for FHR in "${soilinc_fhrs[@]}"; do - echo "cpreq ${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}increment.sfc.i00${FHR}.nc \ - ${memdir}/sfci00${FHR}.nc" - done + echo "cpreq ${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}ensmean_increment.sfc.i00${FHR}.nc \ + ${DATA}/sfci00${FHR}.nc" + done - if [[ "${DO_LAND_IAU}" = ".true." ]]; then + if [[ "${DO_LAND_IAU}" = ".true." ]]; then for FHI in "${landifhrs[@]}"; do - echo "cpreq ${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}increment.sfc.i00${FHI}.nc \ - ${memdir}/sfci00${FHI}.nc" + echo "cpreq ${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}ensmean_increment.sfc.i00${FHI}.nc \ + ${DATA}/sfci00${FHI}.nc" done fi - } > "cmdfile.${imem}" -done + } >> "cmdfile_in.0" + in_dir="'./'" + + # Create commands to copy output files + rm -f cmdfile_out.0 + touch cmdfile_out.0 + chmod 755 cmdfile_out.0 + { + echo "#!/bin/bash" + + if [[ "${DO_LAND_IAU}" = ".false." || "${RUN}" == "gdas" || "${RUN}" == "gfs" ]]; then + for FHR in "${soilinc_fhrs[@]}"; do + for n in $(seq 1 "${ntiles}"); do + echo "cpfs ${DATA}/sfci00${FHR}.mem1.tile${n}.nc \ + ${COMOUT_ATMOS_ANALYSIS_MEM}/increment.sfc.i00${FHR}.tile${n}.nc" + done + done + fi + + if [[ "${DO_LAND_IAU}" = ".true." ]]; then + for n in $(seq 1 "${ntiles}"); do + echo "cpfs ${DATA}/sfci.mem1.tile${n}.nc \ + ${COMOUT_ATMOS_ANALYSIS_MEM}/increment.sfc.tile${n}.nc" + done + fi + } >> "cmdfile_out.0" + -# Create master MPMD command file -rm -f cmdfile -touch cmdfile -chmod 755 cmdfile +fi + +# Create master MPMD command files for input and output +rm -f cmdfile_in cmdfile_out +touch cmdfile_in cmdfile_out # Append all members' command files to master command file -{ -echo "${DATA}/cmdfile.0" # fixed files -for imem in $(seq 1 "${NMEM_REGRID}"); do - echo "${DATA}/cmdfile.${imem}" -done -} >> cmdfile +if [[ -f cmdfile_in.0 ]]; then + echo "${DATA}/cmdfile_in.0" >> cmdfile_in +fi +if [[ -f cmdfile_out.0 ]]; then + echo "${DATA}/cmdfile_out.0" >> cmdfile_out +fi +if [[ "${NMEM_REGRID}" -gt 1 ]]; then + for imem in $(seq 1 "${NMEM_REGRID}"); do + if [[ -f "cmdfile_in.${imem}" ]]; then + echo "${DATA}/cmdfile_in.${imem}" >> cmdfile_in + fi + if [[ -f "cmdfile_out.${imem}" ]]; then + echo "${DATA}/cmdfile_out.${imem}" >> cmdfile_out + fi + done +fi # Run MPMD to stage input files -"${USHgfs}/run_mpmd.sh" "cmdfile" && true +"${USHgfs}/run_mpmd.sh" "cmdfile_in" && true export err=$? if [[ ${err} -ne 0 ]]; then - err_exit "run_mpmd.sh failed!" + err_exit "run_mpmd.sh failed to copy input and fix data!" fi +mv mpmd.out mpmd_in.out # Finish defining input/output directory list export out_dir="${in_dir}" -# -# Regrid soil increments and save to COMOUT -# +# Regrid soil increments # Increments for offline analysis # If land IAU --> deterministic only. If no land IAU --> both deterministic and ensemble @@ -180,10 +239,11 @@ if [[ "${DO_LAND_IAU}" = ".false." || "${RUN}" == "gdas" || "${RUN}" == "gfs" ]] atparse < "${regrid_nml_tmpl}" >> "regrid.nml" # Run regrid executable - ${APRUN_REGRID} "${REGRID_EXEC}" 1>"${PGMOUT}" 2>"${PGMERR}" + export pgm="${REGRID_EXEC}" + ${APRUN_REGRID} "${REGRID_EXEC}" export err=$? if [[ ${err} -ne 0 ]]; then - err_exit "${REGRID_EXEC} failed, ABORT!" + err_exit "${REGRID_EXEC} failed to regrid soil increments (without LANDIAU), ABORT!" fi done fi @@ -202,75 +262,19 @@ if [[ "${DO_LAND_IAU}" = ".true." ]]; then # Run regrid executable export pgm="${REGRID_EXEC}" - ${APRUN_REGRID} "${REGRID_EXEC}" 1>"${PGMOUT}" 2>"${PGMERR}" + ${APRUN_REGRID} "${REGRID_EXEC}" export err=$? if [[ ${err} -ne 0 ]]; then - err_exit "${pgm} failed, ABORT!" + err_exit "${pgm} failed to regrid soil increments (with LANDIAU), ABORT!" fi fi -# -# Save regridded files to COMOUT -# - -for imem in $(seq 1 "${NMEM_REGRID}"); do - cmem=$(printf %03i "${imem}") - memchar="mem${cmem}" - - # If deterministic job, COMOUT_ATMOS_ANALYSIS_MEM is just COMOUT_ATMOS_ANALYSIS - if (( NMEM_REGRID > 1 )); then - MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \ - COMOUT_ATMOS_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL - - memdir="${DATA}/${memchar}" - else - # If deterministic job, memdir is just DATA - memdir="${DATA}" - fi - - # Create MPMD command file for this member - rm -f "cmdfile.${imem}" - touch "cmdfile.${imem}" - chmod 755 "cmdfile.${imem}" - - { - echo "#!/bin/bash" - - if [[ "${DO_LAND_IAU}" = ".false." || "${RUN}" == "gdas" || "${RUN}" == "gfs" ]]; then - for FHR in "${soilinc_fhrs[@]}"; do - for n in $(seq 1 "${ntiles}"); do - echo "cpfs ${memdir}/sfci00${FHR}.mem${imem}.tile${n}.nc \ - ${COMOUT_ATMOS_ANALYSIS_MEM}/increment.sfc.i00${FHR}.tile${n}.nc" - done - done - fi - - if [[ "${DO_LAND_IAU}" = ".true." ]]; then - for n in $(seq 1 "${ntiles}"); do - echo "cpfs ${memdir}/sfci.mem${imem}.tile${n}.nc \ - ${COMOUT_ATMOS_ANALYSIS_MEM}/increment.sfc.tile${n}.nc" - done - fi - } > "cmdfile.${imem}" -done - -# Create master MPMD command file -rm -f cmdfile -touch cmdfile -chmod 755 cmdfile - -# Append all members' command files to master command file -{ -for imem in $(seq 1 "${NMEM_REGRID}"); do - echo "${DATA}/cmdfile.${imem}" -done -} >> cmdfile - # Run MPMD to save output files -"${USHgfs}/run_mpmd.sh" "cmdfile" && true +"${USHgfs}/run_mpmd.sh" "cmdfile_out" && true export err=$? if [[ ${err} -ne 0 ]]; then - err_exit "run_mpmd.sh failed!" + err_exit "run_mpmd.sh failed to copy output files to COMOUT, ABORT!" fi +mv mpmd.out mpmd_out.out exit 0 From 0e0c976208a79867ba2f35dd8d1a956adf64e938 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Mon, 24 Nov 2025 11:48:18 -0500 Subject: [PATCH 05/15] replace link w/ copy and fix COMOUT where it should be COMIN --- jobs/JGLOBAL_ATMOS_ANALYSIS_CALC | 1 + ush/gaussian_sfcanl.sh | 128 ++++++++++--------------------- 2 files changed, 41 insertions(+), 88 deletions(-) diff --git a/jobs/JGLOBAL_ATMOS_ANALYSIS_CALC b/jobs/JGLOBAL_ATMOS_ANALYSIS_CALC index cf59158117a..bfb5c74a3ad 100755 --- a/jobs/JGLOBAL_ATMOS_ANALYSIS_CALC +++ b/jobs/JGLOBAL_ATMOS_ANALYSIS_CALC @@ -28,6 +28,7 @@ RUN=${rCDUMP} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \ + COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \ COMIN_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ diff --git a/ush/gaussian_sfcanl.sh b/ush/gaussian_sfcanl.sh index 70780e63a8a..5f970bb8ffa 100755 --- a/ush/gaussian_sfcanl.sh +++ b/ush/gaussian_sfcanl.sh @@ -23,35 +23,12 @@ # CASE Forecast model and restart resolution. Defaults to C768. # CASE_HIST History file output resolution. Defaults to $CASE. # DONST Process NST fields when 'yes'. Default is 'no'. -# OUTPUT_FILE Output gaussian analysis file format. Default is "nemsio" -# Set to "netcdf" for netcdf output file -# Otherwise, output in nemsio. # FIXWGTS Weight file to use for interpolation -# DATA Working directory -# (if nonexistent will be made, used and deleted) -# Defaults to current working directory # COMOUT Output directory # (if nonexistent will be made) # defaults to current working directory -# XC Suffix to add to executables. Defaults to none. # GAUSFCANLEXE Program executable. # Defaults to $EXECgfs/gaussian_sfcanl.x -# INISCRIPT Preprocessing script. Defaults to none. -# LOGSCRIPT Log posting script. Defaults to none. -# ENDSCRIPT Postprocessing script -# defaults to none -# PGMOUT Executable standard output -# defaults to $pgmout, then to '&1' -# PGMERR Executable standard error -# defaults to $pgmerr, then to '&1' -# pgmout Executable standard output default -# pgmerr Executable standard error default -# REDOUT standard output redirect ('1>' or '1>>') -# defaults to '1>', or to '1>>' to append if $PGMOUT is a file -# REDERR standard error redirect ('2>' or '2>>') -# defaults to '2>', or to '2>>' to append if $PGMERR is a file -# VERBOSE Verbose flag (YES or NO) -# defaults to NO # gfs_ver Version number of gfs directory. Default is # v15.0.0. # OMP_NUM_ @@ -60,15 +37,11 @@ # Default is none. # # Exported Shell Variables: -# PGM Current program name # pgm -# ERR Last return code # err # # Modules and files referenced: -# scripts : $INISCRIPT -# $LOGSCRIPT -# $ENDSCRIPT +# scripts : # # programs : $GAUSFCANLEXE # @@ -77,10 +50,9 @@ # ${FIXgfs}/am/global_hyblev.l65.txt # # input data : ${COMIN_ATMOS_RESTART}/${PDY}.${cyc}0000.sfcanl_data.tile*.nc +# ${COMIN_ATMOS_ANALYSIS}/increment.sfc.i006.tile${i}.nc" # -# output data: $PGMOUT -# $PGMERR -# $COMOUT/${APREFIX}analysis.sfc.a006.nc +# output data: $COMOUT/${APREFIX}analysis.sfc.a006.nc # # Remarks: # @@ -95,7 +67,6 @@ # # Attributes: # Language: POSIX shell -# Machine: IBM SP # ################################################################################ @@ -112,86 +83,61 @@ LEVSP1=$(($LEVS+1)) FIXWGTS=${FIXWGTS:-${FIXorog}/${CASE}/fv3_SCRIP_${CASE}_GRIDSPEC_lon${LONB_SFC}_lat${LATB_SFC}.gaussian.neareststod.nc} # Filenames. -XC=${XC:-} GAUSFCANLEXE=${GAUSFCANLEXE:-$EXECgfs/gaussian_sfcanl.x} SIGLEVEL=${SIGLEVEL:-${FIXgfs}/am/global_hyblev.l${LEVSP1}.txt} # Other variables. -export PGMOUT=${PGMOUT:-${pgmout:-'&1'}} -export PGMERR=${PGMERR:-${pgmerr:-'&2'}} -export REDOUT=${REDOUT:-'1>'} -export REDERR=${REDERR:-'2>'} # Set defaults ################################################################################ # Preprocessing -${INISCRIPT:-} -pwd=$(pwd) -if [[ ! -d "${COMOUT_ATMOS_ANALYSIS}" ]]; then - mkdir -p "${COMOUT_ATMOS_ANALYSIS}" -fi ################################################################################ # Make surface analysis -export PGM=$GAUSFCANLEXE -export pgm=$PGM -$LOGSCRIPT - -iy=${PDY:0:4} -im=${PDY:4:2} -id=${PDY:6:2} -ih=${cyc} - -export OMP_NUM_THREADS=${OMP_NUM_THREADS_SFC:-1} - # input interpolation weights -${NLN} "${FIXWGTS}" "./weights.nc" +cpreq "${FIXWGTS}" "./weights.nc" # input analysis tiles (with nst records) -${NLN} "${COMIN_ATMOS_RESTART}/${PDY}.${cyc}0000.sfcanl_data.tile1.nc" "./anal.tile1.nc" -${NLN} "${COMIN_ATMOS_RESTART}/${PDY}.${cyc}0000.sfcanl_data.tile2.nc" "./anal.tile2.nc" -${NLN} "${COMIN_ATMOS_RESTART}/${PDY}.${cyc}0000.sfcanl_data.tile3.nc" "./anal.tile3.nc" -${NLN} "${COMIN_ATMOS_RESTART}/${PDY}.${cyc}0000.sfcanl_data.tile4.nc" "./anal.tile4.nc" -${NLN} "${COMIN_ATMOS_RESTART}/${PDY}.${cyc}0000.sfcanl_data.tile5.nc" "./anal.tile5.nc" -${NLN} "${COMIN_ATMOS_RESTART}/${PDY}.${cyc}0000.sfcanl_data.tile6.nc" "./anal.tile6.nc" +cpreq "${COMIN_ATMOS_RESTART}/${PDY}.${cyc}0000.sfcanl_data.tile1.nc" "./anal.tile1.nc" +cpreq "${COMIN_ATMOS_RESTART}/${PDY}.${cyc}0000.sfcanl_data.tile2.nc" "./anal.tile2.nc" +cpreq "${COMIN_ATMOS_RESTART}/${PDY}.${cyc}0000.sfcanl_data.tile3.nc" "./anal.tile3.nc" +cpreq "${COMIN_ATMOS_RESTART}/${PDY}.${cyc}0000.sfcanl_data.tile4.nc" "./anal.tile4.nc" +cpreq "${COMIN_ATMOS_RESTART}/${PDY}.${cyc}0000.sfcanl_data.tile5.nc" "./anal.tile5.nc" +cpreq "${COMIN_ATMOS_RESTART}/${PDY}.${cyc}0000.sfcanl_data.tile6.nc" "./anal.tile6.nc" # input orography tiles -${NLN} "${FIXorog}/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile1.nc" "./orog.tile1.nc" -${NLN} "${FIXorog}/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile2.nc" "./orog.tile2.nc" -${NLN} "${FIXorog}/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile3.nc" "./orog.tile3.nc" -${NLN} "${FIXorog}/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile4.nc" "./orog.tile4.nc" -${NLN} "${FIXorog}/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile5.nc" "./orog.tile5.nc" -${NLN} "${FIXorog}/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile6.nc" "./orog.tile6.nc" - -${NLN} "${SIGLEVEL}" "./vcoord.txt" +cpreq "${FIXorog}/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile1.nc" "./orog.tile1.nc" +cpreq "${FIXorog}/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile2.nc" "./orog.tile2.nc" +cpreq "${FIXorog}/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile3.nc" "./orog.tile3.nc" +cpreq "${FIXorog}/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile4.nc" "./orog.tile4.nc" +cpreq "${FIXorog}/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile5.nc" "./orog.tile5.nc" +cpreq "${FIXorog}/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile6.nc" "./orog.tile6.nc" -# output gaussian global surface analysis files -${NLN} "${COMOUT_ATMOS_ANALYSIS}/${APREFIX}analysis.sfc.a006.nc" "./sfc.gaussian.analysis.file" +cpreq "${SIGLEVEL}" "./vcoord.txt" # Namelist uses booleans now -if [[ ${DONST} == "YES" ]]; then do_nst='.true.'; else do_nst='.false.'; fi +if [[ "${DONST}" == "YES" ]]; then + do_nst=".true." +else + do_nst=".false." +fi -#Add soil increments to gdas gaussian sfcanal if they are not added by gcycle (i.e., when landiau=true) +# Add soil increments to gdas gaussian sfcanal if they are not added by gcycle (i.e., when landiau=true) LSOIL_INCR=${LSOIL_INCR:-2} -if [[ "${DO_LAND_IAU:-.false.}" == ".true." ]]; then +if [[ "${DO_LAND_IAU:-.false.}" == ".true." ]]; then for i in $(seq 1 6); do - sfc_inc="${COMOUT_ATMOS_ANALYSIS}/increment.sfc.i006.tile${i}.nc" - if [[ ! -f "${sfc_inc}" ]]; then - echo "Error! gaussian sfc analysis missing increment file ${sfc_inc}" - exit 1 - else - ${NLN} "${sfc_inc}" "./sfc_inc.tile${i}.nc" - fi + sfc_inc="${COMIN_ATMOS_ANALYSIS}/increment.sfc.i006.tile${i}.nc" + cpreq "${sfc_inc}" "./sfc_inc.tile${i}.nc" done fi # Executable namelist cat < fort.41 - &setup - yy=${iy}, - mm=${im}, - dd=${id}, - hh=${ih}, +&setup + yy=${PDY:0:4}, + mm=${PDY:4:2}, + dd=${PDY:6:2}, + hh=${cyc}, igaus=${LONB_SFC}, jgaus=${LATB_SFC}, donst=${do_nst}, @@ -200,19 +146,25 @@ cat < fort.41 add_soil_inc=${DO_LAND_IAU}, lsoil_incr=${LSOIL_INCR}, sfc_inc_file="./sfc_inc", - / +/ EOF +cat fort.41 +export pgm="${GAUSFCANLEXE}" +export OMP_NUM_THREADS=${OMP_NUM_THREADS_SFC:-1} ${APRUNSFC} "${GAUSFCANLEXE}" - export err=$? if [[ ${err} -ne 0 ]]; then echo "FATAL ERROR: ${GAUSFCANLEXE} returned non-zero exit status!" exit "${err}" fi +# output gaussian global surface analysis files +if [[ -f "sfc.gaussian.analysis.file" ]]; then + cpfs "./sfc.gaussian.analysis.file" "${COMOUT_ATMOS_ANALYSIS}/${APREFIX}analysis.sfc.a006.nc" +fi + ################################################################################ # Postprocessing -cd "${pwd}" exit 0 From bc2365340307c66daf247779acb556da2ce91cf5 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Mon, 24 Nov 2025 13:18:59 -0500 Subject: [PATCH 06/15] bugfix --- ush/regrid_gsiSfcIncr_to_tile.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ush/regrid_gsiSfcIncr_to_tile.sh b/ush/regrid_gsiSfcIncr_to_tile.sh index 67f875217f0..99d29ce72ac 100755 --- a/ush/regrid_gsiSfcIncr_to_tile.sh +++ b/ush/regrid_gsiSfcIncr_to_tile.sh @@ -88,11 +88,10 @@ if [[ "${NMEM_REGRID}" -gt 1 ]]; then echo "INFO: Preparing to regrid surface increments for ${NMEM_REGRID} ensemble members." for imem in $(seq 1 "${NMEM_REGRID}"); do - memdir=$(printf %03i "mem${imem}") + memdir=$(printf "mem%03i" "${imem}") MEMDIR=${memdir} YMD=${PDY} HH=${cyc} declare_from_tmpl \ - COMIN_SOIL_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL - MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \ + COMIN_SOIL_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL \ COMOUT_ATMOS_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL # Create MPMD command file for this member From 985975cbefa0549dd74e9e9eca111bfd75547a9c Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Mon, 24 Nov 2025 13:44:21 -0500 Subject: [PATCH 07/15] fix archive yamls for name fixes --- parm/archive/enkf.yaml.j2 | 10 +++++----- parm/archive/gcafs_arcdir.yaml.j2 | 2 +- parm/archive/gfs_arcdir.yaml.j2 | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2 index 57205315f90..0e2d9a45f78 100644 --- a/parm/archive/enkf.yaml.j2 +++ b/parm/archive/enkf.yaml.j2 @@ -67,10 +67,10 @@ enkf: # Ensemble mean state {% if not DO_JEDIATMENS %} {% set da_stat_files = ["enkfstat.txt", - "gsistat.ensmean.tar", - "cnvstat.ensmean.tar", - "oznstat.ensmean.tar", - "radstat.ensmean.tar"] %} + "gsistat_ensmean.txt", + "cnvstat_ensmean.tar", + "oznstat_ensmean.tar", + "radstat_ensmean.tar"] %} {% set da_conf_files = [] %} {% else %} {% if lobsdiag_forenkf %} @@ -107,7 +107,7 @@ enkf: # soil DA increments {% if DO_GSISOILDA %} {% for fhr in IAUFHRS %} - - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}increment.sfc.i{{ '%03d' % fhr }}.nc" + - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}ensmean_increment.sfc.i{{ '%03d' % fhr }}.nc" {% endfor %} {% endif %} diff --git a/parm/archive/gcafs_arcdir.yaml.j2 b/parm/archive/gcafs_arcdir.yaml.j2 index e1d84d89960..325ffba80e6 100644 --- a/parm/archive/gcafs_arcdir.yaml.j2 +++ b/parm/archive/gcafs_arcdir.yaml.j2 @@ -83,7 +83,7 @@ {% else %} {% do enkf_files.append([COMIN_ATMOS_ANALYSIS_ENSSTAT ~ "/" ~ head ~ "enkfstat.txt", ARCDIR ~ "/enkfstat." ~ RUN ~ "." ~ cycle_YMDH ]) %} - {% do enkf_files.append([COMIN_ATMOS_ANALYSIS_ENSSTAT ~ "/" ~ head ~ "gsistat.ensmean.tar", + {% do enkf_files.append([COMIN_ATMOS_ANALYSIS_ENSSTAT ~ "/" ~ head ~ "gsistat_ensmean.txt", ARCDIR ~ "/gsistat." ~ RUN ~ "." ~ cycle_YMDH ~ ".ensmean"]) %} {% endif %} diff --git a/parm/archive/gfs_arcdir.yaml.j2 b/parm/archive/gfs_arcdir.yaml.j2 index c7b50b7d812..a5bbbd7ac63 100644 --- a/parm/archive/gfs_arcdir.yaml.j2 +++ b/parm/archive/gfs_arcdir.yaml.j2 @@ -130,7 +130,7 @@ {% else %} {% do enkf_files.append([COMIN_ATMOS_ANALYSIS_ENSSTAT ~ "/" ~ head ~ "enkfstat.txt", ARCDIR ~ "/enkfstat." ~ RUN ~ "." ~ cycle_YMDH ]) %} - {% do enkf_files.append([COMIN_ATMOS_ANALYSIS_ENSSTAT ~ "/" ~ head ~ "gsistat.ensmean.tar", + {% do enkf_files.append([COMIN_ATMOS_ANALYSIS_ENSSTAT ~ "/" ~ head ~ "gsistat_ensmean.txt", ARCDIR ~ "/gsistat." ~ RUN ~ "." ~ cycle_YMDH ~ ".ensmean"]) %} {% endif %} From 8b80a13547ea9df00c88dd08561582393e475acf Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Mon, 24 Nov 2025 14:22:22 -0500 Subject: [PATCH 08/15] more bugfixes --- parm/archive/gdas_restarta.yaml.j2 | 2 +- scripts/exgdas_enkf_update.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2 index b581c76b986..ef72fd97d38 100644 --- a/parm/archive/gdas_restarta.yaml.j2 +++ b/parm/archive/gdas_restarta.yaml.j2 @@ -47,7 +47,7 @@ gdas_restarta: - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}cnvstat.tar" - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_int.txt" - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}analysis.dtf.a006.nc" - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}increment.done.txt" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}analysis.done.txt" {% else %} - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}rad_varbc_params.tar" {% endif %} diff --git a/scripts/exgdas_enkf_update.sh b/scripts/exgdas_enkf_update.sh index 9253c69a110..9e066ef3305 100755 --- a/scripts/exgdas_enkf_update.sh +++ b/scripts/exgdas_enkf_update.sh @@ -176,7 +176,7 @@ for FHR in ${nfhrs}; do if [[ "${DO_GSISOILDA}" == "YES" ]]; then ${NLN} "${COMIN_ATMOS_HISTORY_STAT_PREV}/${GPREFIX}ensmean.sfc.f00${FHR}.nc" \ "bfg_${PDY}${cyc}_fhr0${FHR}_ensmean" - ${NLN} "${COMOUT_ATMOS_ANALYSIS_STAT}/${APREFIX}ensmean_increment.sfc.i00${FHR}.nc" + ${NLN} "${COMOUT_ATMOS_ANALYSIS_STAT}/${APREFIX}ensmean_increment.sfc.i00${FHR}.nc" \ "sfcincr_${PDY}${cyc}_fhr0${FHR}_ensmean" fi From 5ed09a528f9da577e3a90b4f07099a1b9d39e852 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Mon, 24 Nov 2025 14:28:21 -0500 Subject: [PATCH 09/15] pull our analdiag and ediag in its own env/ section --- env/GAEAC6.env | 3 +-- env/HERA.env | 4 ++++ env/HERCULES.env | 5 ++++- env/ORION.env | 6 +++++- env/URSA.env | 4 ++++ env/WCOSS2.env | 4 ++++ 6 files changed, 22 insertions(+), 4 deletions(-) diff --git a/env/GAEAC6.env b/env/GAEAC6.env index 0e9913a01d2..f0ea8a85818 100755 --- a/env/GAEAC6.env +++ b/env/GAEAC6.env @@ -203,8 +203,7 @@ case ${step} in ntasks_gausfcanl=${ntasks_gausfcanl:-1} export APRUN_GAUSFCANL="${launcher} -n ${ntasks_gausfcanl} --cpus-per-task=${NTHREADS_GAUSFCANL}" ;; - - "analdiag" | "ediag") + "analdiag" | "ediag") export USE_CFP="YES" ;; "offlineanl") diff --git a/env/HERA.env b/env/HERA.env index 57aefcc66af..2e972f059ec 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -208,6 +208,10 @@ elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then ntasks_gausfcanl=${ntasks_gausfcanl:-1} export APRUN_GAUSFCANL="${launcher} -n ${ntasks_gausfcanl} --cpus-per-task=${NTHREADS_GAUSFCANL}" +elif [[ "${step}" = "analdiag" ]] || [[ "${step}" = "ediag" ]]; then + + export USE_CFP="YES" + elif [[ "${step}" = "offlineanl" ]]; then export NTHREADS_CHGRES=${threads_per_task_chgres:-12} diff --git a/env/HERCULES.env b/env/HERCULES.env index 46a6db49216..75dd8e4070a 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -162,7 +162,7 @@ case ${step} in "anlstat") export NTHREADS_ANLSTAT=${NTHREADSmax} - export APRUN_ANLSTAT="${APRUN_default} --cpus-per-task=${NTHREADS_ANLSTAT}" + export APRUN_ANLSTAT="${APRUN_default} --cpus-per-task=${NTHREADS_ANLSTAT}" ;; "ecen_fv3jedi") @@ -222,6 +222,9 @@ case ${step} in # REGRID requires 6 tasks for reproducibility ntasks_regrid=6 export APRUN_REGRID="${launcher} -n ${ntasks_regrid} " + ;; + "analdiag" | "ediag") + export USE_CFP="YES" ;; "offlineanl") diff --git a/env/ORION.env b/env/ORION.env index 10f896037ab..3d150ef6f2f 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -162,7 +162,7 @@ elif [[ "${step}" = "marineanlletkf" ]]; then elif [[ "${step}" = "anlstat" ]]; then export NTHREADS_ANLSTAT=${NTHREADSmax} - export APRUN_ANLSTAT="${APRUN_default} --cpus-per-task=${NTHREADS_ANLSTAT}" + export APRUN_ANLSTAT="${APRUN_default} --cpus-per-task=${NTHREADS_ANLSTAT}" elif [[ "${step}" = "ecen_fv3jedi" ]]; then @@ -221,6 +221,10 @@ elif [[ "${step}" = "sfcanl" ]]; then ntasks_regrid=6 export APRUN_REGRID="${launcher} -n ${ntasks_regrid} " +elif [[ "${step}" = "analdiag" ]] || [[ "${step}" = "ediag" ]]; then + + export USE_CFP="YES" + elif [[ "${step}" = "offlineanl" ]]; then export NTHREADS_CHGRES=${threads_per_task:-12} diff --git a/env/URSA.env b/env/URSA.env index 1b952df431c..4759a83c0b4 100644 --- a/env/URSA.env +++ b/env/URSA.env @@ -197,6 +197,10 @@ elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then ntasks_gausfcanl=${ntasks_gausfcanl:-1} export APRUN_GAUSFCANL="${launcher} -n ${ntasks_gausfcanl} --cpus-per-task=${NTHREADS_GAUSFCANL}" +elif [[ "${step}" = "analdiag" ]] || [[ "${step}" = "ediag" ]]; then + + export USE_CFP="YES" + elif [[ "${step}" = "offlineanl" ]]; then export NTHREADS_CHGRES=${threads_per_task_chgres:-12} diff --git a/env/WCOSS2.env b/env/WCOSS2.env index b0da0aa559a..0edbb100308 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -210,6 +210,10 @@ elif [[ "${step}" = "sfcanl" ]]; then ntasks_regrid=6 export APRUN_REGRID="${launcher} -n ${ntasks_regrid} " +elif [[ "${step}" = "analdiag" ]] || [[ "${step}" = "ediag" ]]; then + + export USE_CFP="YES" + elif [[ "${step}" = "offlineanl" ]]; then export NTHREADS_CHGRES=${threads_per_task_chgres:-14} From 0500ea98a7ccacf2109d7b70d719917f0eef43ba Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Mon, 24 Nov 2025 15:11:56 -0500 Subject: [PATCH 10/15] shellcheck series of fixes starting now --- jobs/JGDAS_ENKF_UPDATE | 1 + ush/regrid_gsiSfcIncr_to_tile.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/jobs/JGDAS_ENKF_UPDATE b/jobs/JGDAS_ENKF_UPDATE index 15ab2512c39..2370b18ac8e 100755 --- a/jobs/JGDAS_ENKF_UPDATE +++ b/jobs/JGDAS_ENKF_UPDATE @@ -9,6 +9,7 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "eupd" -c "base anal eupd" ############################################## # Begin JOB SPECIFIC work ############################################## +# shellcheck disable=SC2155 export GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") export GDUMP="enkfgdas" # EnKF ensemble always uses RUN=enkfgdas for prior members diff --git a/ush/regrid_gsiSfcIncr_to_tile.sh b/ush/regrid_gsiSfcIncr_to_tile.sh index 99d29ce72ac..5e2fe1cf59f 100755 --- a/ush/regrid_gsiSfcIncr_to_tile.sh +++ b/ush/regrid_gsiSfcIncr_to_tile.sh @@ -140,7 +140,7 @@ if [[ "${NMEM_REGRID}" -gt 1 ]]; then } >> "cmdfile_out.${imem}" done # for imem in $(seq 1 "${NMEM_REGRID}"); do - in_dir=$(seq -s ", " -f "'./mem%03g/'" 1 ${NMEM_REGRID}) + in_dir=$(seq -s ", " -f "'./mem%03g/'" 1 "${NMEM_REGRID}") else # deterministic member only (NMEM_REGRID=1) From 74643213dd45c5abf5d877c241be28c5dc7517de Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Mon, 24 Nov 2025 15:17:58 -0500 Subject: [PATCH 11/15] more shellcheck fixes --- scripts/exglobal_atmos_analysis.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/exglobal_atmos_analysis.sh b/scripts/exglobal_atmos_analysis.sh index 2b8a30dfe7d..b5e63c1c5c2 100755 --- a/scripts/exglobal_atmos_analysis.sh +++ b/scripts/exglobal_atmos_analysis.sh @@ -20,7 +20,6 @@ # Set environment. # Directories. -pwd=$(pwd) # Base variables rCDUMP=${rCDUMP:-"gdas"} From 507a4c2e58deef63266dc5e7698b3291576f7241 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Wed, 26 Nov 2025 16:02:56 -0500 Subject: [PATCH 12/15] fixes after merging develop --- scripts/exglobal_diag.sh | 67 +++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 32 deletions(-) diff --git a/scripts/exglobal_diag.sh b/scripts/exglobal_diag.sh index 3e4538ac7e7..48119092bb6 100755 --- a/scripts/exglobal_diag.sh +++ b/scripts/exglobal_diag.sh @@ -29,10 +29,10 @@ COMPRESS=${COMPRESS:-gzip} # Analysis files APREFIX=${APREFIX:-"${RUN}.t${cyc}z."} -RADSTAT=${RADSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}radstat.tar} -PCPSTAT=${PCPSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}pcpstat} CNVSTAT=${CNVSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}cnvstat.tar} +PCPSTAT=${PCPSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}pcpstat} OZNSTAT=${OZNSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}oznstat.tar} +RADSTAT=${RADSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}radstat.tar} # Remove stat file if file already exists rm -f "${RADSTAT}" "${PCPSTAT}" "${CNVSTAT}" "${OZNSTAT}" @@ -42,14 +42,17 @@ GENDIAG=${GENDIAG:-"YES"} GSIDIAG=${GSIDIAG:-"${COMIN_ATMOS_ANALYSIS}/${APREFIX}gsidiags${DIAG_SUFFIX:-}.tar"} USE_BUILD_GSINFO=${USE_BUILD_GSINFO:-"NO"} DIAG_COMPRESS=${DIAG_COMPRESS:-"YES"} +if [[ "${DIAG_COMPRESS:-}" == "YES" ]]; then + COMPRESS_SUFFIX=".gz" +fi DIAG_TARBALL=${DIAG_TARBALL:-"YES"} # Set script / GSI control parameters ################################################################################ -if [[ "${GENDIAG}" != "YES" ]]; then - echo "INFO: GENDIAG set to NO. Skipping diagnostic file generation." - exit 0 +if [[ "${GENDIAG}" != "YES" ]] ; then + echo "INFO: GENDIAG set to NO. Skipping diagnostic file generation." + exit 0 fi ################################################################################ @@ -58,13 +61,15 @@ cpreq "${GSIDIAG}" ./gsidiags.tar tar -xvf gsidiags.tar export err=$? if [[ ${err} -ne 0 ]]; then - err_exit "Unable to unpack gsidiags.tar file!" + err_exit "Unable to unpack gsidiags.tar file!" fi # Set up lists and variables for various types of diagnostic files. +ntype=3 + diagtype[0]="conv conv_gps conv_ps conv_pw conv_q conv_sst conv_t conv_tcp conv_uv conv_spd" diagtype[1]="pcp_ssmi_dmsp pcp_tmi_trmm" -if [[ "${USE_BUILD_GSINFO}" == "YES" ]]; then +if [[ ${USE_BUILD_GSINFO} == "YES" ]]; then diagtype[2]=$(cat "${BUILD_GSINFO_DIR}/ozinfo/satellites") diagtype[3]=$(cat "${BUILD_GSINFO_DIR}/satinfo/satellites") else @@ -72,6 +77,7 @@ else diagtype[3]="msu_n14 sndr_g08 sndr_g11 sndr_g12 sndr_g13 sndr_g08_prep sndr_g11_prep sndr_g12_prep sndr_g13_prep sndrd1_g11 sndrd2_g11 sndrd3_g11 sndrd4_g11 sndrd1_g12 sndrd2_g12 sndrd3_g12 sndrd4_g12 sndrd1_g13 sndrd2_g13 sndrd3_g13 sndrd4_g13 sndrd1_g14 sndrd2_g14 sndrd3_g14 sndrd4_g14 sndrd1_g15 sndrd2_g15 sndrd3_g15 sndrd4_g15 amsua_n15 amsua_n16 amsua_n17 amsub_n15 amsub_n16 amsub_n17 hsb_aqua airs_aqua amsua_aqua imgr_g08 imgr_g11 imgr_g12 imgr_g14 imgr_g15 ssmi_f13 ssmi_f15 amsua_n18 amsua_metop-a mhs_n18 mhs_metop-a amsre_low_aqua amsre_mid_aqua amsre_hig_aqua ssmis_f16 ssmis_f17 ssmis_f18 ssmis_f19 ssmis_f20 iasi_metop-a amsua_n19 mhs_n19 seviri_m08 seviri_m09 seviri_m10 seviri_m11 cris_npp cris-fsr_npp cris-fsr_n20 atms_npp atms_n20 amsua_metop-b mhs_metop-b iasi_metop-b avhrr_metop-b avhrr_n18 avhrr_n19 avhrr_metop-a amsr2_gcom-w1 gmi_gpm saphir_meghat ahi_himawari8 abi_g16 abi_g17 amsua_metop-c mhs_metop-c iasi_metop-c avhrr_metop-c viirs-m_npp viirs-m_j1 abi_g18 ahi_himawari9 viirs-m_j2 cris-fsr_n21 atms_n21 abi_g19" fi + diaglist[0]=listcnv diaglist[1]=listpcp diaglist[2]=listozn @@ -82,10 +88,13 @@ diagfile[1]="${PCPSTAT}" diagfile[2]="${OZNSTAT}" diagfile[3]="${RADSTAT}" -prefix="dir.*/" +numfile[0]=0 +numfile[1]=0 +numfile[2]=0 +numfile[3]=0 rm -f "${DATA}/diag.sh" -cat >"${DATA}/diag.sh" < "${DATA}/diag.sh" << EOF #!/bin/bash set -x @@ -133,28 +142,29 @@ touch cmdfile loops="01 03" for loop in ${loops}; do case ${loop} in - 01) string=ges ;; - 03) string=anl ;; - *) string=${loop} ;; + 01) string=ges;; + 03) string=anl;; + *) string=${loop};; esac echo "START loop ${string}" - n=0 - while [[ ${n} -lt ${#diagtype[@]} ]]; do + n=-1 + while [[ ${n} -lt ${ntype} ]] ;do + n=$(( n + 1 )) for type in ${diagtype[n]}; do - # shellcheck disable=SC2312 - count=$(find -L ./ -path "./${prefix}${type}_${loop}*" -type f -printf '.' | wc -c) + #shellcheck disable=SC2012,SC2312 + count=$(ls dir.*/"${type}_${loop}"* 2>/dev/null | wc -l) if [[ ${count} -eq 0 ]]; then continue fi - echo "${DATA}/diag.sh ${type} ${loop} ${string} ${count} ${DIAG_SUFFIX:-}.nc4" >>cmdfile - echo "diag_${type}_${string}.${PDY}${cyc}*" >>"${diaglist[n]}" + echo "${DATA}/diag.sh ${type} ${loop} ${string} ${count} ${DIAG_SUFFIX:-}.nc4" >> cmdfile + echo "diag_${type}_${string}.${PDY}${cyc}${DIAG_SUFFIX:-}.nc4${COMPRESS_SUFFIX:-}" >> "${diaglist[n]}" + numfile[n]=$(( numfile[n] + 1 )) done - n=$((n + 1)) done echo "END loop ${string}" done -ncmd=$(wc -l Date: Mon, 1 Dec 2025 13:57:22 -0500 Subject: [PATCH 13/15] after effects of shellcheck PR still hitting --- scripts/exgdas_enkf_update.sh | 2 +- ush/gaussian_sfcanl.sh | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/exgdas_enkf_update.sh b/scripts/exgdas_enkf_update.sh index 0c1074e4a0c..a6acf52e4cc 100755 --- a/scripts/exgdas_enkf_update.sh +++ b/scripts/exgdas_enkf_update.sh @@ -177,7 +177,7 @@ for FHR in ${nfhrs}; do if [[ "${DO_GSISOILDA}" == "YES" ]]; then ${NLN} "${COMIN_ATMOS_HISTORY_STAT_PREV}/${GPREFIX}ensmean.sfc.f00${FHR}.nc" \ "bfg_${PDY}${cyc}_fhr0${FHR}_ensmean" - ${NLN} "${COMIN_ATMOS_ANALYSIS_STAT}/${APREFIX}increment.sfc.i00${FHR}.nc" \ + ${NLN} "${COMIN_ATMOS_ANALYSIS_STAT}/${APREFIX}ensmean_increment.sfc.i00${FHR}.nc" \ "sfcincr_${PDY}${cyc}_fhr0${FHR}_ensmean" fi done diff --git a/ush/gaussian_sfcanl.sh b/ush/gaussian_sfcanl.sh index 5f970bb8ffa..0652c0117c5 100755 --- a/ush/gaussian_sfcanl.sh +++ b/ush/gaussian_sfcanl.sh @@ -75,15 +75,15 @@ CASE_HIST=${CASE_HIST:-${CASE}} resh=${CASE_HIST:1} LONB_CASE=$((resh*4)) LATB_CASE=$((resh*2)) -LONB_SFC=${LONB_SFC:-$LONB_CASE} -LATB_SFC=${LATB_SFC:-$LATB_CASE} +LONB_SFC=${LONB_SFC:-${LONB_CASE}} +LATB_SFC=${LATB_SFC:-${LATB_CASE}} DONST=${DONST:-"NO"} LEVS=${LEVS:-64} -LEVSP1=$(($LEVS+1)) +LEVSP1=$(( LEVS + 1 )) FIXWGTS=${FIXWGTS:-${FIXorog}/${CASE}/fv3_SCRIP_${CASE}_GRIDSPEC_lon${LONB_SFC}_lat${LATB_SFC}.gaussian.neareststod.nc} # Filenames. -GAUSFCANLEXE=${GAUSFCANLEXE:-$EXECgfs/gaussian_sfcanl.x} +GAUSFCANLEXE=${GAUSFCANLEXE:-${EXECgfs}/gaussian_sfcanl.x} SIGLEVEL=${SIGLEVEL:-${FIXgfs}/am/global_hyblev.l${LEVSP1}.txt} # Other variables. From d52f323f4653fb0ef6385f6bbb14c538a29757ae Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Wed, 3 Dec 2025 09:55:53 -0500 Subject: [PATCH 14/15] apply shfmt suggested changes --- jobs/JGDAS_ENKF_UPDATE | 2 +- scripts/exgdas_enkf_update.sh | 26 +++++++++++++------------- scripts/exglobal_atmos_analysis.sh | 2 +- scripts/exglobal_diag.sh | 21 ++++++++++----------- 4 files changed, 25 insertions(+), 26 deletions(-) diff --git a/jobs/JGDAS_ENKF_UPDATE b/jobs/JGDAS_ENKF_UPDATE index eb91d875927..43d39b21735 100755 --- a/jobs/JGDAS_ENKF_UPDATE +++ b/jobs/JGDAS_ENKF_UPDATE @@ -11,7 +11,7 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "eupd" -c "base anal eupd" ############################################## # shellcheck disable=SC2155 export GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") -export GDUMP="enkfgdas" # EnKF ensemble always uses RUN=enkfgdas for prior members +export GDUMP="enkfgdas" # EnKF ensemble always uses RUN=enkfgdas for prior members export APREFIX="${RUN}.t${cyc}z." export GPREFIX="${GDUMP}.t${GDATE:8:2}z." diff --git a/scripts/exgdas_enkf_update.sh b/scripts/exgdas_enkf_update.sh index a6acf52e4cc..c44ba3e2bb9 100755 --- a/scripts/exgdas_enkf_update.sh +++ b/scripts/exgdas_enkf_update.sh @@ -65,7 +65,7 @@ hofx_2m_sfcfile=${hofx_2m_sfcfile:-".false."} ATMGES_ENSMEAN="${COMIN_ATMOS_HISTORY_STAT_PREV}/${GPREFIX}ensmean.atm.f006.nc" LONB_ENKF=${LONB_ENKF:-$(${NCLEN} "${ATMGES_ENSMEAN}" grid_xt)} # get LONB_ENKF LATB_ENKF=${LATB_ENKF:-$(${NCLEN} "${ATMGES_ENSMEAN}" grid_yt)} # get LATB_ENFK -LEVS_ENKF=${LEVS_ENKF:-$(${NCLEN} "${ATMGES_ENSMEAN}" pfull)} # get LEVS_ENFK +LEVS_ENKF=${LEVS_ENKF:-$(${NCLEN} "${ATMGES_ENSMEAN}" pfull)} # get LEVS_ENFK WRITE_INCR_ZERO="incvars_to_zero= ${INCREMENTS_TO_ZERO}," if [[ "${DO_CALC_INCREMENT}" == "YES" ]]; then write_fv3_incr=".false." @@ -89,11 +89,11 @@ fi ################################################################################ # Fixed files -cpreq "${SATANGL}" satbias_angle -cpreq "${SCANINFO}" scaninfo +cpreq "${SATANGL}" satbias_angle +cpreq "${SCANINFO}" scaninfo cpreq "${HYBENSINFO}" hybens_info -cpreq "${ANAVINFO}" anavinfo -cpreq "${VLOCALEIG}" vlocal_eig.dat +cpreq "${ANAVINFO}" anavinfo +cpreq "${VLOCALEIG}" vlocal_eig.dat if [[ "${SATINFO}" == "generate" ]]; then "${USHgfs}/create_gsi_info.sh" sat "${PDY}${cyc}" "${DATA}" else @@ -124,15 +124,15 @@ done nfhrs="${IAUFHRS_ENKF//,/ }" for imem in $(seq 1 "${NMEM_ENS}"); do - smem=$((imem + mem_offset)) - if [[ ${smem} -gt ${NMEM_ENS_MAX} ]]; then - smem=$((smem - NMEM_ENS_MAX)) - fi - gmemchar="mem"$(printf "%03i" "${smem}") - memchar="mem"$(printf "%03i" "${imem}") + smem=$((imem + mem_offset)) + if [[ ${smem} -gt ${NMEM_ENS_MAX} ]]; then + smem=$((smem - NMEM_ENS_MAX)) + fi + gmemchar="mem"$(printf "%03i" "${smem}") + memchar="mem"$(printf "%03i" "${imem}") - MEMDIR=${gmemchar} RUN=${GDUMP} YMD=${GDATE:0:8} HH=${GDATE:8:2} declare_from_tmpl -x \ - COMIN_ATMOS_HISTORY_MEM_PREV:COM_ATMOS_HISTORY_TMPL + MEMDIR=${gmemchar} RUN=${GDUMP} YMD=${GDATE:0:8} HH=${GDATE:8:2} declare_from_tmpl -x \ + COMIN_ATMOS_HISTORY_MEM_PREV:COM_ATMOS_HISTORY_TMPL MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl -x \ COMOUT_ATMOS_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL diff --git a/scripts/exglobal_atmos_analysis.sh b/scripts/exglobal_atmos_analysis.sh index fbc917a14c4..241eb658366 100755 --- a/scripts/exglobal_atmos_analysis.sh +++ b/scripts/exglobal_atmos_analysis.sh @@ -896,7 +896,7 @@ fi # Diagnostic files # if requested, GSI diagnostic file directories for use later -if [[ "${GENDIAG}" == "YES" ]] ; then +if [[ "${GENDIAG}" == "YES" ]]; then tar -cvf gsidiags.tar dir.???? export err=$? if [[ ${err} -ne 0 ]]; then diff --git a/scripts/exglobal_diag.sh b/scripts/exglobal_diag.sh index 48119092bb6..68231b6fb72 100755 --- a/scripts/exglobal_diag.sh +++ b/scripts/exglobal_diag.sh @@ -50,7 +50,7 @@ DIAG_TARBALL=${DIAG_TARBALL:-"YES"} # Set script / GSI control parameters ################################################################################ -if [[ "${GENDIAG}" != "YES" ]] ; then +if [[ "${GENDIAG}" != "YES" ]]; then echo "INFO: GENDIAG set to NO. Skipping diagnostic file generation." exit 0 fi @@ -77,7 +77,6 @@ else diagtype[3]="msu_n14 sndr_g08 sndr_g11 sndr_g12 sndr_g13 sndr_g08_prep sndr_g11_prep sndr_g12_prep sndr_g13_prep sndrd1_g11 sndrd2_g11 sndrd3_g11 sndrd4_g11 sndrd1_g12 sndrd2_g12 sndrd3_g12 sndrd4_g12 sndrd1_g13 sndrd2_g13 sndrd3_g13 sndrd4_g13 sndrd1_g14 sndrd2_g14 sndrd3_g14 sndrd4_g14 sndrd1_g15 sndrd2_g15 sndrd3_g15 sndrd4_g15 amsua_n15 amsua_n16 amsua_n17 amsub_n15 amsub_n16 amsub_n17 hsb_aqua airs_aqua amsua_aqua imgr_g08 imgr_g11 imgr_g12 imgr_g14 imgr_g15 ssmi_f13 ssmi_f15 amsua_n18 amsua_metop-a mhs_n18 mhs_metop-a amsre_low_aqua amsre_mid_aqua amsre_hig_aqua ssmis_f16 ssmis_f17 ssmis_f18 ssmis_f19 ssmis_f20 iasi_metop-a amsua_n19 mhs_n19 seviri_m08 seviri_m09 seviri_m10 seviri_m11 cris_npp cris-fsr_npp cris-fsr_n20 atms_npp atms_n20 amsua_metop-b mhs_metop-b iasi_metop-b avhrr_metop-b avhrr_n18 avhrr_n19 avhrr_metop-a amsr2_gcom-w1 gmi_gpm saphir_meghat ahi_himawari8 abi_g16 abi_g17 amsua_metop-c mhs_metop-c iasi_metop-c avhrr_metop-c viirs-m_npp viirs-m_j1 abi_g18 ahi_himawari9 viirs-m_j2 cris-fsr_n21 atms_n21 abi_g19" fi - diaglist[0]=listcnv diaglist[1]=listpcp diaglist[2]=listozn @@ -142,23 +141,23 @@ touch cmdfile loops="01 03" for loop in ${loops}; do case ${loop} in - 01) string=ges;; - 03) string=anl;; - *) string=${loop};; + 01) string=ges ;; + 03) string=anl ;; + *) string=${loop} ;; esac echo "START loop ${string}" n=-1 - while [[ ${n} -lt ${ntype} ]] ;do - n=$(( n + 1 )) + while [[ ${n} -lt ${ntype} ]]; do + n=$((n + 1)) for type in ${diagtype[n]}; do #shellcheck disable=SC2012,SC2312 - count=$(ls dir.*/"${type}_${loop}"* 2>/dev/null | wc -l) + count=$(ls dir.*/"${type}_${loop}"* 2> /dev/null | wc -l) if [[ ${count} -eq 0 ]]; then continue fi echo "${DATA}/diag.sh ${type} ${loop} ${string} ${count} ${DIAG_SUFFIX:-}.nc4" >> cmdfile echo "diag_${type}_${string}.${PDY}${cyc}${DIAG_SUFFIX:-}.nc4${COMPRESS_SUFFIX:-}" >> "${diaglist[n]}" - numfile[n]=$(( numfile[n] + 1 )) + numfile[n]=$((numfile[n] + 1)) done done echo "END loop ${string}" @@ -199,8 +198,8 @@ fi if [[ "${DIAG_TARBALL}" == "YES" ]]; then echo "START tar diagnostic files" n=-1 - while [[ ${n} -lt ${ntype} ]] ;do - n=$((n+1)) + while [[ ${n} -lt ${ntype} ]]; do + n=$((n + 1)) TAROPTS="-uvf" if [[ ! -s "${diagfile[n]}" ]]; then TAROPTS="-cvf" From 763e182ad285ee7eb3ac9936cda899ec62d5c0f6 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Wed, 3 Dec 2025 11:59:37 -0500 Subject: [PATCH 15/15] Apply suggestion from @aerorahul --- scripts/exglobal_atmos_analysis.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exglobal_atmos_analysis.sh b/scripts/exglobal_atmos_analysis.sh index 241eb658366..6eb8804a59d 100755 --- a/scripts/exglobal_atmos_analysis.sh +++ b/scripts/exglobal_atmos_analysis.sh @@ -20,7 +20,7 @@ # Set environment. # Directories. - +cd "${DATA}" || exit 1 # Base variables rCDUMP=${rCDUMP:-"gdas"} GDUMP=${GDUMP:-"gdas"}