diff --git a/dev/parm/config/gfs/config.resources b/dev/parm/config/gfs/config.resources index 1c8ca9d24a1..f6bd3ab387f 100644 --- a/dev/parm/config/gfs/config.resources +++ b/dev/parm/config/gfs/config.resources @@ -758,12 +758,19 @@ case ${step} in fi ;; - "analdiag") + "analdiag" | "ediag") walltime="00:15:00" ntasks=96 # Should be at least twice ediag's tasks + memory="48GB" + #if [[ "${step}" == "ediag" ]]; then + # ntasks=48 + #fi threads_per_task=1 + if [[ ${ntasks} -gt ${max_tasks_per_node} ]]; then + ntasks=${max_tasks_per_node} + export is_exclusive=True + fi tasks_per_node=$(( max_tasks_per_node / threads_per_task )) - memory="48GB" ;; "anlstat") @@ -853,7 +860,7 @@ case ${step} in MEDTHREADS=${nthreads_mediator:-1} MEDPETS=${MEDPETS:-${FV3PETS}} (( MEDPETS = FV3PETS )) - (( MEDTHREADS = FV3THREADS )) + (( MEDTHREADS = FV3THREADS )) (( "${MEDPETS}" > 4800 )) && MEDPETS=4800 export MEDPETS MEDTHREADS echo "MEDIATOR using (threads, PETS) = (${MEDTHREADS}, ${MEDPETS})" @@ -979,7 +986,7 @@ case ${step} in ;; "upp") - #ntasks cannot be greater than $CASE_HIST + #ntasks cannot be greater than $CASE_HIST #define resources based on $CASE_HIST for upp case "${CASE_HIST}" in "C48" | "C96") @@ -1255,14 +1262,6 @@ case ${step} in fi ;; - "ediag") - walltime="00:15:00" - ntasks=48 - threads_per_task=1 - tasks_per_node=$(( max_tasks_per_node / threads_per_task )) - memory="30GB" - ;; - "eupd") walltime="00:45:00" case "${CASE}" in diff --git a/env/GAEAC6.env b/env/GAEAC6.env index 77e7bdd05d3..5ab863acb16 100755 --- a/env/GAEAC6.env +++ b/env/GAEAC6.env @@ -199,7 +199,10 @@ case ${step} in ntasks_gausfcanl=${ntasks_gausfcanl:-1} export APRUN_GAUSFCANL="${launcher} -n ${ntasks_gausfcanl} --cpus-per-task=${NTHREADS_GAUSFCANL}" ;; - "offlineanl") + "analdiag" | "ediag") + export USE_CFP="YES" + ;; + "offlineanl") export NTHREADS_CHGRES=${threads_per_task_chgres:-12} if [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]]; then diff --git a/env/HERA.env b/env/HERA.env index 8d6619e0da1..76af27e6748 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -203,6 +203,10 @@ elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then ntasks_gausfcanl=${ntasks_gausfcanl:-1} export APRUN_GAUSFCANL="${launcher} -n ${ntasks_gausfcanl} --cpus-per-task=${NTHREADS_GAUSFCANL}" +elif [[ "${step}" = "analdiag" ]] || [[ "${step}" = "ediag" ]]; then + + export USE_CFP="YES" + elif [[ "${step}" = "offlineanl" ]]; then export NTHREADS_CHGRES=${threads_per_task_chgres:-12} diff --git a/env/HERCULES.env b/env/HERCULES.env index 9aa6feb9ed8..493e23db4a3 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -158,7 +158,7 @@ case ${step} in "anlstat") export NTHREADS_ANLSTAT=${NTHREADSmax} - export APRUN_ANLSTAT="${APRUN_default} --cpus-per-task=${NTHREADS_ANLSTAT}" + export APRUN_ANLSTAT="${APRUN_default} --cpus-per-task=${NTHREADS_ANLSTAT}" ;; "ecen_fv3jedi") @@ -218,6 +218,9 @@ case ${step} in # REGRID requires 6 tasks for reproducibility ntasks_regrid=6 export APRUN_REGRID="${launcher} -n ${ntasks_regrid} " + ;; + "analdiag" | "ediag") + export USE_CFP="YES" ;; "offlineanl") diff --git a/env/ORION.env b/env/ORION.env index ca1d9334dc5..6634fda665f 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -157,7 +157,7 @@ elif [[ "${step}" = "marineanlletkf" ]]; then elif [[ "${step}" = "anlstat" ]]; then export NTHREADS_ANLSTAT=${NTHREADSmax} - export APRUN_ANLSTAT="${APRUN_default} --cpus-per-task=${NTHREADS_ANLSTAT}" + export APRUN_ANLSTAT="${APRUN_default} --cpus-per-task=${NTHREADS_ANLSTAT}" elif [[ "${step}" = "ecen_fv3jedi" ]]; then @@ -216,6 +216,10 @@ elif [[ "${step}" = "sfcanl" ]]; then ntasks_regrid=6 export APRUN_REGRID="${launcher} -n ${ntasks_regrid} " +elif [[ "${step}" = "analdiag" ]] || [[ "${step}" = "ediag" ]]; then + + export USE_CFP="YES" + elif [[ "${step}" = "offlineanl" ]]; then export NTHREADS_CHGRES=${threads_per_task:-12} diff --git a/env/URSA.env b/env/URSA.env index 4ee5a07b7e5..80711ea2b3b 100644 --- a/env/URSA.env +++ b/env/URSA.env @@ -192,6 +192,10 @@ elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then ntasks_gausfcanl=${ntasks_gausfcanl:-1} export APRUN_GAUSFCANL="${launcher} -n ${ntasks_gausfcanl} --cpus-per-task=${NTHREADS_GAUSFCANL}" +elif [[ "${step}" = "analdiag" ]] || [[ "${step}" = "ediag" ]]; then + + export USE_CFP="YES" + elif [[ "${step}" = "offlineanl" ]]; then export NTHREADS_CHGRES=${threads_per_task_chgres:-12} diff --git a/env/WCOSS2.env b/env/WCOSS2.env index 5d89728674c..867c0a4eab2 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -205,6 +205,10 @@ elif [[ "${step}" = "sfcanl" ]]; then ntasks_regrid=6 export APRUN_REGRID="${launcher} -n ${ntasks_regrid} " +elif [[ "${step}" = "analdiag" ]] || [[ "${step}" = "ediag" ]]; then + + export USE_CFP="YES" + elif [[ "${step}" = "offlineanl" ]]; then export NTHREADS_CHGRES=${threads_per_task_chgres:-14} diff --git a/jobs/JGDAS_ATMOS_ANALYSIS_DIAG b/jobs/JGDAS_ATMOS_ANALYSIS_DIAG index ad83614bff8..553a58e1160 100755 --- a/jobs/JGDAS_ATMOS_ANALYSIS_DIAG +++ b/jobs/JGDAS_ATMOS_ANALYSIS_DIAG @@ -1,27 +1,28 @@ #! /usr/bin/env bash -source "${HOMEgfs}/ush/jjob_header.sh" -e "anal" -c "base anal analdiag" +source "${HOMEgfs}/ush/jjob_header.sh" -e "analdiag" -c "base anal analdiag" ############################################## # Set variables used in the script ############################################## -export DO_CALC_ANALYSIS=${DO_CALC_ANALYSIS:-"YES"} +#export DO_CALC_ANALYSIS=${DO_CALC_ANALYSIS:-"YES"} ############################################## # Begin JOB SPECIFIC work ############################################## # shellcheck disable=SC2153 -GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") -export gPDY=${GDATE:0:8} -export gcyc=${GDATE:8:2} -export GDUMP="gdas" -export GDUMP_ENS="enkf${GDUMP}" +#GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") +#export gPDY=${GDATE:0:8} +#export gcyc=${GDATE:8:2} +#export GDUMP="gdas" +#export GDUMP_ENS="enkf${GDUMP}" -export OPREFIX="${RUN/enkf/}.t${cyc}z." -export GPREFIX="${GDUMP}.t${gcyc}z." -export APREFIX="${RUN}.t${cyc}z." +#export OPREFIX="${RUN/enkf/}.t${cyc}z." +#export GPREFIX="${GDUMP}.t${gcyc}z." +#export APREFIX="${RUN}.t${cyc}z." YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \ COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL mkdir -m 775 -p "${COMOUT_ATMOS_ANALYSIS}" diff --git a/jobs/JGDAS_ENKF_DIAG b/jobs/JGDAS_ENKF_DIAG index 70567283f2f..fa494dfb847 100755 --- a/jobs/JGDAS_ENKF_DIAG +++ b/jobs/JGDAS_ENKF_DIAG @@ -1,6 +1,6 @@ #! /usr/bin/env bash -source "${HOMEgfs}/ush/jjob_header.sh" -e "eobs" -c "base anal eobs analdiag ediag" +source "${HOMEgfs}/ush/jjob_header.sh" -e "ediag" -c "base anal eobs analdiag ediag" ############################################## # Set variables used in the script @@ -10,83 +10,76 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "eobs" -c "base anal eobs analdiag edi # Begin JOB SPECIFIC work ############################################## # shellcheck disable=SC2153 -GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") -export gPDY=${GDATE:0:8} -export gcyc=${GDATE:8:2} -export GDUMP="gdas" -export GDUMP_ENS="enkf${GDUMP}" +#GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") +#export gPDY=${GDATE:0:8} +#export gcyc=${GDATE:8:2} +#export GDUMP="gdas" +#export GDUMP_ENS="enkf${GDUMP}" export CASE=${CASE_ENS} -export OPREFIX="${RUN/enkf/}.t${cyc}z." +#export OPREFIX="${RUN/enkf/}.t${cyc}z." export APREFIX="${RUN}.t${cyc}z." -export GPREFIX="${GDUMP_ENS}.t${gcyc}z." -GPREFIX_DET="${GDUMP}.t${gcyc}z." +#export GPREFIX="${GDUMP_ENS}.t${gcyc}z." +#GPREFIX_DET="${GDUMP}.t${gcyc}z." -RUN=${RUN/enkf/} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ - COMIN_OBS:COM_OBS_TMPL +#RUN=${RUN/enkf/} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ +# COMIN_OBS:COM_OBS_TMPL MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \ COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL -RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ - COMIN_OBS_PREV:COM_OBS_TMPL \ - COMIN_ATMOS_ANALYSIS_DET_PREV:COM_ATMOS_ANALYSIS_TMPL +#RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ +# COMIN_OBS_PREV:COM_OBS_TMPL \ +# COMIN_ATMOS_ANALYSIS_DET_PREV:COM_ATMOS_ANALYSIS_TMPL -MEMDIR="ensstat" RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ - COMIN_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL +#MEMDIR="ensstat" RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ +# COMIN_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL -export ATMGES_ENSMEAN="${COMIN_ATMOS_HISTORY_PREV}/${GPREFIX}ensmean.atm.f006.nc" -if [[ ! -f "${ATMGES_ENSMEAN}" ]]; then - export err=1 - err_exit "FILE MISSING: ATMGES_ENSMEAN == ${ATMGES_ENSMEAN}" -fi +#export ATMGES_ENSMEAN="${COMIN_ATMOS_HISTORY_PREV}/${GPREFIX}ensmean.atm.f006.nc" +#if [[ ! -f "${ATMGES_ENSMEAN}" ]]; then +# export err=1 +# err_exit "FILE MISSING: ATMGES_ENSMEAN == ${ATMGES_ENSMEAN}" +#fi # Link observational data -export PREPQC="${COMIN_OBS}/${OPREFIX}prepbufr" -if [[ ! -f ${PREPQC} ]]; then - echo "WARNING: Global PREPBUFR FILE ${PREPQC} MISSING" -fi -export TCVITL="${COMIN_OBS}/${OPREFIX}syndata.tcvitals.tm00" -if [[ ${DONST} == "YES" ]]; then - export NSSTBF="${COMIN_OBS}/${OPREFIX}nsstbufr" -fi -export PREPQCPF="${COMIN_OBS}/${OPREFIX}prepbufr.acft_profiles" +#export PREPQC="${COMIN_OBS}/${OPREFIX}prepbufr" +#if [[ ! -f ${PREPQC} ]]; then +# echo "WARNING: Global PREPBUFR FILE ${PREPQC} MISSING" +#fi +#export TCVITL="${COMIN_OBS}/${OPREFIX}syndata.tcvitals.tm00" +#if [[ ${DONST} == "YES" ]]; then +# export NSSTBF="${COMIN_OBS}/${OPREFIX}nsstbufr" +#fi +#export PREPQCPF="${COMIN_OBS}/${OPREFIX}prepbufr.acft_profiles" # Guess Bias correction coefficients related to control -export GBIAS=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias.txt -export GBIASPC=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_pc.txt -export GBIASAIR=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_air.txt -export GRADSTAT=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}radstat.tar +#export GBIAS=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias.txt +#export GBIASPC=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_pc.txt +#export GBIASAIR=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_air.txt +#export GRADSTAT=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}radstat.tar # Bias correction coefficients related to ensemble mean -export ABIAS="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias.txt" -export ABIASPC="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_pc.txt" -export ABIASAIR="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_air.txt" -export ABIASe="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_int.txt" +#export ABIAS="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias.txt" +#export ABIASPC="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_pc.txt" +#export ABIASAIR="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_air.txt" +#export ABIASe="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_int.txt" # Diagnostics related to ensemble mean -export GSISTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}gsistat.ensmean.tar" -export CNVSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}cnvstat.ensmean.tar" -export OZNSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}oznstat.ensmean.tar" -export RADSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}radstat.ensmean.tar" +export CNVSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}cnvstat_ensmean.tar" +export OZNSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}oznstat_ensmean.tar" +export RADSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}radstat_ensmean.tar" # Select observations based on ensemble mean -export RUN_SELECT="YES" -export USE_SELECT="NO" -export SELECT_OBS="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}obsinput.ensmean" +#export RUN_SELECT="YES" +#export USE_SELECT="NO" +#export SELECT_OBS="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}obsinput.ensmean" export DIAG_SUFFIX="_ensmean" export DIAG_COMPRESS="NO" # GSI namelist options specific to eobs -export SETUP_INVOBS="passive_bc=.false.,${SETUP_INVOBS}" - -# Ensure clean stat tarballs for ensemble mean -for fstat in ${CNVSTAT} ${OZNSTAT} ${RADSTAT}; do - if [[ -f "${fstat}" ]]; then - rm -f "${fstat}" - fi -done +#export SETUP_INVOBS="passive_bc=.false.,${SETUP_INVOBS}" ############################################################### # Run relevant script diff --git a/jobs/JGDAS_ENKF_SELECT_OBS b/jobs/JGDAS_ENKF_SELECT_OBS index 3417f6f6e77..f5fe813ef9d 100755 --- a/jobs/JGDAS_ENKF_SELECT_OBS +++ b/jobs/JGDAS_ENKF_SELECT_OBS @@ -84,15 +84,15 @@ export ABIASAIR="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_air.txt" export ABIASe="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_int.txt" # Diagnostics related to ensemble mean -export GSISTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}gsistat.ensmean.txt" -export CNVSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}cnvstat.ensmean.tar" -export OZNSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}oznstat.ensmean.tar" -export RADSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}radstat.ensmean.tar" +export GSISTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}gsistat_ensmean.txt" +export CNVSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}cnvstat_ensmean.tar" +export OZNSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}oznstat_ensmean.tar" +export RADSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}radstat_ensmean.tar" # Select observations based on ensemble mean export RUN_SELECT="YES" export USE_SELECT="NO" -export SELECT_OBS="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}obsinput.ensmean" +export SELECT_OBS="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}obsinput_ensmean.tar" export DIAG_SUFFIX="_ensmean" diff --git a/jobs/JGDAS_ENKF_UPDATE b/jobs/JGDAS_ENKF_UPDATE index 3b0d829591b..43d39b21735 100755 --- a/jobs/JGDAS_ENKF_UPDATE +++ b/jobs/JGDAS_ENKF_UPDATE @@ -9,15 +9,12 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "eupd" -c "base anal eupd" ############################################## # Begin JOB SPECIFIC work ############################################## -# shellcheck disable=SC2153 -GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") -export gPDY=${GDATE:0:8} -export gcyc=${GDATE:8:2} -export GDUMP="gdas" -export GDUMP_ENS="enkf${GDUMP}" +# shellcheck disable=SC2155 +export GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") +export GDUMP="enkfgdas" # EnKF ensemble always uses RUN=enkfgdas for prior members export APREFIX="${RUN}.t${cyc}z." -export GPREFIX="${GDUMP_ENS}.t${gcyc}z." +export GPREFIX="${GDUMP}.t${GDATE:8:2}z." MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMIN_ATMOS_ANALYSIS_STAT:COM_ATMOS_ANALYSIS_TMPL @@ -25,7 +22,7 @@ MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMOUT_ATMOS_ANALYSIS_STAT:COM_ATMOS_ANALYSIS_TMPL -MEMDIR="ensstat" RUN="enkfgdas" YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ +MEMDIR="ensstat" RUN="enkfgdas" YMD=${GDATE:0:8} HH=${GDATE:8:2} declare_from_tmpl -rx \ COMIN_ATMOS_HISTORY_STAT_PREV:COM_ATMOS_HISTORY_TMPL ############################################################### diff --git a/jobs/JGLOBAL_ATMOS_ANALYSIS_CALC b/jobs/JGLOBAL_ATMOS_ANALYSIS_CALC index cf59158117a..bfb5c74a3ad 100755 --- a/jobs/JGLOBAL_ATMOS_ANALYSIS_CALC +++ b/jobs/JGLOBAL_ATMOS_ANALYSIS_CALC @@ -28,6 +28,7 @@ RUN=${rCDUMP} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \ + COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \ COMIN_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2 index c19781a1c3e..73c92aeb806 100644 --- a/parm/archive/enkf.yaml.j2 +++ b/parm/archive/enkf.yaml.j2 @@ -69,10 +69,10 @@ enkf: # Ensemble mean state {% if not DO_JEDIATMENS %} {% set da_stat_files = ["enkfstat.txt", - "gsistat.ensmean.txt", - "cnvstat.ensmean.tar", - "oznstat.ensmean.tar", - "radstat.ensmean.tar"] %} + "gsistat_ensmean.txt", + "cnvstat_ensmean.tar", + "oznstat_ensmean.tar", + "radstat_ensmean.tar"] %} {% set da_conf_files = [] %} {% else %} {% if lobsdiag_forenkf %} @@ -109,7 +109,7 @@ enkf: # soil DA increments {% if DO_GSISOILDA %} {% for fhr in IAUFHRS %} - - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}increment.sfc.i{{ '%03d' % fhr }}.nc" + - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}ensmean_increment.sfc.i{{ '%03d' % fhr }}.nc" {% endfor %} {% endif %} diff --git a/parm/archive/gcafs_arcdir.yaml.j2 b/parm/archive/gcafs_arcdir.yaml.j2 index c4a548c4e94..325ffba80e6 100644 --- a/parm/archive/gcafs_arcdir.yaml.j2 +++ b/parm/archive/gcafs_arcdir.yaml.j2 @@ -83,7 +83,7 @@ {% else %} {% do enkf_files.append([COMIN_ATMOS_ANALYSIS_ENSSTAT ~ "/" ~ head ~ "enkfstat.txt", ARCDIR ~ "/enkfstat." ~ RUN ~ "." ~ cycle_YMDH ]) %} - {% do enkf_files.append([COMIN_ATMOS_ANALYSIS_ENSSTAT ~ "/" ~ head ~ "gsistat.ensmean.txt", + {% do enkf_files.append([COMIN_ATMOS_ANALYSIS_ENSSTAT ~ "/" ~ head ~ "gsistat_ensmean.txt", ARCDIR ~ "/gsistat." ~ RUN ~ "." ~ cycle_YMDH ~ ".ensmean"]) %} {% endif %} diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2 index 60615e0c7ad..94074d2239b 100644 --- a/parm/archive/gdas_restarta.yaml.j2 +++ b/parm/archive/gdas_restarta.yaml.j2 @@ -46,7 +46,7 @@ gdas_restarta: - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}cnvstat.tar" - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_int.txt" - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}analysis.dtf.a006.nc" - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}increment.done.txt" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}analysis.done.txt" {% else %} - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}rad_varbc_params.tar" {% endif %} diff --git a/parm/archive/gfs_arcdir.yaml.j2 b/parm/archive/gfs_arcdir.yaml.j2 index 877691ff5f9..ed873b21c8b 100644 --- a/parm/archive/gfs_arcdir.yaml.j2 +++ b/parm/archive/gfs_arcdir.yaml.j2 @@ -140,7 +140,7 @@ {% else %} {% do enkf_files.append([COMIN_ATMOS_ANALYSIS_ENSSTAT ~ "/" ~ head ~ "enkfstat.txt", ARCDIR ~ "/enkfstat." ~ RUN ~ "." ~ cycle_YMDH ]) %} - {% do enkf_files.append([COMIN_ATMOS_ANALYSIS_ENSSTAT ~ "/" ~ head ~ "gsistat.ensmean.txt", + {% do enkf_files.append([COMIN_ATMOS_ANALYSIS_ENSSTAT ~ "/" ~ head ~ "gsistat_ensmean.txt", ARCDIR ~ "/gsistat." ~ RUN ~ "." ~ cycle_YMDH ~ ".ensmean"]) %} {% endif %} diff --git a/scripts/exgdas_enkf_update.sh b/scripts/exgdas_enkf_update.sh index df981d16973..c44ba3e2bb9 100755 --- a/scripts/exgdas_enkf_update.sh +++ b/scripts/exgdas_enkf_update.sh @@ -18,56 +18,32 @@ ################################################################################ # Directories. -pwd=$(pwd) +cd "${DATA}" || exit 1 # Utilities NCLEN=${NCLEN:-${USHgfs}/getncdimlen} -USE_CFP=${USE_CFP:-"NO"} -CFP_MP=${CFP_MP:-"NO"} -APRUNCFP=${APRUNCFP:-""} APRUN_ENKF=${APRUN_ENKF:-${APRUN:-""}} NTHREADS_ENKF=${NTHREADS_ENKF:-${NTHREADS:-1}} # Executables ENKFEXEC=${ENKFEXEC:-${EXECgfs}/enkf.x} -# Filenames. -GPREFIX=${GPREFIX:-""} -APREFIX=${APREFIX:-""} +APREFIX=${APREFIX:-${RUN}.t${cyc}z.} +GPREFIX=${GPREFIX:-${RUN}.t${GDATE:8:2}z.} -SMOOTH_ENKF=${SMOOTH_ENKF:-"YES"} - -GBIASe=${GBIASe:-${APREFIX}abias_int.ensmean.txt} -CNVSTAT=${CNVSTAT:-${APREFIX}cnvstat.tar} -OZNSTAT=${OZNSTAT:-${APREFIX}oznstat.tar} -RADSTAT=${RADSTAT:-${APREFIX}radstat.tar} -ENKFSTAT=${ENKFSTAT:-${APREFIX}enkfstat.txt} +# GBIASe=${GBIASe:-${APREFIX}abias_int.ensmean.txt} # TODO: remove (see comment and TODO below) Also, this is not a "G"BIAS (See the name, it has APREFIX in its name) +CNVSTAT="${APREFIX}cnvstat_ensmean.tar" +OZNSTAT="${APREFIX}oznstat_ensmean.tar" +RADSTAT="${APREFIX}radstat_ensmean.tar" # Namelist parameters -USE_CORRELATED_OBERRS=${USE_CORRELATED_OBERRS:-"NO"} -NAM_ENKF=${NAM_ENKF:-""} -SATOBS_ENKF=${SATOBS_ENKF:-""} -OZOBS_ENKF=${OZOBS_ENKF:-""} -use_correlated_oberrs=${use_correlated_oberrs:-".false."} -if [[ "${USE_CORRELATED_OBERRS}" == "YES" ]]; then +if [[ "${USE_CORRELATED_OBERRS:-}" == "YES" ]]; then use_correlated_oberrs=".true." fi -imp_physics=${imp_physics:-"99"} -lupp=${lupp:-".true."} corrlength=${corrlength:-1250} lnsigcutoff=${lnsigcutoff:-2.5} analpertwt=${analpertwt:-0.85} -readin_localization_enkf=${readin_localization_enkf:-".true."} -reducedgrid=${reducedgrid:-".false."} -letkf_flag=${letkf_flag:-".false."} -getkf=${getkf:-".false."} -denkf=${denkf:-".false."} -nobsl_max=${nobsl_max:-10000} -write_spread_diag=${write_spread_diag:-".false."} cnvw_option=${cnvw_option:-".false."} -netcdf_diag=${netcdf_diag:-".true."} -modelspace_vloc=${modelspace_vloc:-".false."} # if true, 'vlocal_eig.dat' is needed -taperanalperts=${taperanalperts:-".false."} IAUFHRS_ENKF=${IAUFHRS_ENKF:-"6,"} NMEM_ENS_MAX=${NMEM_ENS:-80} if [[ "${RUN}" == "enkfgfs" ]]; then @@ -90,9 +66,6 @@ ATMGES_ENSMEAN="${COMIN_ATMOS_HISTORY_STAT_PREV}/${GPREFIX}ensmean.atm.f006.nc" LONB_ENKF=${LONB_ENKF:-$(${NCLEN} "${ATMGES_ENSMEAN}" grid_xt)} # get LONB_ENKF LATB_ENKF=${LATB_ENKF:-$(${NCLEN} "${ATMGES_ENSMEAN}" grid_yt)} # get LATB_ENFK LEVS_ENKF=${LEVS_ENKF:-$(${NCLEN} "${ATMGES_ENSMEAN}" pfull)} # get LEVS_ENFK -use_gfs_ncio=".true." -use_gfs_nemsio=".false." -paranc=${paranc:-".true."} WRITE_INCR_ZERO="incvars_to_zero= ${INCREMENTS_TO_ZERO}," if [[ "${DO_CALC_INCREMENT}" == "YES" ]]; then write_fv3_incr=".false." @@ -110,71 +83,55 @@ HYBENSINFO=${HYBENSINFO:-${FIXgfs}/gsi/global_hybens_info.l${LEVS_ENKF}.txt} ANAVINFO=${ANAVINFO:-${FIXgfs}/gsi/global_anavinfo.l${LEVS_ENKF}.txt} VLOCALEIG=${VLOCALEIG:-${FIXgfs}/gsi/vlocal_eig_l${LEVS_ENKF}.dat} ENKF_SUFFIX="s" -if [[ "${SMOOTH_ENKF}" == "NO" ]]; then +if [[ "${SMOOTH_ENKF:-YES}" == "NO" ]]; then ENKF_SUFFIX="" fi ################################################################################ # Fixed files +cpreq "${SATANGL}" satbias_angle +cpreq "${SCANINFO}" scaninfo +cpreq "${HYBENSINFO}" hybens_info +cpreq "${ANAVINFO}" anavinfo +cpreq "${VLOCALEIG}" vlocal_eig.dat if [[ "${SATINFO}" == "generate" ]]; then "${USHgfs}/create_gsi_info.sh" sat "${PDY}${cyc}" "${DATA}" else - ${NLN} "${SATINFO}" satinfo + cpreq "${SATINFO}" satinfo fi if [[ "${CONVINFO}" == "generate" ]]; then "${USHgfs}/create_gsi_info.sh" conv "${PDY}${cyc}" "${DATA}" "${USE_2M_OBS}" else - ${NLN} "${CONVINFO}" convinfo + cpreq "${CONVINFO}" convinfo fi if [[ "${OZINFO}" == "generate" ]]; then "${USHgfs}/create_gsi_info.sh" oz "${PDY}${cyc}" "${DATA}" else - ${NLN} "${OZINFO}" ozinfo + cpreq "${OZINFO}" ozinfo fi -${NLN} "${SATANGL}" satbias_angle -${NLN} "${SCANINFO}" scaninfo -${NLN} "${HYBENSINFO}" hybens_info -${NLN} "${ANAVINFO}" anavinfo -${NLN} "${VLOCALEIG}" vlocal_eig.dat # Bias correction coefficients based on the ensemble mean -${NLN} "${COMIN_ATMOS_ANALYSIS_STAT}/${GBIASe}" "satbias_in" +#${NLN} "${COMIN_ATMOS_ANALYSIS_STAT}/${GBIASe}" "satbias_in" # This file does not exist when test was run # TODO: remove ################################################################################ +# Ensemble guess, observational data and analyses/increments -if [[ "${USE_CFP}" == "YES" ]]; then - rm -f "${DATA}/untar.sh" "${DATA}/mp_untar.sh" - cat > "${DATA}/untar.sh" << EOFuntar -#!/bin/sh -memchar=\$1 -COMOUT_ATMOS_ANALYSIS=\$2 flist="${CNVSTAT} ${OZNSTAT} ${RADSTAT}" -for ftype in \$flist; do - if [[ "\${memchar}" == "ensmean" ]]; then - fname=\${COMOUT_ATMOS_ANALYSIS}/\${ftype%.tar}.ensmean.tar - else - fname=\${COMOUT_ATMOS_ANALYSIS}/\${ftype} - fi - tar -xvf \$fname +for ftype in ${flist}; do + fname="${COMIN_ATMOS_ANALYSIS_STAT}/${ftype}" + tar -xvf "${fname}" done -EOFuntar - chmod 755 "${DATA}/untar.sh" -fi - -################################################################################ -# Ensemble guess, observational data and analyses/increments -echo "${DATA}/untar.sh ensmean ${COMIN_ATMOS_ANALYSIS_STAT}" | tee -a "${DATA}/mp_untar.sh" nfhrs="${IAUFHRS_ENKF//,/ }" for imem in $(seq 1 "${NMEM_ENS}"); do smem=$((imem + mem_offset)) - if ((smem > NMEM_ENS_MAX)); then + if [[ ${smem} -gt ${NMEM_ENS_MAX} ]]; then smem=$((smem - NMEM_ENS_MAX)) fi gmemchar="mem"$(printf "%03i" "${smem}") memchar="mem"$(printf "%03i" "${imem}") - MEMDIR=${gmemchar} RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -x \ + MEMDIR=${gmemchar} RUN=${GDUMP} YMD=${GDATE:0:8} HH=${GDATE:8:2} declare_from_tmpl -x \ COMIN_ATMOS_HISTORY_MEM_PREV:COM_ATMOS_HISTORY_TMPL MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl -x \ @@ -220,18 +177,11 @@ for FHR in ${nfhrs}; do if [[ "${DO_GSISOILDA}" == "YES" ]]; then ${NLN} "${COMIN_ATMOS_HISTORY_STAT_PREV}/${GPREFIX}ensmean.sfc.f00${FHR}.nc" \ "bfg_${PDY}${cyc}_fhr0${FHR}_ensmean" - ${NLN} "${COMIN_ATMOS_ANALYSIS_STAT}/${APREFIX}increment.sfc.i00${FHR}.nc" \ + ${NLN} "${COMIN_ATMOS_ANALYSIS_STAT}/${APREFIX}ensmean_increment.sfc.i00${FHR}.nc" \ "sfcincr_${PDY}${cyc}_fhr0${FHR}_ensmean" fi done -# Run with MPMD or serial -"${USHgfs}/run_mpmd.sh" "${DATA}/mp_untar.sh" && true -export err=$? -if [[ ${err} -ne 0 ]]; then - err_exit "Failed to untar input data!" -fi - ################################################################################ # Create global_enkf namelist cat > enkf.nml << EOFnml @@ -248,19 +198,19 @@ cat > enkf.nml << EOFnml sprd_tol=1.e30,paoverpb_thresh=0.98, nlons=${LONA_ENKF},nlats=${LATA_ENKF},nlevs=${LEVS_ENKF},nanals=${NMEM_ENS}, deterministic=.true.,sortinc=.true.,lupd_satbiasc=.false., - reducedgrid=${reducedgrid},readin_localization=${readin_localization_enkf}., - use_gfs_nemsio=${use_gfs_nemsio},use_gfs_ncio=${use_gfs_ncio},imp_physics=${imp_physics},lupp=${lupp}, + reducedgrid=${reducedgrid:-.false.},readin_localization=${readin_localization_enkf:-.true.}, + use_gfs_nemsio=.false.,use_gfs_ncio=.true.,imp_physics=${imp_physics:-99},lupp=${lupp:-.true.}, univaroz=.false.,adp_anglebc=.true.,angord=4,use_edges=.false.,emiss_bc=.true., - letkf_flag=${letkf_flag},nobsl_max=${nobsl_max},denkf=${denkf},getkf=${getkf}., + letkf_flag=${letkf_flag:-.false.},nobsl_max=${nobsl_max:-10000},denkf=${denkf:-.false.},getkf=${getkf:-.false.}, nhr_anal=${IAUFHRS_ENKF},nhr_state=${IAUFHRS_ENKF}, - lobsdiag_forenkf=.true.,taperanalperts=${taperanalperts}, - write_spread_diag=${write_spread_diag}, - modelspace_vloc=${modelspace_vloc}, - use_correlated_oberrs=${use_correlated_oberrs}, - netcdf_diag=${netcdf_diag},cnvw_option=${cnvw_option}, - paranc=${paranc},write_fv3_incr=${write_fv3_incr}, + lobsdiag_forenkf=.true.,taperanalperts=${taperanalperts:-.false.}, + write_spread_diag=${write_spread_diag:-".false."}, + modelspace_vloc=${modelspace_vloc:-.false.}, + use_correlated_oberrs=${use_correlated_oberrs:-.false.}, + netcdf_diag=.true.,cnvw_option=${cnvw_option}, + paranc=${paranc:-.true.},write_fv3_incr=${write_fv3_incr}, ${WRITE_INCR_ZERO} - ${NAM_ENKF} + ${NAM_ENKF:-} / &satobs_enkf sattypes_rad(1) = 'amsua_n15', dsis(1) = 'amsua_n15', @@ -343,7 +293,7 @@ cat > enkf.nml << EOFnml sattypes_rad(78)= 'atms_n21', dsis(78)= 'atms_n21', sattypes_rad(79)= 'cris-fsr_n21', dsis(79)= 'cris-fsr_n21', sattypes_rad(80)= 'abi_g19', dsis(80)= 'abi_g19', - ${SATOBS_ENKF} + ${SATOBS_ENKF:-} / &ozobs_enkf sattypes_oz(1) = 'sbuv2_n16', @@ -362,7 +312,7 @@ cat > enkf.nml << EOFnml sattypes_oz(14) = 'ompstc8_n21', sattypes_oz(15) = 'ompsnp_n21', sattypes_oz(16) = 'gome_metop-c', - ${OZOBS_ENKF} + ${OZOBS_ENKF:-} / EOFnml @@ -381,10 +331,10 @@ if [[ ${err} -ne 0 ]]; then fi # Cat runtime output files. -cat stdout stderr > "${COMOUT_ATMOS_ANALYSIS_STAT}/${ENKFSTAT}" +cat stdout stderr > enkfstat.txt +cpfs enkfstat.txt "${COMOUT_ATMOS_ANALYSIS_STAT}/${APREFIX}enkfstat.txt" ################################################################################ # Postprocessing -cd "${pwd}" || exit 1 -exit "${err}" +exit 0 diff --git a/scripts/exglobal_atmos_analysis.sh b/scripts/exglobal_atmos_analysis.sh index 604350dcf45..6eb8804a59d 100755 --- a/scripts/exglobal_atmos_analysis.sh +++ b/scripts/exglobal_atmos_analysis.sh @@ -20,8 +20,7 @@ # Set environment. # Directories. -pwd=$(pwd) - +cd "${DATA}" || exit 1 # Base variables rCDUMP=${rCDUMP:-"gdas"} GDUMP=${GDUMP:-"gdas"} @@ -39,7 +38,6 @@ export CHGRP_CMD=${CHGRP_CMD:-"chgrp ${group_name:-rstprod}"} export NCLEN=${NCLEN:-${USHgfs}/getncdimlen} COMPRESS=${COMPRESS:-gzip} UNCOMPRESS=${UNCOMPRESS:-gunzip} -APRUNCFP=${APRUNCFP:-""} APRUN_GSI=${APRUN_GSI:-${APRUN:-""}} NTHREADS_GSI=${NTHREADS_GSI:-${NTHREADS:-1}} @@ -57,8 +55,6 @@ IASI_CADS=${IASI_CADS:-".false."} CRIS_CADS=${CRIS_CADS:-".false."} # Diagnostic files options -netcdf_diag=${netcdf_diag:-".true."} -binary_diag=${binary_diag:-".false."} lobsdiag_forenkf=${lobsdiag_forenkf:-".false."} # IAU @@ -186,11 +182,7 @@ ABIAS=${ABIAS:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias.txt} ABIASPC=${ABIASPC:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_pc.txt} ABIASAIR=${ABIASAIR:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_air.txt} ABIASe=${ABIASe:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_int.txt} -RADSTAT=${RADSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}radstat.tar} GSISTAT=${GSISTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}gsistat.txt} -PCPSTAT=${PCPSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}pcpstat} -CNVSTAT=${CNVSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}cnvstat.tar} -OZNSTAT=${OZNSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}oznstat.tar} # Increment files ATMINC=${ATMINC:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}increment.atm.i006.nc} @@ -199,17 +191,8 @@ ATMINC=${ATMINC:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}increment.atm.i006.nc} RUN_SELECT=${RUN_SELECT:-"NO"} USE_SELECT=${USE_SELECT:-"NO"} USE_RADSTAT=${USE_RADSTAT:-"YES"} -SELECT_OBS=${SELECT_OBS:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}obsinput} +SELECT_OBS=${SELECT_OBS:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}obsinput.tar} GENDIAG=${GENDIAG:-"YES"} -DIAG_SUFFIX=${DIAG_SUFFIX:-""} -if [[ "${netcdf_diag}" == ".true." ]]; then - DIAG_SUFFIX="${DIAG_SUFFIX}.nc4" -fi -DIAG_COMPRESS=${DIAG_COMPRESS:-"YES"} -DIAG_TARBALL=${DIAG_TARBALL:-"YES"} -USE_CFP=${USE_CFP:-"NO"} -CFP_MP=${CFP_MP:-"NO"} -DIAG_DIR=${DIAG_DIR:-${COMOUT_ATMOS_ANALYSIS}/gsidiags} # Set script / GSI control parameters DOHYBVAR=${DOHYBVAR:-"NO"} @@ -332,8 +315,7 @@ RAPIDREFRESH_CLDSURF=${RAPIDREFRESH_CLDSURF:-""} CHEM=${CHEM:-""} NST=${NST:-""} -#uGSI Namelist parameters -lrun_subdirs=${lrun_subdirs:-".true."} +# GSI Namelist parameters if [[ "${DOHYBVAR}" == "YES" ]]; then l_hyb_ens=.true. export l4densvar=${l4densvar:-".false."} @@ -632,25 +614,6 @@ if [[ "${JCAP}" -ne "${JCAP_A}" ]]; then fi fi -############################################################## -# Diagnostic files -# if requested, link GSI diagnostic file directories for use later -if [[ "${GENDIAG}" == "YES" ]]; then - if [[ "${lrun_subdirs}" == ".true." ]]; then - if [[ -d "${DIAG_DIR}" ]]; then - rm -rf "${DIAG_DIR}" - fi - ntasks_m1="$((ntasks - 1))" - for pe in $(seq 0 "${ntasks_m1}"); do - pedir="dir.$(printf %04i "${pe}")" - mkdir -p "${DIAG_DIR}/${pedir}" - ${NLN} "${DIAG_DIR}/${pedir}" "${pedir}" - done - else - err_exit "lrun_subdirs must be true. lrun_subdirs=${lrun_subdirs}" - fi -fi - ############################################################## # Output files ${NLN} "${ATMANL}" siganl @@ -698,39 +661,32 @@ fi ############################################################## # If requested, copy and de-tar guess radstat file if [[ "${USE_RADSTAT}" == "YES" ]]; then - if [[ -f "${DATA}/unzip.sh" ]]; then - rm -f "${DATA}/unzip.sh" - fi - if [[ -f "${DATA}/mp_unzip.sh" ]]; then - rm -f "${DATA}/mp_unzip.sh" - fi - cat > "${DATA}/unzip.sh" << EOFunzip -#!/bin/sh - diag_file=\$1 - diag_suffix=\$2 - fname=\$(echo \$diag_file | cut -d'.' -f1) - fdate=\$(echo \$diag_file | cut -d'.' -f2) - ${UNCOMPRESS} \$diag_file - fnameges=\$(echo \$fname | sed 's/_ges//g') - ${NMV} \$fname.\$fdate\$diag_suffix \$fnameges -EOFunzip - - chmod 755 "${DATA}/unzip.sh" + rm -f "${DATA}/unzip_radstat.sh" + cat > "${DATA}/unzip_radstat.sh" << EOF +#!/bin/bash +diag_file=\$1 +diag_suffix=\$2 +fname=\$(echo \$diag_file | cut -d'.' -f1) +fdate=\$(echo \$diag_file | cut -d'.' -f2) +${UNCOMPRESS} \$diag_file +fnameges=\$(echo \$fname | sed 's/_ges//g') +${NMV} \$fname.\$fdate\$diag_suffix \$fnameges +EOF + chmod 755 "${DATA}/unzip_radstat.sh" + rm -f "${DATA}/cmdfile" #shellcheck disable=SC2312 - listdiag="$(tar xvf radstat.gdas | cut -d' ' -f2 | grep _ges)" + listdiag=$(tar -xvf radstat.gdas | cut -d' ' -f2 | grep _ges) for type in ${listdiag}; do diag_file=$(echo "${type}" | cut -d',' -f1) - echo "${DATA}/unzip.sh ${diag_file} ${DIAG_SUFFIX}" | tee -a "${DATA}/mp_unzip.sh" + echo "${DATA}/unzip_radstat.sh ${diag_file} ${DIAG_SUFFIX:-}.nc4" >> "${DATA}/cmdfile" done - chmod 755 "${DATA}/mp_unzip.sh" - "${USHgfs}/run_mpmd.sh" "${DATA}/mp_unzip.sh" && true + + "${USHgfs}/run_mpmd.sh" "${DATA}/cmdfile" && true export err=$? if [[ ${err} -ne 0 ]]; then - export pgm="run_mpmd.sh" - err_exit "run_mpmd failed while unziping diagnotic files." + err_exit "Failed to unzip radstat.gdas file!" fi - fi # if [[ $USE_RADSTAT == "YES" ] ############################################################## @@ -768,12 +724,12 @@ cat > gsiparm.anl << EOF use_pbl=.false.,use_compress=.true.,nsig_ext=45,gpstop=50.,commgpstop=45.,commgpserrinf=1.0, use_gfs_nemsio=.false.,use_gfs_ncio=.true.,sfcnst_comb=.true., use_readin_anl_sfcmask=${USE_READIN_ANL_SFCMASK}, - lrun_subdirs=${lrun_subdirs}, + lrun_subdirs=.true., crtm_coeffs_path='./crtm_coeffs/', newpc4pred=.true.,adp_anglebc=.true.,angord=4,passive_bc=.true.,use_edges=.false., diag_precon=.true.,step_start=1.e-3,emiss_bc=.true.,nhr_obsbin=${nhr_obsbin:-3}, cwoption=3,imp_physics=${imp_physics},lupp=${lupp},cnvw_option=${cnvw_option},cao_check=${cao_check}, - netcdf_diag=${netcdf_diag},binary_diag=${binary_diag}, + netcdf_diag=.true.,binary_diag=.false., lobsdiag_forenkf=${lobsdiag_forenkf}, write_fv3_incr=${write_fv3_increment}, nhr_anal=${IAUFHRS}, @@ -927,7 +883,7 @@ fi ################################################################################ # Postprocessing -cd "${pwd}" || exit 1 +cd "${DATA}" || exit 1 ############################################################## # Add this statement to release the forecast job once the @@ -937,6 +893,18 @@ cd "${pwd}" || exit 1 if [[ "${SENDECF}" == "YES" && "${RUN}" != "enkf" ]]; then ecflow_client --event release_fcst fi + +# Diagnostic files +# if requested, GSI diagnostic file directories for use later +if [[ "${GENDIAG}" == "YES" ]]; then + tar -cvf gsidiags.tar dir.???? + export err=$? + if [[ ${err} -ne 0 ]]; then + err_exit "Failed to tar GSI diagnostic directories!" + fi + cpfs gsidiags.tar "${COMOUT_ATMOS_ANALYSIS}/${APREFIX}gsidiags${DIAG_SUFFIX:-}.tar" +fi + # shellcheck disable=SC2312 echo "${rCDUMP} ${PDY}${cyc} atminc done at $(date)" > "${COMOUT_ATMOS_ANALYSIS}/${APREFIX}increment.done.txt" diff --git a/scripts/exglobal_diag.sh b/scripts/exglobal_diag.sh index 92ea7bdaa8f..68231b6fb72 100755 --- a/scripts/exglobal_diag.sh +++ b/scripts/exglobal_diag.sh @@ -18,233 +18,218 @@ ################################################################################ # Set environment. +cd "${DATA}" || exit 8 # Base variables -GDUMP="${GDUMP:-gdas}" # Utilities -export CHGRP_CMD=${CHGRP_CMD:-"chgrp ${group_name:-rstprod}"} -export NCLEN=${NCLEN:-${USHgfs}/getncdimlen} -export CATEXEC=${CATEXEC:-${ncdiag_ROOT:-${gsi_ncdiag_ROOT}}/bin/ncdiag_cat_serial.x} +CHGRP_CMD=${CHGRP_CMD:-"chgrp ${group_name:-rstprod}"} +CATEXEC=${CATEXEC:-${ncdiag_ROOT:-${gsi_ncdiag_ROOT}}/bin/ncdiag_cat_serial.x} COMPRESS=${COMPRESS:-gzip} -UNCOMPRESS=${UNCOMPRESS:-gunzip} -APRUNCFP=${APRUNCFP:-""} - -# Diagnostic files options -netcdf_diag=${netcdf_diag:-".true."} -binary_diag=${binary_diag:-".false."} # Analysis files -export APREFIX=${APREFIX:-""} -RADSTAT=${RADSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}radstat.tar} -PCPSTAT=${PCPSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}pcpstat} +APREFIX=${APREFIX:-"${RUN}.t${cyc}z."} CNVSTAT=${CNVSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}cnvstat.tar} +PCPSTAT=${PCPSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}pcpstat} OZNSTAT=${OZNSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}oznstat.tar} +RADSTAT=${RADSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}radstat.tar} # Remove stat file if file already exists rm -f "${RADSTAT}" "${PCPSTAT}" "${CNVSTAT}" "${OZNSTAT}" # Obs diag GENDIAG=${GENDIAG:-"YES"} -DIAG_SUFFIX=${DIAG_SUFFIX:-""} -if [[ "${netcdf_diag}" == ".true." ]]; then - DIAG_SUFFIX="${DIAG_SUFFIX}.nc4" -fi +GSIDIAG=${GSIDIAG:-"${COMIN_ATMOS_ANALYSIS}/${APREFIX}gsidiags${DIAG_SUFFIX:-}.tar"} +USE_BUILD_GSINFO=${USE_BUILD_GSINFO:-"NO"} DIAG_COMPRESS=${DIAG_COMPRESS:-"YES"} -COMPRESS_SUFFIX=${COMPRESS_SUFFIX:-""} -if [[ ${DIAG_COMPRESS} == "YES" ]]; then - COMPRESS_SUFFIX="${COMPRESS_SUFFIX}.gz" +if [[ "${DIAG_COMPRESS:-}" == "YES" ]]; then + COMPRESS_SUFFIX=".gz" fi DIAG_TARBALL=${DIAG_TARBALL:-"YES"} -USE_CFP=${USE_CFP:-"NO"} -CFP_MP=${CFP_MP:-"NO"} -DIAG_DIR=${DIAG_DIR:-${COMOUT_ATMOS_ANALYSIS}/gsidiags} -REMOVE_DIAG_DIR=${REMOVE_DIAG_DIR:-"NO"} # Set script / GSI control parameters -lrun_subdirs=${lrun_subdirs:-".true."} ################################################################################ -# If requested, generate diagnostic files -if [[ "${GENDIAG}" == "YES" ]]; then - if [[ "${lrun_subdirs}" == ".true." ]]; then - for pe in "${DIAG_DIR}/"dir.*; do - pedir="$(basename -- "${pe}")" - ${NLN} "${pe}" "${DATA}/${pedir}" - done - else - err_exit "lrun_subdirs must be true. Abort job" - fi +if [[ "${GENDIAG}" != "YES" ]]; then + echo "INFO: GENDIAG set to NO. Skipping diagnostic file generation." + exit 0 +fi - # Set up lists and variables for various types of diagnostic files. - diagtype[0]="conv conv_gps conv_ps conv_pw conv_q conv_sst conv_t conv_tcp conv_uv conv_spd" - diagtype[1]="pcp_ssmi_dmsp pcp_tmi_trmm" - if [[ "${USE_BUILD_GSINFO}" == "YES" ]]; then - diagtype[2]=$(cat "${BUILD_GSINFO_DIR}/ozinfo/satellites") - diagtype[3]=$(cat "${BUILD_GSINFO_DIR}/satinfo/satellites") - else - diagtype[2]="sbuv2_n16 sbuv2_n17 sbuv2_n18 sbuv2_n19 gome_metop-a gome_metop-b omi_aura mls30_aura ompsnp_npp ompstc8_npp ompstc8_n20 ompsnp_n20 ompstc8_n21 ompsnp_n21 ompslp_npp gome_metop-c" - diagtype[3]="msu_n14 sndr_g08 sndr_g11 sndr_g12 sndr_g13 sndr_g08_prep sndr_g11_prep sndr_g12_prep sndr_g13_prep sndrd1_g11 sndrd2_g11 sndrd3_g11 sndrd4_g11 sndrd1_g12 sndrd2_g12 sndrd3_g12 sndrd4_g12 sndrd1_g13 sndrd2_g13 sndrd3_g13 sndrd4_g13 sndrd1_g14 sndrd2_g14 sndrd3_g14 sndrd4_g14 sndrd1_g15 sndrd2_g15 sndrd3_g15 sndrd4_g15 amsua_n15 amsua_n16 amsua_n17 amsub_n15 amsub_n16 amsub_n17 hsb_aqua airs_aqua amsua_aqua imgr_g08 imgr_g11 imgr_g12 imgr_g14 imgr_g15 ssmi_f13 ssmi_f15 amsua_n18 amsua_metop-a mhs_n18 mhs_metop-a amsre_low_aqua amsre_mid_aqua amsre_hig_aqua ssmis_f16 ssmis_f17 ssmis_f18 ssmis_f19 ssmis_f20 iasi_metop-a amsua_n19 mhs_n19 seviri_m08 seviri_m09 seviri_m10 seviri_m11 cris_npp cris-fsr_npp cris-fsr_n20 atms_npp atms_n20 amsua_metop-b mhs_metop-b iasi_metop-b avhrr_metop-b avhrr_n18 avhrr_n19 avhrr_metop-a amsr2_gcom-w1 gmi_gpm saphir_meghat ahi_himawari8 abi_g16 abi_g17 amsua_metop-c mhs_metop-c iasi_metop-c avhrr_metop-c viirs-m_npp viirs-m_j1 abi_g18 ahi_himawari9 viirs-m_j2 cris-fsr_n21 atms_n21 abi_g19" - fi +################################################################################ +# Copy gsidiags.tar file from COMIN to DATA and untar +cpreq "${GSIDIAG}" ./gsidiags.tar +tar -xvf gsidiags.tar +export err=$? +if [[ ${err} -ne 0 ]]; then + err_exit "Unable to unpack gsidiags.tar file!" +fi - diaglist[0]=listcnv - diaglist[1]=listpcp - diaglist[2]=listozn - diaglist[3]=listrad - - diagfile[0]="${CNVSTAT}" - diagfile[1]="${PCPSTAT}" - diagfile[2]="${OZNSTAT}" - diagfile[3]="${RADSTAT}" - - # Set diagnostic file prefix based on lrun_subdirs variable - if [[ "${lrun_subdirs}" == ".true." ]]; then - prefix="dir.*/" - else - prefix="pe*" - fi +# Set up lists and variables for various types of diagnostic files. +ntype=3 - rm -f "${DATA}/diag.sh" "${DATA}/mp_diag.sh" - cat > "${DATA}/diag.sh" << EOFdiag -#!/bin/sh -lrun_subdirs=\$1 -binary_diag=\$2 -type=\$3 -loop=\$4 -string=\$5 -PDY=\$6 -cyc=\$7 -DIAG_COMPRESS=\$8 -DIAG_SUFFIX=\$9 -if [[ "\${lrun_subdirs}" == ".true." ]]; then - prefix="dir.*/" +diagtype[0]="conv conv_gps conv_ps conv_pw conv_q conv_sst conv_t conv_tcp conv_uv conv_spd" +diagtype[1]="pcp_ssmi_dmsp pcp_tmi_trmm" +if [[ ${USE_BUILD_GSINFO} == "YES" ]]; then + diagtype[2]=$(cat "${BUILD_GSINFO_DIR}/ozinfo/satellites") + diagtype[3]=$(cat "${BUILD_GSINFO_DIR}/satinfo/satellites") else - prefix="pe*" -fi -count=\$(find -L ./ -path "./\${prefix}\${type}_\${loop}*" -type f -printf '.' | wc -c) -file="diag_\${type}_\${string}.\${PDY}\${cyc}\${DIAG_SUFFIX}" -if [[ "\${binary_diag}" == ".true." ]] || [[ "\${count}" -eq 1 ]]; then - cat \${prefix}\${type}_\${loop}* > "\${file}" -else - ${CATEXEC} -o "\${file}" \${prefix}\${type}_\${loop}* + diagtype[2]="sbuv2_n16 sbuv2_n17 sbuv2_n18 sbuv2_n19 gome_metop-a gome_metop-b omi_aura mls30_aura ompsnp_npp ompstc8_npp ompstc8_n20 ompsnp_n20 ompstc8_n21 ompsnp_n21 ompslp_npp gome_metop-c" + diagtype[3]="msu_n14 sndr_g08 sndr_g11 sndr_g12 sndr_g13 sndr_g08_prep sndr_g11_prep sndr_g12_prep sndr_g13_prep sndrd1_g11 sndrd2_g11 sndrd3_g11 sndrd4_g11 sndrd1_g12 sndrd2_g12 sndrd3_g12 sndrd4_g12 sndrd1_g13 sndrd2_g13 sndrd3_g13 sndrd4_g13 sndrd1_g14 sndrd2_g14 sndrd3_g14 sndrd4_g14 sndrd1_g15 sndrd2_g15 sndrd3_g15 sndrd4_g15 amsua_n15 amsua_n16 amsua_n17 amsub_n15 amsub_n16 amsub_n17 hsb_aqua airs_aqua amsua_aqua imgr_g08 imgr_g11 imgr_g12 imgr_g14 imgr_g15 ssmi_f13 ssmi_f15 amsua_n18 amsua_metop-a mhs_n18 mhs_metop-a amsre_low_aqua amsre_mid_aqua amsre_hig_aqua ssmis_f16 ssmis_f17 ssmis_f18 ssmis_f19 ssmis_f20 iasi_metop-a amsua_n19 mhs_n19 seviri_m08 seviri_m09 seviri_m10 seviri_m11 cris_npp cris-fsr_npp cris-fsr_n20 atms_npp atms_n20 amsua_metop-b mhs_metop-b iasi_metop-b avhrr_metop-b avhrr_n18 avhrr_n19 avhrr_metop-a amsr2_gcom-w1 gmi_gpm saphir_meghat ahi_himawari8 abi_g16 abi_g17 amsua_metop-c mhs_metop-c iasi_metop-c avhrr_metop-c viirs-m_npp viirs-m_j1 abi_g18 ahi_himawari9 viirs-m_j2 cris-fsr_n21 atms_n21 abi_g19" fi -if [[ "\${DIAG_COMPRESS}" == "YES" ]]; then - ${COMPRESS} "\${file}" + +diaglist[0]=listcnv +diaglist[1]=listpcp +diaglist[2]=listozn +diaglist[3]=listrad + +diagfile[0]="${CNVSTAT}" +diagfile[1]="${PCPSTAT}" +diagfile[2]="${OZNSTAT}" +diagfile[3]="${RADSTAT}" + +numfile[0]=0 +numfile[1]=0 +numfile[2]=0 +numfile[3]=0 + +rm -f "${DATA}/diag.sh" +cat > "${DATA}/diag.sh" << EOF +#!/bin/bash +set -x + +type=\$1 +loop=\$2 +string=\$3 +count=\$4 +suffix=\$5 + +# Match files with this prefix +diag_files=dir.*/\${type}_\${loop} + +# Name of combined diagnostic file from matched files +out_diag_file=diag_\${type}_\${string}.${PDY}${cyc}\${suffix} + +# Combine diagnostic files +if [[ \${count} -gt 1 ]]; then + ${CATEXEC} -o \${out_diag_file} \${diag_files}* +else + cat \${diag_files}* > "\${out_diag_file}" fi -EOFdiag - chmod 755 "${DATA}/diag.sh" - - # Collect diagnostic files as a function of loop and type. - # Loop over first and last outer loops to generate innovation - # diagnostic files for indicated observation types (groups) - # - # NOTE: Since we set miter=2 in GSI namelist SETUP, outer - # loop 03 will contain innovations with respect to - # the analysis. Creation of o-a innovation files - # is triggered by write_diag(3)=.true. The setting - # write_diag(1)=.true. turns on creation of o-g - # innovation files. - - loops="01 03" - for loop in ${loops}; do - case ${loop} in - 01) string=ges ;; - 03) string=anl ;; - *) string=${loop} ;; - esac - # shellcheck disable=SC2312 - echo "$(date) START loop ${string}" >&2 - for ((n = 0; n < ${#diagtype[@]}; n++)); do - for type in ${diagtype[n]}; do - # shellcheck disable=SC2312 - count=$(find -L ./ -path "./${prefix}${type}_${loop}*" -type f -printf '.' | wc -c) - if [[ "${count}" -gt 0 ]]; then - echo "${DATA}/diag.sh ${lrun_subdirs} ${binary_diag} ${type} ${loop} ${string} ${PDY} ${cyc} ${DIAG_COMPRESS} ${DIAG_SUFFIX}" | tee -a "${DATA}/mp_diag.sh" - echo "diag_${type}_${string}.${PDY}${cyc}${DIAG_SUFFIX}${COMPRESS_SUFFIX}" >> "${diaglist[n]}" - fi - done - done - # shellcheck disable=SC2312 - echo "$(date) END loop ${string}" >&2 - done - # We should already be in $DATA, but extra cd to be sure. - cd "${DATA}" || exit 1 +# Compress diagnostic file if requested +if [[ "${DIAG_COMPRESS:-}" == "YES" ]]; then + ${COMPRESS} "\${out_diag_file}" +fi - export err=0 - if [[ -s "${DATA}/mp_diag.sh" ]]; then - chmod 755 "${DATA}/mp_diag.sh" - "${USHgfs}/run_mpmd.sh" "${DATA}/mp_diag.sh" && true - # Delay err exit to avoid rstprod leakage - err=$? - fi +exit 0 +EOF +chmod 755 "${DATA}/diag.sh" - # Restrict diagnostic files containing rstprod data - if [[ "${CHGRP_RSTPROD}" == "YES" ]]; then - rlist="conv_gps conv_ps conv_pw conv_q conv_sst conv_t conv_uv saphir" - for rtype in ${rlist}; do - for rfile in *"${rtype}"*; do - if [[ -s "${rfile}" ]]; then - ${CHGRP_CMD} "${rfile}" - fi - done +# Collect diagnostic files as a function of loop and type. +# Loop over first and last outer loops to generate innovation +# diagnostic files for indicated observation types (groups) +# +# NOTE: Since we set miter=2 in GSI namelist SETUP, outer +# loop 03 will contain innovations with respect to +# the analysis. Creation of o-a innovation files +# is triggered by write_diag(3)=.true. The setting +# write_diag(1)=.true. turns on creation of o-g +# innovation files. + +rm -f cmdfile +touch cmdfile +loops="01 03" +for loop in ${loops}; do + case ${loop} in + 01) string=ges ;; + 03) string=anl ;; + *) string=${loop} ;; + esac + echo "START loop ${string}" + n=-1 + while [[ ${n} -lt ${ntype} ]]; do + n=$((n + 1)) + for type in ${diagtype[n]}; do + #shellcheck disable=SC2012,SC2312 + count=$(ls dir.*/"${type}_${loop}"* 2> /dev/null | wc -l) + if [[ ${count} -eq 0 ]]; then + continue + fi + echo "${DATA}/diag.sh ${type} ${loop} ${string} ${count} ${DIAG_SUFFIX:-}.nc4" >> cmdfile + echo "diag_${type}_${string}.${PDY}${cyc}${DIAG_SUFFIX:-}.nc4${COMPRESS_SUFFIX:-}" >> "${diaglist[n]}" + numfile[n]=$((numfile[n] + 1)) done - fi + done + echo "END loop ${string}" +done - # Now exit if compression failed +ncmd=$(wc -l < cmdfile) +if [[ ${ncmd} -eq 0 ]]; then + echo "WARNING: No diagnostic files found to process!" + exit 0 +fi + +# MPMD can only be executed on a single node, +# so break up cmdfile into parts of tasks_per_node size files +# and run them sequentially +split -l "${tasks_per_node}" ./cmdfile cmdfile_part_ +cmdfile_parts=$(ls cmdfile_part_*) +for partfile in ${cmdfile_parts}; do + "${USHgfs}/run_mpmd.sh" "${partfile}" && true + export err=$? if [[ ${err} -ne 0 ]]; then - err_exit "Failed to compress one or more observation diagnostic files!" + err_exit "Failed to create one or more observation diagnostic files for ${partfile}!" fi - - # If requested, create diagnostic file tarballs - if [[ "${DIAG_TARBALL}" == "YES" ]]; then - # shellcheck disable=SC2312 - echo "$(date) START tar diagnostic files" >&2 - for ((n = 0; n < ${#diagtype[@]}; n++)); do - TAROPTS="-uvf" - if [[ ! -s "${diagfile[n]}" ]]; then - TAROPTS="-cvf" - fi - if [[ -s "${diaglist[n]}" ]]; then - tar "${TAROPTS}" "${diagfile[n]}" -T "${diaglist[n]}" - export err=$? - if [[ ${err} -ne 0 ]]; then - # Delay err_exit to avoid rstprod leakage - err_msg="Unable to create ${diagfile[n]}!" - break - fi +done + +# Restrict diagnostic files containing rstprod data +if [[ "${CHGRP_RSTPROD}" == "YES" ]]; then + rlist="conv_gps conv_ps conv_pw conv_q conv_sst conv_t conv_uv saphir" + for rtype in ${rlist}; do + for rfile in *"${rtype}"*; do + if [[ -s "${rfile}" ]]; then + ${CHGRP_CMD} "${rfile}" fi done + done +fi - # Restrict CNVSTAT - chmod 750 "${CNVSTAT}" - if [[ "${CHGRP_RSTPROD}" == "YES" ]]; then - ${CHGRP_CMD} "${CNVSTAT}" +# If requested, create diagnostic file tarballs +if [[ "${DIAG_TARBALL}" == "YES" ]]; then + echo "START tar diagnostic files" + n=-1 + while [[ ${n} -lt ${ntype} ]]; do + n=$((n + 1)) + TAROPTS="-uvf" + if [[ ! -s "${diagfile[n]}" ]]; then + TAROPTS="-cvf" fi - - # Restrict RADSTAT - if [[ -s "${RADSTAT}" ]]; then - chmod 750 "${RADSTAT}" - if [[ "${CHGRP_RSTPROD}" == "YES" ]]; then - ${CHGRP_CMD} "${RADSTAT}" + if [[ ${numfile[n]} -gt 0 ]]; then + tar "${TAROPTS}" "${diagfile[n]}" -T "${diaglist[n]}" + export err=$? + if [[ ${err} -ne 0 ]]; then + err_exit "Unable to create ${diagfile[n]}!" fi fi + done + echo "END tar diagnostic files" - # Now exit if there was an error - if [[ ${err} -ne 0 ]]; then - err_exit "${err_msg}" - fi + # Restrict CNVSTAT + chmod 750 "${CNVSTAT}" + if [[ "${CHGRP_RSTPROD}" == "YES" ]]; then + ${CHGRP_CMD} "${CNVSTAT}" + fi - # shellcheck disable=SC2312 - echo "$(date) END tar diagnostic files" >&2 + # Restrict RADSTAT + if [[ -s "${RADSTAT}" ]]; then + chmod 750 "${RADSTAT}" + if [[ "${CHGRP_RSTPROD}" == "YES" ]]; then + ${CHGRP_CMD} "${RADSTAT}" + fi fi -fi # End diagnostic file generation block - if [[ "${GENDIAG}" == "YES" ]] +fi ################################################################################ # Postprocessing -# Remove $DIAG_DIR -if [[ "${REMOVE_DIAG_DIR}" == "YES" ]]; then - rm -rf "${DIAG_DIR}" -fi exit 0 diff --git a/sorc/build_gdas.sh b/sorc/build_gdas.sh index 32213153423..e49cc00429c 100755 --- a/sorc/build_gdas.sh +++ b/sorc/build_gdas.sh @@ -13,11 +13,9 @@ while getopts ":j:dv" option; do v) _opts+="-v ";; :) echo "[${BASH_SOURCE[0]}]: ${option} requires an argument" - usage ;; *) echo "[${BASH_SOURCE[0]}]: Unrecognized option: ${option}" - usage ;; esac done diff --git a/sorc/build_gsi_enkf.sh b/sorc/build_gsi_enkf.sh index 09cf250acf2..badba6c332d 100755 --- a/sorc/build_gsi_enkf.sh +++ b/sorc/build_gsi_enkf.sh @@ -9,11 +9,9 @@ while getopts ":j:dv" option; do v) BUILD_VERBOSE="YES";; :) echo "[${BASH_SOURCE[0]}]: ${option} requires an argument" - usage ;; *) echo "[${BASH_SOURCE[0]}]: Unrecognized option: ${option}" - usage ;; esac done @@ -33,4 +31,3 @@ REGRESSION_TESTS=NO \ "${HOMEgfs_}/sorc/gsi_enkf.fd/ush/build.sh" exit - diff --git a/sorc/build_gsi_monitor.sh b/sorc/build_gsi_monitor.sh index d355067f127..e982f14f703 100755 --- a/sorc/build_gsi_monitor.sh +++ b/sorc/build_gsi_monitor.sh @@ -12,11 +12,9 @@ while getopts ":j:dv" option; do v) BUILD_VERBOSE="YES";; :) echo "[${BASH_SOURCE[0]}]: ${option} requires an argument" - usage ;; *) echo "[${BASH_SOURCE[0]}]: Unrecognized option: ${option}" - usage ;; esac done diff --git a/sorc/build_ufs_utils.sh b/sorc/build_ufs_utils.sh index 2284b151343..e6698bb5008 100755 --- a/sorc/build_ufs_utils.sh +++ b/sorc/build_ufs_utils.sh @@ -12,11 +12,9 @@ while getopts ":j:dv" option; do v) BUILD_VERBOSE="YES";; :) echo "[${BASH_SOURCE[0]}]: ${option} requires an argument" - usage ;; *) echo "[${BASH_SOURCE[0]}]: Unrecognized option: ${option}" - usage ;; esac done @@ -31,4 +29,3 @@ BUILD_VERBOSE=${BUILD_VERBOSE:-} \ "${HOMEgfs_}/sorc/ufs_utils.fd/build_all.sh" exit - diff --git a/sorc/build_ww3prepost.sh b/sorc/build_ww3prepost.sh index 19133b12972..5408a5c390c 100755 --- a/sorc/build_ww3prepost.sh +++ b/sorc/build_ww3prepost.sh @@ -16,11 +16,9 @@ while getopts ":j:a:dvw" option; do w) PDLIB="OFF" ;; :) echo "[${BASH_SOURCE[0]}]: ${option} requires an argument" - usage ;; *) echo "[${BASH_SOURCE[0]}]: Unrecognized option: ${option}" - usage ;; esac done diff --git a/ush/gaussian_sfcanl.sh b/ush/gaussian_sfcanl.sh index 70780e63a8a..0652c0117c5 100755 --- a/ush/gaussian_sfcanl.sh +++ b/ush/gaussian_sfcanl.sh @@ -23,35 +23,12 @@ # CASE Forecast model and restart resolution. Defaults to C768. # CASE_HIST History file output resolution. Defaults to $CASE. # DONST Process NST fields when 'yes'. Default is 'no'. -# OUTPUT_FILE Output gaussian analysis file format. Default is "nemsio" -# Set to "netcdf" for netcdf output file -# Otherwise, output in nemsio. # FIXWGTS Weight file to use for interpolation -# DATA Working directory -# (if nonexistent will be made, used and deleted) -# Defaults to current working directory # COMOUT Output directory # (if nonexistent will be made) # defaults to current working directory -# XC Suffix to add to executables. Defaults to none. # GAUSFCANLEXE Program executable. # Defaults to $EXECgfs/gaussian_sfcanl.x -# INISCRIPT Preprocessing script. Defaults to none. -# LOGSCRIPT Log posting script. Defaults to none. -# ENDSCRIPT Postprocessing script -# defaults to none -# PGMOUT Executable standard output -# defaults to $pgmout, then to '&1' -# PGMERR Executable standard error -# defaults to $pgmerr, then to '&1' -# pgmout Executable standard output default -# pgmerr Executable standard error default -# REDOUT standard output redirect ('1>' or '1>>') -# defaults to '1>', or to '1>>' to append if $PGMOUT is a file -# REDERR standard error redirect ('2>' or '2>>') -# defaults to '2>', or to '2>>' to append if $PGMERR is a file -# VERBOSE Verbose flag (YES or NO) -# defaults to NO # gfs_ver Version number of gfs directory. Default is # v15.0.0. # OMP_NUM_ @@ -60,15 +37,11 @@ # Default is none. # # Exported Shell Variables: -# PGM Current program name # pgm -# ERR Last return code # err # # Modules and files referenced: -# scripts : $INISCRIPT -# $LOGSCRIPT -# $ENDSCRIPT +# scripts : # # programs : $GAUSFCANLEXE # @@ -77,10 +50,9 @@ # ${FIXgfs}/am/global_hyblev.l65.txt # # input data : ${COMIN_ATMOS_RESTART}/${PDY}.${cyc}0000.sfcanl_data.tile*.nc +# ${COMIN_ATMOS_ANALYSIS}/increment.sfc.i006.tile${i}.nc" # -# output data: $PGMOUT -# $PGMERR -# $COMOUT/${APREFIX}analysis.sfc.a006.nc +# output data: $COMOUT/${APREFIX}analysis.sfc.a006.nc # # Remarks: # @@ -95,7 +67,6 @@ # # Attributes: # Language: POSIX shell -# Machine: IBM SP # ################################################################################ @@ -104,94 +75,69 @@ CASE_HIST=${CASE_HIST:-${CASE}} resh=${CASE_HIST:1} LONB_CASE=$((resh*4)) LATB_CASE=$((resh*2)) -LONB_SFC=${LONB_SFC:-$LONB_CASE} -LATB_SFC=${LATB_SFC:-$LATB_CASE} +LONB_SFC=${LONB_SFC:-${LONB_CASE}} +LATB_SFC=${LATB_SFC:-${LATB_CASE}} DONST=${DONST:-"NO"} LEVS=${LEVS:-64} -LEVSP1=$(($LEVS+1)) +LEVSP1=$(( LEVS + 1 )) FIXWGTS=${FIXWGTS:-${FIXorog}/${CASE}/fv3_SCRIP_${CASE}_GRIDSPEC_lon${LONB_SFC}_lat${LATB_SFC}.gaussian.neareststod.nc} # Filenames. -XC=${XC:-} -GAUSFCANLEXE=${GAUSFCANLEXE:-$EXECgfs/gaussian_sfcanl.x} +GAUSFCANLEXE=${GAUSFCANLEXE:-${EXECgfs}/gaussian_sfcanl.x} SIGLEVEL=${SIGLEVEL:-${FIXgfs}/am/global_hyblev.l${LEVSP1}.txt} # Other variables. -export PGMOUT=${PGMOUT:-${pgmout:-'&1'}} -export PGMERR=${PGMERR:-${pgmerr:-'&2'}} -export REDOUT=${REDOUT:-'1>'} -export REDERR=${REDERR:-'2>'} # Set defaults ################################################################################ # Preprocessing -${INISCRIPT:-} -pwd=$(pwd) -if [[ ! -d "${COMOUT_ATMOS_ANALYSIS}" ]]; then - mkdir -p "${COMOUT_ATMOS_ANALYSIS}" -fi ################################################################################ # Make surface analysis -export PGM=$GAUSFCANLEXE -export pgm=$PGM -$LOGSCRIPT - -iy=${PDY:0:4} -im=${PDY:4:2} -id=${PDY:6:2} -ih=${cyc} - -export OMP_NUM_THREADS=${OMP_NUM_THREADS_SFC:-1} - # input interpolation weights -${NLN} "${FIXWGTS}" "./weights.nc" +cpreq "${FIXWGTS}" "./weights.nc" # input analysis tiles (with nst records) -${NLN} "${COMIN_ATMOS_RESTART}/${PDY}.${cyc}0000.sfcanl_data.tile1.nc" "./anal.tile1.nc" -${NLN} "${COMIN_ATMOS_RESTART}/${PDY}.${cyc}0000.sfcanl_data.tile2.nc" "./anal.tile2.nc" -${NLN} "${COMIN_ATMOS_RESTART}/${PDY}.${cyc}0000.sfcanl_data.tile3.nc" "./anal.tile3.nc" -${NLN} "${COMIN_ATMOS_RESTART}/${PDY}.${cyc}0000.sfcanl_data.tile4.nc" "./anal.tile4.nc" -${NLN} "${COMIN_ATMOS_RESTART}/${PDY}.${cyc}0000.sfcanl_data.tile5.nc" "./anal.tile5.nc" -${NLN} "${COMIN_ATMOS_RESTART}/${PDY}.${cyc}0000.sfcanl_data.tile6.nc" "./anal.tile6.nc" +cpreq "${COMIN_ATMOS_RESTART}/${PDY}.${cyc}0000.sfcanl_data.tile1.nc" "./anal.tile1.nc" +cpreq "${COMIN_ATMOS_RESTART}/${PDY}.${cyc}0000.sfcanl_data.tile2.nc" "./anal.tile2.nc" +cpreq "${COMIN_ATMOS_RESTART}/${PDY}.${cyc}0000.sfcanl_data.tile3.nc" "./anal.tile3.nc" +cpreq "${COMIN_ATMOS_RESTART}/${PDY}.${cyc}0000.sfcanl_data.tile4.nc" "./anal.tile4.nc" +cpreq "${COMIN_ATMOS_RESTART}/${PDY}.${cyc}0000.sfcanl_data.tile5.nc" "./anal.tile5.nc" +cpreq "${COMIN_ATMOS_RESTART}/${PDY}.${cyc}0000.sfcanl_data.tile6.nc" "./anal.tile6.nc" # input orography tiles -${NLN} "${FIXorog}/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile1.nc" "./orog.tile1.nc" -${NLN} "${FIXorog}/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile2.nc" "./orog.tile2.nc" -${NLN} "${FIXorog}/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile3.nc" "./orog.tile3.nc" -${NLN} "${FIXorog}/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile4.nc" "./orog.tile4.nc" -${NLN} "${FIXorog}/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile5.nc" "./orog.tile5.nc" -${NLN} "${FIXorog}/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile6.nc" "./orog.tile6.nc" - -${NLN} "${SIGLEVEL}" "./vcoord.txt" +cpreq "${FIXorog}/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile1.nc" "./orog.tile1.nc" +cpreq "${FIXorog}/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile2.nc" "./orog.tile2.nc" +cpreq "${FIXorog}/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile3.nc" "./orog.tile3.nc" +cpreq "${FIXorog}/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile4.nc" "./orog.tile4.nc" +cpreq "${FIXorog}/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile5.nc" "./orog.tile5.nc" +cpreq "${FIXorog}/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile6.nc" "./orog.tile6.nc" -# output gaussian global surface analysis files -${NLN} "${COMOUT_ATMOS_ANALYSIS}/${APREFIX}analysis.sfc.a006.nc" "./sfc.gaussian.analysis.file" +cpreq "${SIGLEVEL}" "./vcoord.txt" # Namelist uses booleans now -if [[ ${DONST} == "YES" ]]; then do_nst='.true.'; else do_nst='.false.'; fi +if [[ "${DONST}" == "YES" ]]; then + do_nst=".true." +else + do_nst=".false." +fi -#Add soil increments to gdas gaussian sfcanal if they are not added by gcycle (i.e., when landiau=true) +# Add soil increments to gdas gaussian sfcanal if they are not added by gcycle (i.e., when landiau=true) LSOIL_INCR=${LSOIL_INCR:-2} -if [[ "${DO_LAND_IAU:-.false.}" == ".true." ]]; then +if [[ "${DO_LAND_IAU:-.false.}" == ".true." ]]; then for i in $(seq 1 6); do - sfc_inc="${COMOUT_ATMOS_ANALYSIS}/increment.sfc.i006.tile${i}.nc" - if [[ ! -f "${sfc_inc}" ]]; then - echo "Error! gaussian sfc analysis missing increment file ${sfc_inc}" - exit 1 - else - ${NLN} "${sfc_inc}" "./sfc_inc.tile${i}.nc" - fi + sfc_inc="${COMIN_ATMOS_ANALYSIS}/increment.sfc.i006.tile${i}.nc" + cpreq "${sfc_inc}" "./sfc_inc.tile${i}.nc" done fi # Executable namelist cat < fort.41 - &setup - yy=${iy}, - mm=${im}, - dd=${id}, - hh=${ih}, +&setup + yy=${PDY:0:4}, + mm=${PDY:4:2}, + dd=${PDY:6:2}, + hh=${cyc}, igaus=${LONB_SFC}, jgaus=${LATB_SFC}, donst=${do_nst}, @@ -200,19 +146,25 @@ cat < fort.41 add_soil_inc=${DO_LAND_IAU}, lsoil_incr=${LSOIL_INCR}, sfc_inc_file="./sfc_inc", - / +/ EOF +cat fort.41 +export pgm="${GAUSFCANLEXE}" +export OMP_NUM_THREADS=${OMP_NUM_THREADS_SFC:-1} ${APRUNSFC} "${GAUSFCANLEXE}" - export err=$? if [[ ${err} -ne 0 ]]; then echo "FATAL ERROR: ${GAUSFCANLEXE} returned non-zero exit status!" exit "${err}" fi +# output gaussian global surface analysis files +if [[ -f "sfc.gaussian.analysis.file" ]]; then + cpfs "./sfc.gaussian.analysis.file" "${COMOUT_ATMOS_ANALYSIS}/${APREFIX}analysis.sfc.a006.nc" +fi + ################################################################################ # Postprocessing -cd "${pwd}" exit 0 diff --git a/ush/python/pygfs/task/stage_ic.py b/ush/python/pygfs/task/stage_ic.py index cbec76c78f0..873d4346593 100644 --- a/ush/python/pygfs/task/stage_ic.py +++ b/ush/python/pygfs/task/stage_ic.py @@ -220,6 +220,16 @@ def execute_stage(self, stage_dict: AttrDict) -> None: stage_set = parse_j2yaml(stage_dict.STAGE_IC_YAML_TMPL, stage_dict, allow_missing=False) + # Parse staging yaml to get list of files to stage + stage_set = parse_j2yaml(self.task_config.STAGE_IC_YAML_TMPL, stage_dict, allow_missing=False) + logger.info(stage_set) + + # Write stage_set to a file in DATA for debugging + stage_yaml = f"./stage{stage_dict.member:03}.yaml" + stage_set.save(stage_yaml) + logger.debug(f"Staging yaml written to '{stage_yaml}' for debugging purposes.") + + # stage files to ROTDIR for key in stage_set.keys(): FileHandler(stage_set[key]).sync() diff --git a/ush/regrid_gsiSfcIncr_to_tile.sh b/ush/regrid_gsiSfcIncr_to_tile.sh index aa922229f6c..5e2fe1cf59f 100755 --- a/ush/regrid_gsiSfcIncr_to_tile.sh +++ b/ush/regrid_gsiSfcIncr_to_tile.sh @@ -9,12 +9,6 @@ source "${HOMEgfs}/ush/atparse.bash" # David New, Nov 2025 (parallelization updates) #------------------------------------------------------------------------------------------------- -export PGMOUT=${PGMOUT:-${pgmout:-'&1'}} -export PGMERR=${PGMERR:-${pgmerr:-'&2'}} - -export PGM=${REGRID_EXEC} -export pgm=${PGM} - NMEM_REGRID=${NMEM_REGRID:-1} CASE_IN=${CASE_IN:-${CASE_ENS}} LFHR=${LFHR:-6} @@ -31,13 +25,9 @@ LSOIL_INCR=${LSOIL_INCR:-2} export n_vars=$(( LSOIL_INCR*2 )) -soil_incr_vars="" -for vi in $( seq 1 "${LSOIL_INCR}" ); do - soil_incr_vars=${soil_incr_vars}'"soilt'${vi}'_inc"', -done -for vi in $( seq 1 "${LSOIL_INCR}" ); do - soil_incr_vars=${soil_incr_vars}'"slc'${vi}'_inc"', -done +soilt_incr_vars=$(seq -s ',' -f '"soilt%g_inc"' 1 "${LSOIL_INCR}") +slc_incr_vars=$(seq -s ',' -f '"slc%g_inc"' 1 "${LSOIL_INCR}") +export soil_incr_vars="${soilt_incr_vars},${slc_incr_vars}" if [[ "${DO_LAND_IAU}" = ".true." ]]; then IFS=',' read -ra landifhrs <<< "${IAUFHRS}" @@ -52,9 +42,9 @@ export jres=${LATB_CASE_IN} export ireso=${CASE_OUT:1} export jreso=${CASE_OUT:1} -regrid_nml_tmpl="${PARMgfs}/regrid_sfc/regrid.nml_tmpl" +regrid_nml_tmpl="${PARMgfs}/regrid_sfc/regrid.nml_tmpl" -if (( LFHR >= 0 )); then +if [[ "${LFHR}" -ge 0 ]]; then soilinc_fhrs=("${LFHR}") else # construct restart times for deterministic member soilinc_fhrs=("${assim_freq}") # increment file at middle of window @@ -64,14 +54,14 @@ else # construct restart times for deterministic member fi fi -# # Stage input files -# + +cd "${DATA}" || exit 1 # Create MDMD command file for fixed files -rm -f cmdfile.0 -touch cmdfile.0 -chmod 755 cmdfile.0 +rm -f cmdfile_in.0 +touch cmdfile_in.0 +chmod 755 cmdfile_in.0 # Append fixed files command file to master command file { @@ -91,80 +81,148 @@ for n in $(seq 1 "${ntiles}"); do echo "cpreq ${FIXorog}/${CASE_OUT}/${CASE_OUT}_grid.tile${n}.nc \ ${DATA}/${CASE_OUT}_grid.tile${n}.nc" done -} > cmdfile.0 +} >> cmdfile_in.0 + +if [[ "${NMEM_REGRID}" -gt 1 ]]; then + + echo "INFO: Preparing to regrid surface increments for ${NMEM_REGRID} ensemble members." + for imem in $(seq 1 "${NMEM_REGRID}"); do + + memdir=$(printf "mem%03i" "${imem}") + + MEMDIR=${memdir} YMD=${PDY} HH=${cyc} declare_from_tmpl \ + COMIN_SOIL_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL \ + COMOUT_ATMOS_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL + + # Create MPMD command file for this member + rm -f "cmdfile_in.${imem}" "cmdfile_out.${imem}" + touch "cmdfile_in.${imem}" "cmdfile_out.${imem}" + chmod 755 "cmdfile_in.${imem}" "cmdfile_out.${imem}" -for imem in $(seq 1 "${NMEM_REGRID}"); do - cmem=$(printf %03i "${imem}") - memchar="mem${cmem}" + # Create commands to stage input files + { + echo "#!/bin/bash" - # If deterministic job, COMOUT_ATMOS_ANALYSIS_MEM is just COMOUT_ATMOS_ANALYSIS - if (( NMEM_REGRID > 1 )); then - MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \ - COMIN_SOIL_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL + echo "mkdir -p ${DATA}/${memdir}" - memdir="${DATA}/${memchar}" - mkdir -p "${memdir}" + for FHR in "${soilinc_fhrs[@]}"; do + echo "cpreq ${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}increment.sfc.i00${FHR}.nc \ + ${DATA}/${memdir}/sfci00${FHR}.nc" + done - if [[ "${imem}" -gt 1 ]]; then - in_dir+=", " + if [[ "${DO_LAND_IAU}" = ".true." ]]; then + for FHI in "${landifhrs[@]}"; do + echo "cpreq ${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}increment.sfc.i00${FHI}.nc \ + ${DATA}/${memdir}/sfci00${FHI}.nc" + done + fi + } >> "cmdfile_in.${imem}" + + # Create commands to copy output files + { + echo "#!/bin/bash" + + if [[ "${DO_LAND_IAU}" = ".false." || "${RUN}" == "gdas" || "${RUN}" == "gfs" ]]; then + for FHR in "${soilinc_fhrs[@]}"; do + for n in $(seq 1 "${ntiles}"); do + echo "cpfs ${DATA}/${memdir}/sfci00${FHR}.mem${imem}.tile${n}.nc \ + ${COMOUT_ATMOS_ANALYSIS_MEM}/increment.sfc.i00${FHR}.tile${n}.nc" + done + done fi - in_dir+="\"./${memchar}/\"" - else - # If deterministic job, memdir is just DATA - memdir="${DATA}" - in_dir="'./'" - fi + if [[ "${DO_LAND_IAU}" = ".true." ]]; then + for n in $(seq 1 "${ntiles}"); do + echo "cpfs ${DATA}/${memdir}/sfci.mem${imem}.tile${n}.nc \ + ${COMOUT_ATMOS_ANALYSIS_MEM}/increment.sfc.tile${n}.nc" + done + fi + } >> "cmdfile_out.${imem}" - # Create MPMD command file for this member - rm -f "cmdfile.${imem}" - touch "cmdfile.${imem}" - chmod 755 "cmdfile.${imem}" + done # for imem in $(seq 1 "${NMEM_REGRID}"); do + in_dir=$(seq -s ", " -f "'./mem%03g/'" 1 "${NMEM_REGRID}") - # Create commands to stage input files - { - echo "#!/bin/bash" +else # deterministic member only (NMEM_REGRID=1) + echo "INFO: Preparing to regrid surface increments for deterministic member." + + # Create commands to stage input files and append to the cmdfile.0 + { for FHR in "${soilinc_fhrs[@]}"; do - echo "cpreq ${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}increment.sfc.i00${FHR}.nc \ - ${memdir}/sfci00${FHR}.nc" - done + echo "cpreq ${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}ensmean_increment.sfc.i00${FHR}.nc \ + ${DATA}/sfci00${FHR}.nc" + done - if [[ "${DO_LAND_IAU}" = ".true." ]]; then + if [[ "${DO_LAND_IAU}" = ".true." ]]; then for FHI in "${landifhrs[@]}"; do - echo "cpreq ${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}increment.sfc.i00${FHI}.nc \ - ${memdir}/sfci00${FHI}.nc" + echo "cpreq ${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}ensmean_increment.sfc.i00${FHI}.nc \ + ${DATA}/sfci00${FHI}.nc" done fi - } > "cmdfile.${imem}" -done + } >> "cmdfile_in.0" + in_dir="'./'" -# Create master MPMD command file -rm -f cmdfile -touch cmdfile -chmod 755 cmdfile + # Create commands to copy output files + rm -f cmdfile_out.0 + touch cmdfile_out.0 + chmod 755 cmdfile_out.0 + { + echo "#!/bin/bash" + + if [[ "${DO_LAND_IAU}" = ".false." || "${RUN}" == "gdas" || "${RUN}" == "gfs" ]]; then + for FHR in "${soilinc_fhrs[@]}"; do + for n in $(seq 1 "${ntiles}"); do + echo "cpfs ${DATA}/sfci00${FHR}.mem1.tile${n}.nc \ + ${COMOUT_ATMOS_ANALYSIS_MEM}/increment.sfc.i00${FHR}.tile${n}.nc" + done + done + fi + + if [[ "${DO_LAND_IAU}" = ".true." ]]; then + for n in $(seq 1 "${ntiles}"); do + echo "cpfs ${DATA}/sfci.mem1.tile${n}.nc \ + ${COMOUT_ATMOS_ANALYSIS_MEM}/increment.sfc.tile${n}.nc" + done + fi + } >> "cmdfile_out.0" + + +fi + +# Create master MPMD command files for input and output +rm -f cmdfile_in cmdfile_out +touch cmdfile_in cmdfile_out # Append all members' command files to master command file -{ -echo "${DATA}/cmdfile.0" # fixed files -for imem in $(seq 1 "${NMEM_REGRID}"); do - echo "${DATA}/cmdfile.${imem}" -done -} >> cmdfile +if [[ -f cmdfile_in.0 ]]; then + echo "${DATA}/cmdfile_in.0" >> cmdfile_in +fi +if [[ -f cmdfile_out.0 ]]; then + echo "${DATA}/cmdfile_out.0" >> cmdfile_out +fi +if [[ "${NMEM_REGRID}" -gt 1 ]]; then + for imem in $(seq 1 "${NMEM_REGRID}"); do + if [[ -f "cmdfile_in.${imem}" ]]; then + echo "${DATA}/cmdfile_in.${imem}" >> cmdfile_in + fi + if [[ -f "cmdfile_out.${imem}" ]]; then + echo "${DATA}/cmdfile_out.${imem}" >> cmdfile_out + fi + done +fi # Run MPMD to stage input files -"${USHgfs}/run_mpmd.sh" "cmdfile" && true +"${USHgfs}/run_mpmd.sh" "cmdfile_in" && true export err=$? if [[ ${err} -ne 0 ]]; then - err_exit "run_mpmd.sh failed!" + err_exit "run_mpmd.sh failed to copy input and fix data!" fi +mv mpmd.out mpmd_in.out # Finish defining input/output directory list export out_dir="${in_dir}" -# -# Regrid soil increments and save to COMOUT -# +# Regrid soil increments # Increments for offline analysis # If land IAU --> deterministic only. If no land IAU --> both deterministic and ensemble @@ -180,10 +238,11 @@ if [[ "${DO_LAND_IAU}" = ".false." || "${RUN}" == "gdas" || "${RUN}" == "gfs" ]] atparse < "${regrid_nml_tmpl}" >> "regrid.nml" # Run regrid executable - ${APRUN_REGRID} "${REGRID_EXEC}" 1>"${PGMOUT}" 2>"${PGMERR}" + export pgm="${REGRID_EXEC}" + ${APRUN_REGRID} "${REGRID_EXEC}" export err=$? if [[ ${err} -ne 0 ]]; then - err_exit "${REGRID_EXEC} failed, ABORT!" + err_exit "${REGRID_EXEC} failed to regrid soil increments (without LANDIAU), ABORT!" fi done fi @@ -202,75 +261,19 @@ if [[ "${DO_LAND_IAU}" = ".true." ]]; then # Run regrid executable export pgm="${REGRID_EXEC}" - ${APRUN_REGRID} "${REGRID_EXEC}" 1>"${PGMOUT}" 2>"${PGMERR}" + ${APRUN_REGRID} "${REGRID_EXEC}" export err=$? if [[ ${err} -ne 0 ]]; then - err_exit "${pgm} failed, ABORT!" + err_exit "${pgm} failed to regrid soil increments (with LANDIAU), ABORT!" fi fi -# -# Save regridded files to COMOUT -# - -for imem in $(seq 1 "${NMEM_REGRID}"); do - cmem=$(printf %03i "${imem}") - memchar="mem${cmem}" - - # If deterministic job, COMOUT_ATMOS_ANALYSIS_MEM is just COMOUT_ATMOS_ANALYSIS - if (( NMEM_REGRID > 1 )); then - MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \ - COMOUT_ATMOS_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL - - memdir="${DATA}/${memchar}" - else - # If deterministic job, memdir is just DATA - memdir="${DATA}" - fi - - # Create MPMD command file for this member - rm -f "cmdfile.${imem}" - touch "cmdfile.${imem}" - chmod 755 "cmdfile.${imem}" - - { - echo "#!/bin/bash" - - if [[ "${DO_LAND_IAU}" = ".false." || "${RUN}" == "gdas" || "${RUN}" == "gfs" ]]; then - for FHR in "${soilinc_fhrs[@]}"; do - for n in $(seq 1 "${ntiles}"); do - echo "cpfs ${memdir}/sfci00${FHR}.mem${imem}.tile${n}.nc \ - ${COMOUT_ATMOS_ANALYSIS_MEM}/increment.sfc.i00${FHR}.tile${n}.nc" - done - done - fi - - if [[ "${DO_LAND_IAU}" = ".true." ]]; then - for n in $(seq 1 "${ntiles}"); do - echo "cpfs ${memdir}/sfci.mem${imem}.tile${n}.nc \ - ${COMOUT_ATMOS_ANALYSIS_MEM}/increment.sfc.tile${n}.nc" - done - fi - } > "cmdfile.${imem}" -done - -# Create master MPMD command file -rm -f cmdfile -touch cmdfile -chmod 755 cmdfile - -# Append all members' command files to master command file -{ -for imem in $(seq 1 "${NMEM_REGRID}"); do - echo "${DATA}/cmdfile.${imem}" -done -} >> cmdfile - # Run MPMD to save output files -"${USHgfs}/run_mpmd.sh" "cmdfile" && true +"${USHgfs}/run_mpmd.sh" "cmdfile_out" && true export err=$? if [[ ${err} -ne 0 ]]; then - err_exit "run_mpmd.sh failed!" + err_exit "run_mpmd.sh failed to copy output files to COMOUT, ABORT!" fi +mv mpmd.out mpmd_out.out exit 0