Skip to content

Commit ec0cf56

Browse files
weihuang-jediweihuang-jedi
authored
Merge branch 'NOAA-EMC:develop' into feature/use_container_prefix
2 parents d7303eb + 4a26a69 commit ec0cf56

16 files changed

Lines changed: 117 additions & 35 deletions

‎dev/jobs/JGLOBAL_ATMOS_ANALYSIS‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
3636
MEMDIR='ensstat' RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
3737
COMIN_ATMOS_HISTORY_ENS_PREV:COM_ATMOS_HISTORY_TMPL
3838

39+
# shellcheck disable=SC2153
3940
mkdir -p "${COMOUT_ATMOS_ANALYSIS}"
4041

4142
export ATMGES="${COMIN_ATMOS_HISTORY_PREV}/${GPREFIX}atm.f006.nc"
@@ -45,6 +46,14 @@ if [[ ! -f "${ATMGES}" ]]; then
4546
err_exit "FILE MISSING: ATMGES == ${ATMGES}"
4647
fi
4748

49+
export GENDIAG="${GENDIAG:-YES}"
50+
51+
# Create a DATAROOT directory variable for the gsidiags if GENDIAG is YES
52+
if [[ "${GENDIAG}" == "YES" ]]; then
53+
pCOMOUT_ATMOS_ANALYSIS="$(dataroot_com_path "${COMOUT_ATMOS_ANALYSIS}")"
54+
export pCOMOUT_ATMOS_ANALYSIS
55+
fi
56+
4857
# Get LEVS
4958
export LEVS
5059
LEVS=$(${NCLEN} "${ATMGES}" pfull) && true

‎dev/jobs/JGLOBAL_ATMOS_ANALYSIS_DIAG‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,11 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
2626
COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL
2727
mkdir -m 775 -p "${COMOUT_ATMOS_ANALYSIS}"
2828

29+
# Create a DATAROOT directory variable where the gsidiags live
30+
# shellcheck disable=SC2153
31+
pCOMIN_ATMOS_ANALYSIS="$(dataroot_com_path "${COMIN_ATMOS_ANALYSIS}")"
32+
export pCOMIN_ATMOS_ANALYSIS
33+
2934
###############################################################
3035
# Run relevant script
3136
${ANALDIAGSH:-${SCRgfs}/exglobal_diag.sh} && true

‎dev/jobs/JGLOBAL_ENKF_DIAG‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,11 @@ MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
2929
COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \
3030
COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL
3131

32+
# Create a DATAROOT directory variable where the gsidiags live
33+
# shellcheck disable=SC2153
34+
pCOMIN_ATMOS_ANALYSIS="$(dataroot_com_path "${COMIN_ATMOS_ANALYSIS}")"
35+
export pCOMIN_ATMOS_ANALYSIS
36+
3237
#RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
3338
# COMIN_OBS_PREV:COM_OBS_TMPL \
3439
# COMIN_ATMOS_ANALYSIS_DET_PREV:COM_ATMOS_ANALYSIS_TMPL

‎dev/jobs/JGLOBAL_ENKF_SELECT_OBS‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ RUN=${RUN/enkf/} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
2828
COMIN_OBS:COM_OBS_TMPL
2929
MEMDIR='ensstat' YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
3030
COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL
31+
# shellcheck disable=SC2153
3132
declare -rx COMOUT_ATMOS_ANALYSIS_ENS="${COMOUT_ATMOS_ANALYSIS}"
3233

3334
RUN=${RUN/enkf/} YMD=${PDY} HH=${cyc} declare_from_tmpl -r \
@@ -40,6 +41,7 @@ MEMDIR='ensstat' RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
4041
RUN="${GDUMP}" YMD=${gPDY} HH=${gcyc} declare_from_tmpl -r \
4142
COMIN_ATMOS_ANALYSIS_DET_PREV:COM_ATMOS_ANALYSIS_TMPL
4243

44+
# shellcheck disable=SC2153
4345
mkdir -p "${COMOUT_ATMOS_ANALYSIS}"
4446

4547
export ATMGES_ENSMEAN="${COMIN_ATMOS_HISTORY_PREV}/${GPREFIX}${GSUFFIX}atm.f006.nc"
@@ -48,6 +50,14 @@ if [[ ! -f ${ATMGES_ENSMEAN} ]]; then
4850
err_exit "FILE MISSING: ATMGES_ENSMEAN == ${ATMGES_ENSMEAN}"
4951
fi
5052

53+
export GENDIAG="${GENDIAG:-YES}"
54+
55+
# Create a DATAROOT directory variable for the gsidiags if GENDIAG is YES
56+
if [[ "${GENDIAG}" == "YES" ]]; then
57+
pCOMOUT_ATMOS_ANALYSIS="$(dataroot_com_path "${COMOUT_ATMOS_ANALYSIS}")"
58+
export pCOMOUT_ATMOS_ANALYSIS
59+
fi
60+
5161
LEVS=$(${NCLEN} "${ATMGES}" pfull) && true
5262
export err=$?
5363
set_strict
@@ -67,7 +77,7 @@ export PREPQCPF="${COMIN_OBS}/${OPREFIX}prepbufr.acft_profiles"
6777

6878
# Deterministic analysis and increment files
6979
export SFCANL="${COMOUT_ATMOS_ANALYSIS_DET}/${APREFIX_DET}analysis.sfc.a006.nc"
70-
export DTFANL="${COMOUT_ATMOS_ANALYSIS_DET}/${APREFIX_DET}increment.dtf.i006.nc"
80+
export DTFINC="${COMOUT_ATMOS_ANALYSIS_DET}/${APREFIX_DET}increment.dtf.i006.nc"
7181
export ATMANL="${COMOUT_ATMOS_ANALYSIS_DET}/${APREFIX_DET}analysis.atm.a006.nc"
7282
export ATMINC="${COMOUT_ATMOS_ANALYSIS_DET}/${APREFIX_DET}increment.atm.i006.nc"
7383

‎dev/scripts/exglobal_atmos_analysis.sh‎

Lines changed: 34 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,7 @@ GSISTAT=${GSISTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}gsistat.txt}
191191

192192
# Increment files
193193
ATMINC=${ATMINC:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}increment.atm.i006.nc}
194-
DTFANL=${DTFANL:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}increment.dtf.i006.nc}
194+
DTFINC=${DTFINC:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}increment.dtf.i006.nc}
195195

196196
# Obs diag
197197
RUN_SELECT=${RUN_SELECT:-"NO"}
@@ -413,6 +413,16 @@ else
413413
echo "not using correlated obs error"
414414
fi
415415

416+
# If GENDIAG is selected, verify that pCOMOUT_ATMOS_ANALYSIS is set
417+
if [[ "${GENDIAG}" == "YES" ]]; then
418+
if [[ -z "${pCOMOUT_ATMOS_ANALYSIS}" ]]; then
419+
export err=1
420+
err_exit "pCOMOUT_ATMOS_ANALYSIS must be set when GENDIAG=YES"
421+
fi
422+
# Make the gsidiags directory to house the GSI diagnostic data
423+
GSIDIAGDIR=${GSIDIAGDIR:-"${pCOMOUT_ATMOS_ANALYSIS}/gsidiags"}
424+
fi
425+
416426
##############################################################
417427
# CRTM Spectral and Transmittance coefficients
418428
mkdir -p crtm_coeffs
@@ -640,7 +650,7 @@ ${NLN} "${ABIASPC}" satbias_pc.out
640650
${NLN} "${ABIASAIR}" aircftbias_out
641651

642652
if [[ "${DONST}" == "YES" ]]; then
643-
${NLN} "${DTFANL}" dtfanl
653+
${NLN} "${DTFINC}" dtfanl
644654
fi
645655

646656
# If requested, link (and if tarred, de-tar obsinput.tar) into obs_input.* files
@@ -895,19 +905,32 @@ if [[ "${SENDECF}" == "YES" && "${RUN}" != "enkf" ]]; then
895905
ecflow_client --event release_fcst
896906
fi
897907

898-
# Diagnostic files
899-
# if requested, GSI diagnostic file directories for use later
908+
echo "${rCDUMP} ${PDY}${cyc} atminc done at $(date)" > "${COMOUT_ATMOS_ANALYSIS}/${APREFIX}increment.done.txt"
909+
900910
if [[ "${GENDIAG}" == "YES" ]]; then
901-
tar -cvf gsidiags.tar dir.????
902-
export err=$?
903-
if [[ ${err} -ne 0 ]]; then
904-
err_exit "Failed to tar GSI diagnostic directories!"
911+
# Move the gsidiags dir.* directories to pCOMOUT_ATMOS_ANALYSIS for diagnostic jobs
912+
# First, check that the directories exist (we need at least one, so stop after the first match)
913+
count_dirs=$(find . -maxdepth 1 -type d -name 'dir.????' -printf "." -quit | wc -c)
914+
if [[ ${count_dirs:-0} -gt 0 ]]; then
915+
mkdir -p "${GSIDIAGDIR}"
916+
err=$?
917+
918+
if [[ ! -d "${GSIDIAGDIR}" || ${err} -ne 0 ]]; then
919+
err_exit "Failed to create gsidiags directory at ${GSIDIAGDIR}"
920+
fi
921+
922+
for dir in dir.????; do
923+
mv "${dir}" "${GSIDIAGDIR}/"
924+
export err=$?
925+
if [[ ${err} -ne 0 ]]; then
926+
err_exit "Failed to move ${dir} to ${GSIDIAGDIR}/"
927+
fi
928+
done
929+
else
930+
echo "WARNING: No gsidiags dir.* directories found to move."
905931
fi
906-
cpfs gsidiags.tar "${COMOUT_ATMOS_ANALYSIS}/${APREFIX}gsidiags${DIAG_SUFFIX:-}.tar"
907932
fi
908933

909-
echo "${rCDUMP} ${PDY}${cyc} atminc done at $(date)" > "${COMOUT_ATMOS_ANALYSIS}/${APREFIX}increment.done.txt"
910-
911934
################################################################################
912935

913936
exit "${err}"

‎dev/scripts/exglobal_atmos_analysis_calc.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ ATMANL=${ATMANL:-"${COMOUT_ATMOS_ANALYSIS}/${APREFIX}analysis.atm.a006.nc"}
8080

8181
# Increment files
8282
ATMINC=${ATMINC:-"${COMOUT_ATMOS_ANALYSIS}/${APREFIX}increment.atm.i006.nc"}
83-
DTFANL=${DTFANL:-"${COMOUT_ATMOS_ANALYSIS}/${APREFIX}increment.dtf.i006.nc"}
83+
DTFINC=${DTFINC:-"${COMOUT_ATMOS_ANALYSIS}/${APREFIX}increment.dtf.i006.nc"}
8484
if [[ "${RUN}" == "gcdas" ]]; then
8585
AEROINC=${AEROINC:-"${COMIN_CHEM_ANALYSIS}/${APREFIX}aeroinc.nc"}
8686
fi

‎dev/scripts/exglobal_atmos_products.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,9 @@ IFS=':' read -ra grids <<< "${grid_string}"
4040
# Files needed by ${USHgfs}/interp_atmos_master.sh
4141
MASTER_FILE="${COMIN_ATMOS_MASTER}/${PREFIX}master.${fhr3}.grib2"
4242

43+
# Create an index file for the master
44+
${WGRIB2} -s "${MASTER_FILE}" > "${MASTER_FILE}.idx"
45+
4346
for ((nset = 1; nset <= downset; nset++)); do
4447

4548
echo "INFO: Begin processing nset = ${nset}"

‎dev/scripts/exglobal_atmos_sfcanl.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -113,8 +113,8 @@ done
113113
if [[ "${DONST}" == "YES" ]]; then
114114
export NST_FILE=${NST_FILE:-${COMIN_ATMOS_ANALYSIS}/${APREFIX}increment.dtf.i006.nc}
115115
if [[ -s "${NST_FILE}" ]]; then
116-
cpreq "${NST_FILE}" "${DATA}/dtfanl"
117-
export NST_FILE="dtfanl"
116+
cpreq "${NST_FILE}" "${DATA}/dtfinc"
117+
export NST_FILE="dtfinc"
118118
else
119119
export NST_FILE="NULL"
120120
fi

‎dev/scripts/exglobal_diag.sh‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ rm -f "${RADSTAT}" "${PCPSTAT}" "${CNVSTAT}" "${OZNSTAT}"
3939

4040
# Obs diag
4141
GENDIAG=${GENDIAG:-"YES"}
42-
GSIDIAG=${GSIDIAG:-"${COMIN_ATMOS_ANALYSIS}/${APREFIX}gsidiags${DIAG_SUFFIX:-}.tar"}
42+
GSIDIAGDIR=${GSIDIAGDIR:-"${pCOMIN_ATMOS_ANALYSIS}/gsidiags"}
4343
USE_BUILD_GSINFO=${USE_BUILD_GSINFO:-"NO"}
4444
DIAG_COMPRESS=${DIAG_COMPRESS:-"YES"}
4545
if [[ "${DIAG_COMPRESS:-}" == "YES" ]]; then
@@ -55,15 +55,30 @@ if [[ "${GENDIAG}" != "YES" ]]; then
5555
exit 0
5656
fi
5757

58+
# Check that the gsidiags directory exists
59+
if [[ ! -d "${GSIDIAGDIR}" ]]; then
60+
export err=1
61+
err_exit "gsidiags directory ${GSIDIAGDIR} does not exist"
62+
fi
63+
5864
################################################################################
59-
# Copy gsidiags.tar file from COMIN to DATA and untar
60-
cpreq "${GSIDIAG}" ./gsidiags.tar
61-
tar -xvf gsidiags.tar
62-
export err=$?
63-
if [[ ${err} -ne 0 ]]; then
64-
err_exit "Unable to unpack gsidiags.tar file!"
65+
# Link to the gsidiags directory if it is populated
66+
count_dirs=$(find "${GSIDIAGDIR}" -maxdepth 1 -type d -name "dir.*" | wc -l)
67+
if [[ ${count_dirs} -eq 0 ]]; then
68+
export err=1
69+
err_exit "No gsidiags directories found in ${GSIDIAGDIR}"
6570
fi
6671

72+
# Continue if there is at least one file to process
73+
# Note -quit stops find after the first match
74+
count_files=$(find "${GSIDIAGDIR}"/dir.* -maxdepth 1 -type f -printf '.' -quit | wc -c)
75+
if [[ ${count_files} -eq 0 ]]; then
76+
echo "WARNING: No diagnostic files found to process!"
77+
exit 0
78+
fi
79+
80+
${NLN} "${GSIDIAGDIR}/"dir.* .
81+
6782
# Set up lists and variables for various types of diagnostic files.
6883
ntype=3
6984

@@ -153,7 +168,8 @@ for loop in ${loops}; do
153168
if [[ "${dtype}" == *"avhrr"* ]]; then # recast dtype in avhrr form
154169
dtype="avhrr${dtype##avhrr[23]}"
155170
fi
156-
count=$(find ./ -path "./dir.*/${dtype}_${loop}*" -type f -printf "." | wc -c)
171+
# -L to follow symlinks, as dir.* are symlinks to gsidiags/dir.*
172+
count=$(find -L ./dir.* -path "./dir.*/${dtype}_${loop}*" -type f -printf "." | wc -c)
157173
if [[ ${count} -eq 0 ]]; then
158174
continue
159175
fi

‎dev/ush/make_ee2_links.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,8 +111,8 @@ for dir in "${gdas_list[@]}" "${gfs_list[@]}" "${gcdas_list[@]}" "${gcafs_list[@
111111
if [[ -f "${system_prefix}.t${cyc}z.cnvstat" ]]; then
112112
link_file "${system_prefix}.t${cyc}z.cnvstat" "${system_prefix}.t${cyc}z.cnvstat.tar"
113113
fi
114-
if [[ -f "${system_prefix}.t${cyc}z.dtfanl.nc" ]]; then
115-
link_file "${system_prefix}.t${cyc}z.dtfanl.nc" "${system_prefix}.t${cyc}z.increment.dtf.i006.nc"
114+
if [[ -f "${system_prefix}.t${cyc}z.dtfinc.nc" ]]; then
115+
link_file "${system_prefix}.t${cyc}z.dtfinc.nc" "${system_prefix}.t${cyc}z.increment.dtf.i006.nc"
116116
fi
117117
if [[ -f "${system_prefix}.t${cyc}z.gsistat" ]]; then
118118
link_file "${system_prefix}.t${cyc}z.gsistat" "${system_prefix}.t${cyc}z.gsistat.txt"

0 commit comments

Comments
 (0)