diff --git a/dev/drivers/scripts/stats/cam/jevs_cam_rrfs_chem_grid2obs_aeronet_aod_stats.sh b/dev/drivers/scripts/stats/cam/jevs_cam_rrfs_chem_grid2obs_aeronet_aod_stats.sh index af14ffb296..db7ec49399 100755 --- a/dev/drivers/scripts/stats/cam/jevs_cam_rrfs_chem_grid2obs_aeronet_aod_stats.sh +++ b/dev/drivers/scripts/stats/cam/jevs_cam_rrfs_chem_grid2obs_aeronet_aod_stats.sh @@ -48,9 +48,6 @@ export RUN=${RUN:-chem} export VERIF_CASE=${VERIF_CASE:-grid2obs} export MODELNAME=${MODELNAME:-rrfs} -export VDATE=${VDATE:-$(date --date="3 days ago" +%Y%m%d)} -echo "VDATE=${VDATE}" - export DATA_TYPE=aeronet_aod export COMIN=/lfs/h2/emc/vpppg/noscrub/$USER/$NET/${evs_ver_2d} diff --git a/dev/drivers/scripts/stats/cam/jevs_cam_rrfs_chem_grid2obs_airnow_pm10_stats.sh b/dev/drivers/scripts/stats/cam/jevs_cam_rrfs_chem_grid2obs_airnow_pm10_stats.sh index 0d7a8478ff..ca6541970b 100755 --- a/dev/drivers/scripts/stats/cam/jevs_cam_rrfs_chem_grid2obs_airnow_pm10_stats.sh +++ b/dev/drivers/scripts/stats/cam/jevs_cam_rrfs_chem_grid2obs_airnow_pm10_stats.sh @@ -49,9 +49,6 @@ export RUN=${RUN:-chem} export VERIF_CASE=${VERIF_CASE:-grid2obs} export MODELNAME=${MODELNAME:-rrfs} -export VDATE=${VDATE:-$(date --date="3 days ago" +%Y%m%d)} -echo "VDATE=${VDATE}" - export DATA_TYPE=airnow_pm10 export COMIN=/lfs/h2/emc/vpppg/noscrub/$USER/$NET/${evs_ver_2d} diff --git a/dev/drivers/scripts/stats/cam/jevs_cam_rrfs_chem_grid2obs_airnow_pm25_stats.sh b/dev/drivers/scripts/stats/cam/jevs_cam_rrfs_chem_grid2obs_airnow_pm25_stats.sh index 364256bee3..a498bb2b46 100755 --- a/dev/drivers/scripts/stats/cam/jevs_cam_rrfs_chem_grid2obs_airnow_pm25_stats.sh +++ b/dev/drivers/scripts/stats/cam/jevs_cam_rrfs_chem_grid2obs_airnow_pm25_stats.sh @@ -49,9 +49,6 @@ export RUN=${RUN:-chem} export VERIF_CASE=${VERIF_CASE:-grid2obs} export MODELNAME=${MODELNAME:-rrfs} -export VDATE=${VDATE:-$(date --date="3 days ago" +%Y%m%d)} -echo "VDATE=${VDATE}" - export DATA_TYPE=airnow_pm25 export COMIN=/lfs/h2/emc/vpppg/noscrub/$USER/$NET/${evs_ver_2d} diff --git a/dev/modulefiles/cam/cam_plots.sh b/dev/modulefiles/cam/cam_plots.sh index 42d7ba0f8e..91bffcae00 100644 --- a/dev/modulefiles/cam/cam_plots.sh +++ b/dev/modulefiles/cam/cam_plots.sh @@ -16,16 +16,16 @@ module load libjpeg/${libjpeg_ver} module load libpng/${libpng_ver} module load zlib/${zlib_ver} module load jasper/${jasper_ver} -module load udunits/${udunits_ver} +##module load udunits/${udunits_ver} module load gsl/${gsl_ver} -module load netcdf/${netcdf_ver} -module load nco/${nco_ver} +## module load netcdf/${netcdf_ver} +##module load nco/${nco_ver} module load prod_util/${prod_util_ver} -module load cdo/${cdo_ver} +##module load cdo/${cdo_ver} module load grib_util/${grib_util_ver} module load wgrib2/${wgrib2_ver} -module load proj/${proj_ver} -module load geos/${geos_ver} +##module load proj/${proj_ver} +##module load geos/${geos_ver} module load imagemagick/${imagemagick_ver} module load met/${met_ver} module load metplus/${metplus_ver} diff --git a/dev/modulefiles/cam/cam_prep.sh b/dev/modulefiles/cam/cam_prep.sh index 8673616c93..385d33a9c0 100644 --- a/dev/modulefiles/cam/cam_prep.sh +++ b/dev/modulefiles/cam/cam_prep.sh @@ -16,12 +16,12 @@ module load libjpeg/${libjpeg_ver} module load libpng/${libpng_ver} module load zlib/${zlib_ver} module load jasper/${jasper_ver} -module load udunits/${udunits_ver} +##module load udunits/${udunits_ver} module load gsl/${gsl_ver} module load netcdf/${netcdf_ver} -module load nco/${nco_ver} +##module load nco/${nco_ver} module load prod_util/${prod_util_ver} -module load cdo/${cdo_ver} +##module load cdo/${cdo_ver} module load grib_util/${grib_util_ver} module load wgrib2/${wgrib2_ver} module load met/${met_ver} diff --git a/dev/modulefiles/cam/cam_stats.sh b/dev/modulefiles/cam/cam_stats.sh index a3178a8867..0286aafa1a 100644 --- a/dev/modulefiles/cam/cam_stats.sh +++ b/dev/modulefiles/cam/cam_stats.sh @@ -16,17 +16,17 @@ module load libjpeg/${libjpeg_ver} module load libpng/${libpng_ver} module load zlib/${zlib_ver} module load jasper/${jasper_ver} -module load udunits/${udunits_ver} +##module load udunits/${udunits_ver} module load gsl/${gsl_ver} -module load netcdf/${netcdf_ver} -module load nco/${nco_ver} +## module load netcdf/${netcdf_ver} +##module load nco/${nco_ver} module load prod_util/${prod_util_ver} -module load cdo/${cdo_ver} +##module load cdo/${cdo_ver} module load grib_util/${grib_util_ver} module load wgrib2/${wgrib2_ver} module load met/${met_ver} module load metplus/${metplus_ver} -module load bufr/${bufr_ver} +##module load bufr/${bufr_ver} module list set -x diff --git a/jobs/JEVS_CAM_PREP b/jobs/JEVS_CAM_PREP index d987165987..d60342f33a 100755 --- a/jobs/JEVS_CAM_PREP +++ b/jobs/JEVS_CAM_PREP @@ -122,12 +122,13 @@ export COMINrrfs=${COMINrrfs:-$(compath.py ${envir}/com/rrfs/${rrfs_ver})} if [ "${RUN}" == "chem" ]; then export DCOMINairnow=${DCOMINairnow:-${DCOMROOT}} export DCOMINaeronet=${DCOMINaeronet:-${DCOMROOT}} - export COMOUTprep=${COMOUTprep:-${COMOUT}/${RUN}.${INITDATE}/${MODELNAME}} - - mkdir -p ${COMOUTprep} + export COMOUT=${COMOUT:-$(compath.py -o ${NET}/${evs_ver})} + export COMOUTprepmdl=${COMOUTprepmdl:-${COMOUT}/${STEP}/${COMPONENT}/${RUN}.${INITDATE}/${MODELNAME}} + export COMOUTprepobs=${COMOUTprepobs:-${COMOUT}/${STEP}/${COMPONENT}/${RUN}.${INITDATE}/obs} # alternative choice is an option different from aqobs export airnow_hourly_type=${airnow_hourly_type:-aqobs} + export HOURLY_NCOL=${HOURLY_NCOL:-34} export restart_mode=${restart_mode:-YES} else @@ -155,7 +156,7 @@ fi # Execute the script #################################### if [ "${RUN}" == "chem" ]; then - ${HOMEevs}/scripts/${STEP}/${COMPONENT}/exevs_${COMPONENT}_${MODELNAME}_${RUN}_${VERIF_CASE}_${STEP}.sh + ${HOMEevs}/scripts/${STEP}/${COMPONENT}/exevs_${STEP}_${COMPONENT}_${MODELNAME}_${RUN}_${VERIF_CASE}.sh else $HOMEevs/scripts/$STEP/$COMPONENT/exevs_${MODELNAME}_${VERIF_CASE}_${STEP}.sh fi diff --git a/jobs/JEVS_CAM_STATS b/jobs/JEVS_CAM_STATS index 21d280349f..0293bdc887 100755 --- a/jobs/JEVS_CAM_STATS +++ b/jobs/JEVS_CAM_STATS @@ -158,7 +158,7 @@ if [ $VERIF_CASE = radar ] || [ $VERIF_CASE = severe ]; then elif [ $MODELNAME = nam_firewxnest ] || [ $MODELNAME = rrfs_firewxnest ]; then $HOMEevs/scripts/${STEP}/${COMPONENT}/exevs_${COMPONENT}_${MODELNAME}_${VERIF_CASE}_${STEP}.sh elif [ "${RUN}" == "chem" ]; then - ${HOMEevs}/scripts/${STEP}/${COMPONENT}/exevs_${COMPONENT}_${MODELNAME}_${RUN}_${VERIF_CASE}_${STEP}.sh + ${HOMEevs}/scripts/${STEP}/${COMPONENT}/exevs_${STEP}_${COMPONENT}_${MODELNAME}_${RUN}_${VERIF_CASE}.sh else $HOMEevs/scripts/${STEP}/${COMPONENT}/exevs_${MODELNAME}_${VERIF_CASE}_${STEP}.sh fi diff --git a/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAERONET_AOD.conf b/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAERONET_AOD.conf index 25cc3c68f4..87d1416259 100755 --- a/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAERONET_AOD.conf +++ b/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAERONET_AOD.conf @@ -77,6 +77,6 @@ POINT_STAT_MESSAGE_TYPE = [filename_templates] FCST_POINT_STAT_INPUT_TEMPLATE = {ENV[RUN]}.{init?fmt=%Y%m%d}/{ENV[MODELNAME]}/{ENV[mdl_cyc]}/{ENV[MODELNAME]}.t{ENV[mdl_cyc]}z.prslev.f{lead?fmt=%3H}.reduced.grib2 -OBS_POINT_STAT_INPUT_TEMPLATE = {ENV[RUN]}.{valid?fmt=%Y%m%d}/{ENV[MODELNAME]}/{ENV[ObsSrc]}_All_{valid?fmt=%Y%m%d}_lev15.nc +OBS_POINT_STAT_INPUT_TEMPLATE = {ENV[RUN]}.{valid?fmt=%Y%m%d}/obs/{ENV[ObsSrc]}_All_{valid?fmt=%Y%m%d}_lev15.nc POINT_STAT_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}.stat diff --git a/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM10.conf b/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM10.conf index 0a9873f3dd..3b2ce3b702 100755 --- a/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM10.conf +++ b/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM10.conf @@ -61,8 +61,8 @@ OBS_VAR1_NAME= PM10 OBS_VAR1_LEVELS= A1 OBS_VAR1_OPTIONS = message_type = "AIRNOW_HOURLY_AQOBS"; censor_thresh = [ <=0.0 ]; censor_val=[ 0.0 ]; set_attr_units = "10^-6g/m^3"; -## POINT_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [ >5, >10, >15, >20, >25, >30, >35, >40, >45, >50, >55, >60, >65 ]; duplicate_flag = UNIQUE; obs_summary = NEAREST; -POINT_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [ >5, >10, >15, >20, >25, >30, >35, >40, >45, >50, >55, >60, >65 ]; duplicate_flag = UNIQUE; +## POINT_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [ >10, >20, >30, >40, >60, >80, >100, >200 ]; duplicate_flag = UNIQUE; obs_summary = NEAREST; +POINT_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [ >10, >20, >30, >40, >60, >80, >100, >200 ]; duplicate_flag = UNIQUE; ## POINT_STAT_MET_CONFIG_OVERRIDES = duplicate_flag = UNIQUE; obs_summary = NEAREST; {ENV[met_config_overrides]} @@ -89,6 +89,6 @@ POINT_STAT_MESSAGE_TYPE = FCST_POINT_STAT_INPUT_TEMPLATE = {ENV[RUN]}.{init?fmt=%Y%m%d}/{ENV[MODELNAME]}/{ENV[mdl_cyc]}/{ENV[MODELNAME]}.t{ENV[mdl_cyc]}z.prslev.f{lead?fmt=%3H}.reduced.grib2 -OBS_POINT_STAT_INPUT_TEMPLATE = {ENV[RUN]}.{valid?fmt=%Y%m%d?shift=-3600}/{ENV[MODELNAME]}/{ENV[ObsSrc]}_{ENV[HOURLY_INPUT_TYPE]}_{valid?fmt=%Y%m%d%H?shift=-3600}.nc +OBS_POINT_STAT_INPUT_TEMPLATE = {ENV[RUN]}.{valid?fmt=%Y%m%d?shift=-3600}/obs/{ENV[ObsSrc]}_{ENV[HOURLY_INPUT_TYPE]}_{valid?fmt=%Y%m%d%H?shift=-3600}.nc POINT_STAT_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}.stat diff --git a/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM25.conf b/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM25.conf index d7b991286e..ae80c44ed3 100755 --- a/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM25.conf +++ b/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM25.conf @@ -61,8 +61,8 @@ OBS_VAR1_NAME= PM25 OBS_VAR1_LEVELS= A1 OBS_VAR1_OPTIONS = message_type = "AIRNOW_HOURLY_AQOBS"; censor_thresh = [ <=0.0 ]; censor_val=[ 0.0 ]; set_attr_units = "10^-6g/m^3"; -## POINT_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [ >5, >10, >15, >20, >25, >30, >35, >40, >45, >50, >55, >60, >65 ]; duplicate_flag = UNIQUE; obs_summary = NEAREST; -POINT_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [ >5, >10, >15, >20, >25, >30, >35, >40, >45, >50, >55, >60, >65 ]; duplicate_flag = UNIQUE; +## POINT_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [ >5, >10, >15, >20, >25, >35, >40, >50, >60, >80, >100 ]; duplicate_flag = UNIQUE; obs_summary = NEAREST; +POINT_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [ >5, >10, >15, >20, >25, >35, >40, >50, >60, >80, >100 ]; duplicate_flag = UNIQUE; ## POINT_STAT_MET_CONFIG_OVERRIDES = duplicate_flag = UNIQUE; obs_summary = NEAREST; {ENV[met_config_overrides]} @@ -89,6 +89,6 @@ POINT_STAT_MESSAGE_TYPE = FCST_POINT_STAT_INPUT_TEMPLATE = {ENV[RUN]}.{init?fmt=%Y%m%d}/{ENV[MODELNAME]}/{ENV[mdl_cyc]}/{ENV[MODELNAME]}.t{ENV[mdl_cyc]}z.prslev.f{lead?fmt=%3H}.reduced.grib2 -OBS_POINT_STAT_INPUT_TEMPLATE = {ENV[RUN]}.{valid?fmt=%Y%m%d?shift=-3600}/{ENV[MODELNAME]}/{ENV[ObsSrc]}_{ENV[HOURLY_INPUT_TYPE]}_{valid?fmt=%Y%m%d%H?shift=-3600}.nc +OBS_POINT_STAT_INPUT_TEMPLATE = {ENV[RUN]}.{valid?fmt=%Y%m%d?shift=-3600}/obs/{ENV[ObsSrc]}_{ENV[HOURLY_INPUT_TYPE]}_{valid?fmt=%Y%m%d%H?shift=-3600}.nc POINT_STAT_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}.stat diff --git a/scripts/prep/cam/exevs_cam_rrfs_chem_grid2obs_prep.sh b/scripts/prep/cam/exevs_prep_cam_rrfs_chem_grid2obs.sh similarity index 76% rename from scripts/prep/cam/exevs_cam_rrfs_chem_grid2obs_prep.sh rename to scripts/prep/cam/exevs_prep_cam_rrfs_chem_grid2obs.sh index 9e3e72925a..6db9296313 100755 --- a/scripts/prep/cam/exevs_cam_rrfs_chem_grid2obs_prep.sh +++ b/scripts/prep/cam/exevs_prep_cam_rrfs_chem_grid2obs.sh @@ -7,8 +7,8 @@ ### Original Author : Ho-Chun Huang ### ### Change Logs: -### -### 01/16/2024 Ho-Chun Huang +### 04/30/2025 Ho-Chun Huang Update warning message for dcom input obs +### and Remove sendmail for missing FCST model output ### ######################################################################## # @@ -45,7 +45,7 @@ for OBTTYPE in ${obstype}; do if [ "${OBTTYPE}" == "aeronet" ]; then flag_process_ascii_aeronet="YES" if [ "${check_restart}" == "YES" ]; then ## Check ASCII2NC AERONET AOD file for RESTART ability - checkfile=${COMOUTprep}/${OBTTYPE}_All_${INITDATE}_lev15.nc + checkfile=${COMOUTprepobs}/${OBTTYPE}_All_${INITDATE}_lev15.nc if [ -s ${checkfile} ]; then msg=$(ncdump -h ${checkfile} 1> /dev/null 2>&1 ; err=$? ; echo ${err} ) if [ ${msg} -eq 0 ]; then flag_process_ascii_aeronet="NO"; fi @@ -68,33 +68,29 @@ for OBTTYPE in ${obstype}; do if [ ${SENDCOM} = "YES" ]; then cpfile=${finalprep}/${OBTTYPE}_All_${INITDATE}_lev15.nc if [ -e ${cpfile} ]; then - cp -v ${cpfile} ${COMOUTprep} + mkdir -p ${COMOUTprepobs} + cp -v ${cpfile} ${COMOUTprepobs} fi fi - else - echo "DEBUG: can not find ${prep_config_file}" fi else if [ ${SENDMAIL} = "YES" ]; then - echo "DEBUG: There is no valid record to be processed for ${checkfile}" >> ${email_msg} - echo "File in question is ${checkfile}" >> ${email_msg} + echo "WARNING: There is no valid record to be processed, ${MODELNAME} ${RUN} ${STEP} will skip ${checkfile}" >> ${email_msg} echo "==============" >> ${email_msg} flag_send_message=YES fi - echo "DEBUG: There is no valid record to be processed for ${checkfile}" + echo "WARNING: There is no valid record to be processed, ${MODELNAME} ${RUN} ${STEP} will skip ${checkfile}" fi else if [ "${flag_process_ascii_aeronet}" == "NO" ]; then echo "DEBUG: ASCII2NC AERONET AOD files has been found. RESTART Skip ASCII2NC processing" elif [ ! -s ${checkfile} ]; then if [ ${SENDMAIL} = "YES" ]; then - echo "DEBUG: No AEORNET Level 1.5 data was available for valid date ${INITDATE}" >> ${email_msg} - echo "DEBUG: Missing file is ${checkfile}" >> ${email_msg} + echo "WARNING: ${checkfile} is missing, ${MODELNAME} ${RUN} ${STEP} will skip this file for valid date ${INITDATE}" >> ${email_msg} echo "==============" >> ${email_msg} flag_send_message=YES fi - echo "DEBUG: No AEORNET Level 1.5 data was available for valid date ${INITDATE}" - echo "DEBUG: Missing file is ${checkfile}" + echo "WARNING: ${checkfile} is missing, ${MODELNAME} ${RUN} ${STEP} will skip this file for valid date ${INITDATE}" fi fi elif [ "${OBTTYPE}" == "airnow" ]; then @@ -115,13 +111,13 @@ for OBTTYPE in ${obstype}; do if [ "${check_restart}" == "YES" ]; then ## Check ASCII2NC AIRNOW files for RESTART ability checkfile="${OBTTYPE}_${HOURLY_OUTPUT_TYPE}_*.nc" - obs_file_count=$(find ${COMOUTprep} -name ${checkfile} | wc -l ) + obs_file_count=$(find ${COMOUTprepobs} -name ${checkfile} | wc -l ) if [ ${obs_file_count} -eq 0 ]; then let ic=0 elif [ ${obs_file_count} -eq ${total_num_file} ]; then ## check corrupted ASCII2NC file vldhr=$(printf %2.2d ${endvhr}) - checkfile="${COMOUTprep}/${OBTTYPE}_${HOURLY_OUTPUT_TYPE}_${INITDATE}${vldhr}.nc" + checkfile="${COMOUTprepobs}/${OBTTYPE}_${HOURLY_OUTPUT_TYPE}_${INITDATE}${vldhr}.nc" msg=$(ncdump -h ${checkfile} 1> /dev/null 2>&1 ; err=$? ; echo ${err} ) if [ ${msg} -eq 0 ]; then let ic=${endvhr}+1 ## skip current AIRNOW Processing @@ -156,30 +152,28 @@ for OBTTYPE in ${obstype}; do export err=$?; err_chk if [ ${SENDCOM} = "YES" ]; then cpfile=${finalprep}/airnow_hourly_aqobs_${INITDATE}${VHOUR}.nc - if [ -e ${cpfile} ]; then cp -v ${cpfile} ${COMOUTprep}; fi + if [ -e ${cpfile} ]; then + mkdir -p ${COMOUTprepobs} + cp -v ${cpfile} ${COMOUTprepobs} + fi fi - else - echo "DEBUG: can not find ${prep_config_file}" fi else if [ ${SENDMAIL} = "YES" ]; then - echo "DEBUG: There is no valid record to be processed for ${checkfile}" >> ${email_msg} - echo "File in question is ${checkfile}" >> ${email_msg} + echo "WARNING: There is no valid record to be processed, ${MODELNAME} ${RUN} ${STEP} will skip the ${checkfile}" >> ${email_msg} echo "==============" >> ${email_msg} flag_send_message=YES fi - echo "DEBUG: There is no valid record to be processed for ${checkfile}" + echo "WARNING: There is no valid record to be processed, ${MODELNAME} ${RUN} ${STEP} will skip the ${checkfile}" fi else if [ ${SENDMAIL} = "YES" ]; then - echo "DEBUG: No AIRNOW ASCII data was available for valid date ${INITDATE}${vldhr}" >> ${email_msg} - echo "Missing file is ${checkfile}" >> ${email_msg} + echo "WARNING: ${checkfile} is missing, ${MODELNAME} ${RUN} ${STEP} will skip this file for valid date ${INITDATE}" >> ${email_msg} echo "==============" >> ${email_msg} flag_send_message=YES fi - echo "DEBUG: No AIRNOW ASCII data was available for valid date ${INITDATE}${vldhr}" - echo "DEBUG: Missing file is ${checkfile}" + echo "WARNING: ${checkfile} is missing, ${MODELNAME} ${RUN} ${STEP} will skip this file for valid date ${INITDATE}" fi ((ic++)) done @@ -201,20 +195,22 @@ match_aod_2="entire atmosphere" # match_pm25_1="MASSDEN" match_pm25_2="8 m above ground" -match_pm25_3="aerosol=Missing" +match_pm25_3="aerosol=Total Aerosol" match_pm25_4="aerosol_size <2.5e-06" +match_pm25_sp="aerosol=Total Aerosol" # match_pm10_1="MASSDEN" match_pm10_2="8 m above ground" -match_pm10_3="aerosol=Missing" +match_pm10_3="aerosol=Total Aerosol" match_pm10_4="aerosol_size <1e-05" +match_pm10_sp="aerosol=Total Aerosol" # declare -a cyc_opt=( 00 06 12 18 ) let inc=1 for mdl_cyc in "${cyc_opt[@]}"; do com_rrfs=${COMINrrfs}/${MODELNAME}.${INITDATE}/${mdl_cyc} if [ -d ${com_rrfs} ]; then - prep_rrfs=${COMOUTprep}/${mdl_cyc} + prep_rrfs=${COMOUTprepmdl}/${mdl_cyc} if [ ! -d ${prep_rrfs} ]; then mkdir -p ${prep_rrfs}; fi let hour_now=1 let max_hour=84 @@ -259,30 +255,34 @@ for mdl_cyc in "${cyc_opt[@]}"; do if [ -e extract_pm10 ]; then /bin/rm -rf extract_pm10; fi wgrib2 -match "${match_aod_1}" -match "${match_aod_2}" ${check_full_file} -grib extract_aod wgrib2 -match "${match_pm25_1}" -match "${match_pm25_2}" -match "${match_pm25_3}" -match "${match_pm25_4}" ${check_full_file} -grib extract_pm25 + wgrib2 extract_pm25 > extract_pm25_rec + number_of_record=$(wc -l extract_pm25_rec | awk -F" " '{print $1}') + if [ "${number_of_record}" == "0" ]; then + wgrib2 -match "${match_pm25_1}" -match "${match_pm25_2}" -match "${match_pm25_sp}" -match "${match_pm25_4}" ${check_full_file} -grib extract_pm25 + fi + wgrib2 extract_pm25 > extract_pm25_rec + number_of_record=$(wc -l extract_pm25_rec | awk -F" " '{print $1}') + echo "DEBUG: Number of extracted record is ${number_of_record} for file extract_pm25" wgrib2 -match "${match_pm10_1}" -match "${match_pm10_2}" -match "${match_pm10_3}" -match "${match_pm10_4}" ${check_full_file} -grib extract_pm10 + wgrib2 extract_pm10 > extract_pm10_rec + number_of_record=$(wc -l extract_pm10_rec | awk -F" " '{print $1}') + if [ "${number_of_record}" == "0" ]; then + wgrib2 -match "${match_pm10_1}" -match "${match_pm10_2}" -match "${match_pm10_sp}" -match "${match_pm10_4}" ${check_full_file} -grib extract_pm10 + fi + wgrib2 extract_pm10 > extract_pm10_rec + number_of_record=$(wc -l extract_pm10_rec | awk -F" " '{print $1}') + echo "DEBUG: Number of extracted record is ${number_of_record} for file extract_pm10" cat extract_pm25 extract_pm10 extract_aod > ${reduced_rec_grib2} if [ ${SENDCOM} = "YES" ]; then cp -v ${reduced_rec_grib2} ${prep_rrfs} fi else - if [ ${SENDMAIL} = "YES" ]; then - echo "DEBUG: Can not find RRFS aerosol forecast grib2 output" >> ${email_msg} - echo "Missing file is ${check_full_file}" >> ${email_msg} - echo "==============" >> ${email_msg} - flag_send_message=YES - fi - echo "DEBUG: Can not find RRFS aerosol forecast grib2 output" - echo "Missing file is ${check_full_file}" + echo "FCST_OUTPUT_MISSING: RRFS-smoke and dust forecast file ${check_full_file} is missing. The missing RRFS-smoke and dust forecast file will be skipped" fi ((hour_now+=${inc})) done else - if [ ${SENDMAIL} = "YES" ]; then - echo "DEBUG: Can not find RRFS output directory ${com_rrfs}" >> ${email_msg} - echo "==============" >> ${email_msg} - flag_send_message=YES - fi - echo "DEBUG: Can not find RRFS output directory ${com_rrfs}" + echo "FCST_OUTPUT_MISSING: RRFS-smoke and dust output directory ${com_rrfs} is missing. The missing RRFS-smoke and dust forecast files will be skipped" fi done # diff --git a/scripts/stats/cam/exevs_cam_rrfs_chem_grid2obs_stats.sh b/scripts/stats/cam/exevs_stats_cam_rrfs_chem_grid2obs.sh similarity index 70% rename from scripts/stats/cam/exevs_cam_rrfs_chem_grid2obs_stats.sh rename to scripts/stats/cam/exevs_stats_cam_rrfs_chem_grid2obs.sh index ba4cbb3886..bb8c8740cb 100755 --- a/scripts/stats/cam/exevs_cam_rrfs_chem_grid2obs_stats.sh +++ b/scripts/stats/cam/exevs_stats_cam_rrfs_chem_grid2obs.sh @@ -7,7 +7,9 @@ ### ### Change Logs: ### -### 01/01/2025 Ho-Chun Huang +### 04/30/2025 Ho-Chun Huang Remove email function for missing +### Prep-Obs input and Fcst Mdl output +### ### ######################################################################## set -x @@ -38,10 +40,6 @@ grid2obs_list="${DATA_TYPE}" export init_cyc="00 06 12 18" let inc=1 -flag_send_message=NO -email_msg=${DATA}/mailmsg -if [ -e ${email_msg} ]; then /bin/rm -f ${email_msg}; fi - check_restart=$( echo ${restart_mode} | tr a-z A-Z ) for ObsType in ${grid2obs_list}; do @@ -80,37 +78,25 @@ for ObsType in ${grid2obs_list}; do stat_analysis_conf_file=${CONFIGevs}/Statanalysis_fcstRRFSAero_obs${OBSTYPE}.conf if [ "${ObsSrc}" == "aeronet" ]; then - check_file=${EVSINrrfs}/${RUN}.${VDATE}/${MODELNAME}/${ObsSrc}_All_${VDATE}_lev15.nc + check_file=${EVSINrrfs}/${RUN}.${VDATE}/obs/${ObsSrc}_All_${VDATE}_lev15.nc num_obs_found=0 if [ -s ${check_file} ]; then num_obs_found=1 else - echo "DEBUG: Can not find pre-processed ${OBSTYPE} Level 1.5 input ${check_file}" - if [ "${SENDMAIL}" == "YES" ]; then - echo "WARNING: No pre-processed ${OBSTYPE} Level 1.5 was available for ${VDATE} " >> ${email_msg} - echo "Missing file is ${check_file}" >> ${email_msg} - echo "==============" >> ${email_msg} - flag_send_message=YES - fi + echo "PREP_OUTPUT_MISSING: Pre-processed ${ObsSrc} Level 1.5 input ${check_file} is missing. The verification on ${VDATE} will be skipped" fi - echo "index of daily aeronet obs found = ${num_obs_found}" + echo "DEBUG: index of daily aeronet obs found = ${num_obs_found}" elif [ "${ObsSrc}" == "airnow" ]; then cdate=${VDATE}${vhr} vld_date=$(${NDATE} -1 ${cdate} | cut -c1-8) vld_time=$(${NDATE} -1 ${cdate} | cut -c1-10) - check_file=${EVSINrrfs}/${RUN}.${vld_date}/${MODELNAME}/${ObsSrc}_${HOURLY_INPUT_TYPE}_${vld_time}.nc + check_file=${EVSINrrfs}/${RUN}.${vld_date}/obs/${ObsSrc}_${HOURLY_INPUT_TYPE}_${vld_time}.nc num_obs_found=0 if [ -s ${check_file} ]; then num_obs_found=1 else - echo "DEBUG: Can not find pre-processed ${OBSTYPE} hourly input ${check_file}" - if [ "${SENDMAIL}" == "YES" ]; then - echo "WARNING: No ${OBSTYPE} ${HOURLY_INPUT_TYPE} was available for ${vld_date} ${vld_time}" >> ${email_msg} - echo "Missing file is ${check_file}" >> ${email_msg} - echo "==============" >> ${email_msg} - flag_send_message=YES - fi + echo "PREP_OUTPUT_MISSING: Pre-processed ${ObsSrc} hourly input ${check_file} is missing. The verification at ${vhr}Z will be skipped" fi echo "DEBUG: index of hourly ${OBSTYPE} obs found = ${num_obs_found}" fi @@ -147,15 +133,7 @@ for ObsType in ${grid2obs_list}; do let "num_fcst_in_metplus=num_fcst_in_metplus+1" fi else - if [ "${SENDMAIL}" == "YES" ]; then - echo "WARNING: No ${CMODEL} ${ObsType} forecast was available for ${aday} t${acyc}z" >> ${email_msg} - echo "Missing file is ${fcst_file}" >> ${email_msg} - echo "==============" >> ${email_msg} - flag_send_message=YES - fi - - echo "DEBUG: No ${CMODEL} ${ObsType} forecast was available for ${aday} t${acyc}z" - echo "DEBUG: Missing file is ${fcst_file}" + echo "PREP_OUTPUT_MISSING: Pre-processed RRFS-smoke and dust output ${fcst_file} is missing. The missing RRFS-smoke and dust forecast file will be skipped" fi fi ((ihr+=${inc})) @@ -173,8 +151,12 @@ for ObsType in ${grid2obs_list}; do run_metplus.py ${point_stat_conf_file} ${config_common} export err=$?; err_chk else - echo "DEBUG: NO ${CMODEL} ${ObsType} FORECAST OR OBS TO VERIFY" - echo "DEBUG: NUM FCST=${num_fcst_in_metplus}, INDEX OBS=${num_obs_found}" + if [ ${num_obs_found} -eq 0 ]; then + echo "DEBUG: There is no pre-processed ${ObsSrc} OBS, the metplus stats process will be skipped" + fi + if [ ${num_fcst_in_metplus} -eq 0 ]; then + echo "DEBUG: There is no pre-processed ${ObsVar} ${CMODEL}-smoke and dust ${mdl_cyc} cycle forecast output validated at ${vhr}Z, the metplus stats process will be skipped" + fi fi if [ "${SENDCOM}" == "YES" ]; then if [ -d ${RUNTIME_STATS}/${VDATE}.stat ]; then ## does not exist if run_metplus.py did not execute @@ -189,28 +171,21 @@ for ObsType in ${grid2obs_list}; do fi done ## init hour loop if [ "${vhr}" == "23" ]; then -## for mdl_cyc in ${init_cyc}; do - stat_file_count=$(find ${COMOUTsmall} -name "*${OutputId}*" | wc -l) - if [ ${stat_file_count} -ne 0 ]; then - cpreq ${COMOUTsmall}/*${OutputId}* ${finalstat} - cd ${finalstat} - run_metplus.py ${stat_analysis_conf_file} ${config_common} - export err=$?; err_chk - if [ ${SENDCOM} = "YES" ]; then - cpfile=${finalstat}/${StatFileId}.v${VDATE}.stat - if [ -s ${cpfile} ]; then - mkdir -p ${COMOUTfinal} - cp -v ${cpfile} ${COMOUTfinal} - fi + stat_file_count=$(find ${COMOUTsmall} -name "*${OutputId}*" | wc -l) + if [ ${stat_file_count} -ne 0 ]; then + cp -v ${COMOUTsmall}/*${OutputId}* ${finalstat} + cd ${finalstat} + run_metplus.py ${stat_analysis_conf_file} ${config_common} + export err=$?; err_chk + if [ ${SENDCOM} = "YES" ]; then + cpfile=${finalstat}/${StatFileId}.v${VDATE}.stat + if [ -s ${cpfile} ]; then + mkdir -p ${COMOUTfinal} + cp -v ${cpfile} ${COMOUTfinal} fi fi -## done + fi fi done ## loop over ObsType -if [ "${flag_send_message}" == "YES" ]; then - export subject="${OBSTYPE} Obs or ${CMODEL} Fcst files Missing for EVS ${COMPONENT} ${RUN} ${VERIF_CASE}" - echo "Job ID: ${jobid}" >> ${email_msg} - cat ${email_msg} | mail -s "${subject}" ${MAILTO} -fi exit diff --git a/ush/cam/cam_chem_screen_airnow_obs_hourly.py b/ush/cam/cam_chem_screen_airnow_obs_hourly.py index 1b47580f82..9b3c6b3402 100755 --- a/ush/cam/cam_chem_screen_airnow_obs_hourly.py +++ b/ush/cam/cam_chem_screen_airnow_obs_hourly.py @@ -2,10 +2,14 @@ ############################################################################### # Name of Script: # Contact(s): Ho-Chun Huang (ho-chun.huang@noaa.gov) -# Purpose of Script: Read AIRNOW PM25/PM10 file and remove bad records with -# inconsistent columns number as header -# +# Purpose of Script: Read Hourly AIRNOW PM25/OZONE file and remove bad records +# with inconsistent columns number as header +# # History Log: +# +# 04/04/2025 Ho-Chun Huang Use default number of column to handle AirNOW +# hourly file even it is a radom text file +# ############################################################################### import os @@ -35,28 +39,45 @@ rfile=open(input_file, 'r') wfile=open(output_file,'w') -count=0 +evs_step=os.environ['STEP'] +evs_run=os.environ['RUN'] +evs_model=os.environ['MODELNAME'] +# +## Check for number of columns using the default 'HOURLY_NCOL' defined in ~/job +# +num_ref_hdr=int(os.environ['HOURLY_NCOL']) +rcount=0 +wcount=0 +bad_rec=0 flag_data=False for line in rfile: if not flag_data: - if line[1:6] != "AQSID": - wfile.write(line) - else: - wfile.write(line) + rcount += 1 + if line[1:6] == "AQSID": line=line.rstrip("\n") hdr=line.split('","') num_hdr=len(hdr) - print(f"header len {num_hdr}") - flag_data=True - count += 1 + if num_hdr == num_ref_hdr: + wfile.write(line+"\n") + wcount += 1 + flag_data=True + print(f"DEBUG :: find header row in line {rcount} with header column = {num_ref_hdr}") + else: + print(f"DEBUG :: Skip line {rcount} with wrong number of columns") else: - count += 1 + rcount += 1 line=line.rstrip("\n") var=[] var=line.split('","') num_var=len(var) - if num_var == num_hdr: + if num_var == num_ref_hdr: wfile.write(line+"\n") + wcount += 1 else: - print(f"DEBUG :: Line {count} has different columns number {num_var} vs standard {num_hdr}") + bad_rec += 1 +if wcount == 0: + print(f"WARNING: {input_file} is corrupt (wrong number of columns). {evs_model} {evs_run} {evs_step} step will skip the corrupted validation file.") +else: + if bad_rec > 0: + print(f"WARNING: {input_file} is corrupt, {bad_rec} line(s) with wrong number of columns. Removing the corrupted line(s) from file and continuing") wfile.close() diff --git a/versions/run.ver b/versions/run.ver index dba09a90d1..de6d34801c 100644 --- a/versions/run.ver +++ b/versions/run.ver @@ -1,33 +1,33 @@ export evs_ver=v2.0.0 export bacio_ver=2.4.1 -export bufr_ver=12.0.1 +export bufr_ver=12.2.0 export cdo_ver=1.9.8 export cfp_ver=2.0.4 export craympich_ver=8.1.19 -export craypals_ver=1.2.2 +export craypals_ver=1.3.2 export craype_ver=2.7.17 export g2_ver=3.4.5 -export g2c_ver=1.7.0 +export g2c_ver=2.2.0 export geos_ver=3.8.1 export grib_util_ver=1.2.4 export gsl_ver=2.7 export hdf5_ver=1.12.2 export imagemagick_ver=7.0.8-7 export intel_ver=19.1.3.304 -export ip_ver=4.0.0 +export ip_ver=5.2.0 export jasper_ver=2.0.25 -export libfabric_ver=1.11.0.0. +export libfabric_ver=1.20.1 export libjpeg_ver=9c export libpng_ver=1.6.37 -export met_ver=12.0.0-rc1 -export metcalcpy_ver=3.0.0-rc1 -export metdataio_ver=3.0.0-rc1 -export metplotpy_ver=3.0.0-rc1 -export metplus_ver=6.0.0-rc1 -export nco_ver=5.0.6 +export met_ver=12.0.1 +export metcalcpy_ver=3.0.0 +export metdataio_ver=3.0.0 +export metplotpy_ver=3.0.0 +export metplus_ver=6.0.0 +export nco_ver=5.2.4 export netcdf_ver=4.7.4 -export PrgEnvintel_ver=8.3.3 +export PrgEnvintel_ver=8.5.0 export prod_envir_ver=2.0.6 export prod_util_ver=2.0.14 export proj_ver=7.1.0 @@ -62,9 +62,10 @@ export obsproc_ver=v1.2 export rap_ver=v5.1 export rrfs_ver=na export rtma_ver=v2.10 -export rtofs_ver=v2.4 +export rtofs_ver=v2.5 export sref_ver=v7.1 export st4_ver=v4.1 +export ukmet_ver=v2.2 export urma_ver=v2.10 export wafs_ver=v7.0