From ff8d00f41f0063a69cc014515f4d50410e990308 Mon Sep 17 00:00:00 2001 From: Ho-Chun Huang Date: Thu, 8 May 2025 18:25:35 +0000 Subject: [PATCH 01/11] for airnow bugfix and senemail msg clean-ip --- .../cam/exevs_cam_rrfs_chem_grid2obs_prep.sh | 46 +++--------- .../cam/exevs_cam_rrfs_chem_grid2obs_stats.sh | 75 +++++++------------ ush/cam/cam_chem_screen_airnow_obs_hourly.py | 49 ++++++++---- 3 files changed, 72 insertions(+), 98 deletions(-) diff --git a/scripts/prep/cam/exevs_cam_rrfs_chem_grid2obs_prep.sh b/scripts/prep/cam/exevs_cam_rrfs_chem_grid2obs_prep.sh index 9e3e72925a..6b2991f04f 100755 --- a/scripts/prep/cam/exevs_cam_rrfs_chem_grid2obs_prep.sh +++ b/scripts/prep/cam/exevs_cam_rrfs_chem_grid2obs_prep.sh @@ -7,8 +7,8 @@ ### Original Author : Ho-Chun Huang ### ### Change Logs: -### -### 01/16/2024 Ho-Chun Huang +### 04/30/2025 Ho-Chun Huang Update warning message for dcom input obs +### and Remove sendmail for missing FCST model output ### ######################################################################## # @@ -71,30 +71,25 @@ for OBTTYPE in ${obstype}; do cp -v ${cpfile} ${COMOUTprep} fi fi - else - echo "DEBUG: can not find ${prep_config_file}" fi else if [ ${SENDMAIL} = "YES" ]; then - echo "DEBUG: There is no valid record to be processed for ${checkfile}" >> ${email_msg} - echo "File in question is ${checkfile}" >> ${email_msg} + echo "WARNING: There is no valid record to be processed, ${MODELNAME} ${RUN} ${STEP} will skip ${checkfile}" >> ${email_msg} echo "==============" >> ${email_msg} flag_send_message=YES fi - echo "DEBUG: There is no valid record to be processed for ${checkfile}" + echo "WARNING: There is no valid record to be processed, ${MODELNAME} ${RUN} ${STEP} will skip ${checkfile}" fi else if [ "${flag_process_ascii_aeronet}" == "NO" ]; then echo "DEBUG: ASCII2NC AERONET AOD files has been found. RESTART Skip ASCII2NC processing" elif [ ! -s ${checkfile} ]; then if [ ${SENDMAIL} = "YES" ]; then - echo "DEBUG: No AEORNET Level 1.5 data was available for valid date ${INITDATE}" >> ${email_msg} - echo "DEBUG: Missing file is ${checkfile}" >> ${email_msg} + echo "WARNING: ${checkfile} is missing, ${MODELNAME} ${RUN} ${STEP} will skip this file for valid date ${INITDATE}" >> ${email_msg} echo "==============" >> ${email_msg} flag_send_message=YES fi - echo "DEBUG: No AEORNET Level 1.5 data was available for valid date ${INITDATE}" - echo "DEBUG: Missing file is ${checkfile}" + echo "WARNING: ${checkfile} is missing, ${MODELNAME} ${RUN} ${STEP} will skip this file for valid date ${INITDATE}" fi fi elif [ "${OBTTYPE}" == "airnow" ]; then @@ -158,28 +153,23 @@ for OBTTYPE in ${obstype}; do cpfile=${finalprep}/airnow_hourly_aqobs_${INITDATE}${VHOUR}.nc if [ -e ${cpfile} ]; then cp -v ${cpfile} ${COMOUTprep}; fi fi - else - echo "DEBUG: can not find ${prep_config_file}" fi else if [ ${SENDMAIL} = "YES" ]; then - echo "DEBUG: There is no valid record to be processed for ${checkfile}" >> ${email_msg} - echo "File in question is ${checkfile}" >> ${email_msg} + echo "WARNING: There is no valid record to be processed, ${MODELNAME} ${RUN} ${STEP} will skip the ${checkfile}" >> ${email_msg} echo "==============" >> ${email_msg} flag_send_message=YES fi - echo "DEBUG: There is no valid record to be processed for ${checkfile}" + echo "WARNING: There is no valid record to be processed, ${MODELNAME} ${RUN} ${STEP} will skip the ${checkfile}" fi else if [ ${SENDMAIL} = "YES" ]; then - echo "DEBUG: No AIRNOW ASCII data was available for valid date ${INITDATE}${vldhr}" >> ${email_msg} - echo "Missing file is ${checkfile}" >> ${email_msg} + echo "WARNING: ${checkfile} is missing, ${MODELNAME} ${RUN} ${STEP} will skip this file for valid date ${INITDATE}" >> ${email_msg} echo "==============" >> ${email_msg} flag_send_message=YES fi - echo "DEBUG: No AIRNOW ASCII data was available for valid date ${INITDATE}${vldhr}" - echo "DEBUG: Missing file is ${checkfile}" + echo "WARNING: ${checkfile} is missing, ${MODELNAME} ${RUN} ${STEP} will skip this file for valid date ${INITDATE}" fi ((ic++)) done @@ -265,24 +255,12 @@ for mdl_cyc in "${cyc_opt[@]}"; do cp -v ${reduced_rec_grib2} ${prep_rrfs} fi else - if [ ${SENDMAIL} = "YES" ]; then - echo "DEBUG: Can not find RRFS aerosol forecast grib2 output" >> ${email_msg} - echo "Missing file is ${check_full_file}" >> ${email_msg} - echo "==============" >> ${email_msg} - flag_send_message=YES - fi - echo "DEBUG: Can not find RRFS aerosol forecast grib2 output" - echo "Missing file is ${check_full_file}" + echo "FCST_OUTPUT_MISSING: RRFS-smoke and dust forecast file ${check_full_file} is missing. The missing RRFS-smoke and dust forecast file will be skipped" fi ((hour_now+=${inc})) done else - if [ ${SENDMAIL} = "YES" ]; then - echo "DEBUG: Can not find RRFS output directory ${com_rrfs}" >> ${email_msg} - echo "==============" >> ${email_msg} - flag_send_message=YES - fi - echo "DEBUG: Can not find RRFS output directory ${com_rrfs}" + echo "FCST_OUTPUT_MISSING: RRFS-smoke and dust output directory ${com_rrfs} is missing. The missing RRFS-smoke and dust forecast files will be skipped" fi done # diff --git a/scripts/stats/cam/exevs_cam_rrfs_chem_grid2obs_stats.sh b/scripts/stats/cam/exevs_cam_rrfs_chem_grid2obs_stats.sh index ba4cbb3886..3c47cbc446 100755 --- a/scripts/stats/cam/exevs_cam_rrfs_chem_grid2obs_stats.sh +++ b/scripts/stats/cam/exevs_cam_rrfs_chem_grid2obs_stats.sh @@ -7,7 +7,9 @@ ### ### Change Logs: ### -### 01/01/2025 Ho-Chun Huang +### 04/30/2025 Ho-Chun Huang Remove email function for missing +### Prep-Obs input and Fcst Mdl output +### ### ######################################################################## set -x @@ -38,10 +40,6 @@ grid2obs_list="${DATA_TYPE}" export init_cyc="00 06 12 18" let inc=1 -flag_send_message=NO -email_msg=${DATA}/mailmsg -if [ -e ${email_msg} ]; then /bin/rm -f ${email_msg}; fi - check_restart=$( echo ${restart_mode} | tr a-z A-Z ) for ObsType in ${grid2obs_list}; do @@ -85,15 +83,9 @@ for ObsType in ${grid2obs_list}; do if [ -s ${check_file} ]; then num_obs_found=1 else - echo "DEBUG: Can not find pre-processed ${OBSTYPE} Level 1.5 input ${check_file}" - if [ "${SENDMAIL}" == "YES" ]; then - echo "WARNING: No pre-processed ${OBSTYPE} Level 1.5 was available for ${VDATE} " >> ${email_msg} - echo "Missing file is ${check_file}" >> ${email_msg} - echo "==============" >> ${email_msg} - flag_send_message=YES - fi + echo "PREP_OUTPUT_MISSING: Pre-processed ${ObsSrc} Level 1.5 input ${check_file} is missing. The verification on ${VDATE} will be skipped" fi - echo "index of daily aeronet obs found = ${num_obs_found}" + echo "DEBUG: index of daily aeronet obs found = ${num_obs_found}" elif [ "${ObsSrc}" == "airnow" ]; then cdate=${VDATE}${vhr} vld_date=$(${NDATE} -1 ${cdate} | cut -c1-8) @@ -104,13 +96,7 @@ for ObsType in ${grid2obs_list}; do if [ -s ${check_file} ]; then num_obs_found=1 else - echo "DEBUG: Can not find pre-processed ${OBSTYPE} hourly input ${check_file}" - if [ "${SENDMAIL}" == "YES" ]; then - echo "WARNING: No ${OBSTYPE} ${HOURLY_INPUT_TYPE} was available for ${vld_date} ${vld_time}" >> ${email_msg} - echo "Missing file is ${check_file}" >> ${email_msg} - echo "==============" >> ${email_msg} - flag_send_message=YES - fi + echo "PREP_OUTPUT_MISSING: Pre-processed ${ObsSrc} hourly input ${check_file} is missing. The verification at ${vhr}Z will be skipped" fi echo "DEBUG: index of hourly ${OBSTYPE} obs found = ${num_obs_found}" fi @@ -147,15 +133,7 @@ for ObsType in ${grid2obs_list}; do let "num_fcst_in_metplus=num_fcst_in_metplus+1" fi else - if [ "${SENDMAIL}" == "YES" ]; then - echo "WARNING: No ${CMODEL} ${ObsType} forecast was available for ${aday} t${acyc}z" >> ${email_msg} - echo "Missing file is ${fcst_file}" >> ${email_msg} - echo "==============" >> ${email_msg} - flag_send_message=YES - fi - - echo "DEBUG: No ${CMODEL} ${ObsType} forecast was available for ${aday} t${acyc}z" - echo "DEBUG: Missing file is ${fcst_file}" + echo "PREP_OUTPUT_MISSING: Pre-processed RRFS-smoke and dust output ${fcst_file} is missing. The missing GEFS-aerosol forecast file will be skipped" fi fi ((ihr+=${inc})) @@ -173,8 +151,12 @@ for ObsType in ${grid2obs_list}; do run_metplus.py ${point_stat_conf_file} ${config_common} export err=$?; err_chk else - echo "DEBUG: NO ${CMODEL} ${ObsType} FORECAST OR OBS TO VERIFY" - echo "DEBUG: NUM FCST=${num_fcst_in_metplus}, INDEX OBS=${num_obs_found}" + if [ ${num_obs_found} -eq 0 ]; then + echo "DEBUG: There is no pre-processed ${ObsSrc} OBS, the metplus stats process will be skipped" + fi + if [ ${num_fcst_in_metplus} -eq 0 ]; then + echo "DEBUG: There is no pre-processed ${ObsVar} ${CMODEL}-smoke and dust ${mdl_cyc} cycle forecast output validated at ${vhr}Z, the metplus stats process will be skipped" + fi fi if [ "${SENDCOM}" == "YES" ]; then if [ -d ${RUNTIME_STATS}/${VDATE}.stat ]; then ## does not exist if run_metplus.py did not execute @@ -189,28 +171,21 @@ for ObsType in ${grid2obs_list}; do fi done ## init hour loop if [ "${vhr}" == "23" ]; then -## for mdl_cyc in ${init_cyc}; do - stat_file_count=$(find ${COMOUTsmall} -name "*${OutputId}*" | wc -l) - if [ ${stat_file_count} -ne 0 ]; then - cpreq ${COMOUTsmall}/*${OutputId}* ${finalstat} - cd ${finalstat} - run_metplus.py ${stat_analysis_conf_file} ${config_common} - export err=$?; err_chk - if [ ${SENDCOM} = "YES" ]; then - cpfile=${finalstat}/${StatFileId}.v${VDATE}.stat - if [ -s ${cpfile} ]; then - mkdir -p ${COMOUTfinal} - cp -v ${cpfile} ${COMOUTfinal} - fi + stat_file_count=$(find ${COMOUTsmall} -name "*${OutputId}*" | wc -l) + if [ ${stat_file_count} -ne 0 ]; then + cpreq ${COMOUTsmall}/*${OutputId}* ${finalstat} + cd ${finalstat} + run_metplus.py ${stat_analysis_conf_file} ${config_common} + export err=$?; err_chk + if [ ${SENDCOM} = "YES" ]; then + cpfile=${finalstat}/${StatFileId}.v${VDATE}.stat + if [ -s ${cpfile} ]; then + mkdir -p ${COMOUTfinal} + cp -v ${cpfile} ${COMOUTfinal} fi fi -## done + fi fi done ## loop over ObsType -if [ "${flag_send_message}" == "YES" ]; then - export subject="${OBSTYPE} Obs or ${CMODEL} Fcst files Missing for EVS ${COMPONENT} ${RUN} ${VERIF_CASE}" - echo "Job ID: ${jobid}" >> ${email_msg} - cat ${email_msg} | mail -s "${subject}" ${MAILTO} -fi exit diff --git a/ush/cam/cam_chem_screen_airnow_obs_hourly.py b/ush/cam/cam_chem_screen_airnow_obs_hourly.py index 1b47580f82..9b3c6b3402 100755 --- a/ush/cam/cam_chem_screen_airnow_obs_hourly.py +++ b/ush/cam/cam_chem_screen_airnow_obs_hourly.py @@ -2,10 +2,14 @@ ############################################################################### # Name of Script: # Contact(s): Ho-Chun Huang (ho-chun.huang@noaa.gov) -# Purpose of Script: Read AIRNOW PM25/PM10 file and remove bad records with -# inconsistent columns number as header -# +# Purpose of Script: Read Hourly AIRNOW PM25/OZONE file and remove bad records +# with inconsistent columns number as header +# # History Log: +# +# 04/04/2025 Ho-Chun Huang Use default number of column to handle AirNOW +# hourly file even it is a radom text file +# ############################################################################### import os @@ -35,28 +39,45 @@ rfile=open(input_file, 'r') wfile=open(output_file,'w') -count=0 +evs_step=os.environ['STEP'] +evs_run=os.environ['RUN'] +evs_model=os.environ['MODELNAME'] +# +## Check for number of columns using the default 'HOURLY_NCOL' defined in ~/job +# +num_ref_hdr=int(os.environ['HOURLY_NCOL']) +rcount=0 +wcount=0 +bad_rec=0 flag_data=False for line in rfile: if not flag_data: - if line[1:6] != "AQSID": - wfile.write(line) - else: - wfile.write(line) + rcount += 1 + if line[1:6] == "AQSID": line=line.rstrip("\n") hdr=line.split('","') num_hdr=len(hdr) - print(f"header len {num_hdr}") - flag_data=True - count += 1 + if num_hdr == num_ref_hdr: + wfile.write(line+"\n") + wcount += 1 + flag_data=True + print(f"DEBUG :: find header row in line {rcount} with header column = {num_ref_hdr}") + else: + print(f"DEBUG :: Skip line {rcount} with wrong number of columns") else: - count += 1 + rcount += 1 line=line.rstrip("\n") var=[] var=line.split('","') num_var=len(var) - if num_var == num_hdr: + if num_var == num_ref_hdr: wfile.write(line+"\n") + wcount += 1 else: - print(f"DEBUG :: Line {count} has different columns number {num_var} vs standard {num_hdr}") + bad_rec += 1 +if wcount == 0: + print(f"WARNING: {input_file} is corrupt (wrong number of columns). {evs_model} {evs_run} {evs_step} step will skip the corrupted validation file.") +else: + if bad_rec > 0: + print(f"WARNING: {input_file} is corrupt, {bad_rec} line(s) with wrong number of columns. Removing the corrupted line(s) from file and continuing") wfile.close() From 171eb9f1cb25dcf2516efc29a21dd2b546ba78bf Mon Sep 17 00:00:00 2001 From: Ho-Chun Huang Date: Fri, 9 May 2025 18:16:17 +0000 Subject: [PATCH 02/11] add HOURLY_NCOL --- jobs/JEVS_CAM_PREP | 1 + 1 file changed, 1 insertion(+) diff --git a/jobs/JEVS_CAM_PREP b/jobs/JEVS_CAM_PREP index d987165987..c563c54e7c 100755 --- a/jobs/JEVS_CAM_PREP +++ b/jobs/JEVS_CAM_PREP @@ -128,6 +128,7 @@ if [ "${RUN}" == "chem" ]; then # alternative choice is an option different from aqobs export airnow_hourly_type=${airnow_hourly_type:-aqobs} + export HOURLY_NCOL=${HOURLY_NCOL:-34} export restart_mode=${restart_mode:-YES} else From 7c7146bc194b57eda8260faebaa8add29a90e86d Mon Sep 17 00:00:00 2001 From: Ho-Chun Huang Date: Mon, 12 May 2025 10:26:43 +0000 Subject: [PATCH 03/11] incremental changes 05/12/2025 1 --- scripts/stats/cam/exevs_cam_rrfs_chem_grid2obs_stats.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/stats/cam/exevs_cam_rrfs_chem_grid2obs_stats.sh b/scripts/stats/cam/exevs_cam_rrfs_chem_grid2obs_stats.sh index 3c47cbc446..a0bc329219 100755 --- a/scripts/stats/cam/exevs_cam_rrfs_chem_grid2obs_stats.sh +++ b/scripts/stats/cam/exevs_cam_rrfs_chem_grid2obs_stats.sh @@ -133,7 +133,7 @@ for ObsType in ${grid2obs_list}; do let "num_fcst_in_metplus=num_fcst_in_metplus+1" fi else - echo "PREP_OUTPUT_MISSING: Pre-processed RRFS-smoke and dust output ${fcst_file} is missing. The missing GEFS-aerosol forecast file will be skipped" + echo "PREP_OUTPUT_MISSING: Pre-processed RRFS-smoke and dust output ${fcst_file} is missing. The missing RRFS-smoke and dust forecast file will be skipped" fi fi ((ihr+=${inc})) From 812781bdc3ef850cccb9b60e6e86df521a59369d Mon Sep 17 00:00:00 2001 From: Ho-Chun Huang Date: Fri, 13 Jun 2025 20:23:41 +0000 Subject: [PATCH 04/11] incremental changes 06/13/2025 1 --- dev/modulefiles/cam/cam_plots.sh | 12 +++--- dev/modulefiles/cam/cam_plots.sh_org | 38 +++++++++++++++++++ dev/modulefiles/cam/cam_prep.sh | 6 +-- dev/modulefiles/cam/cam_stats.sh | 10 ++--- dev/modulefiles/cam/cam_stats.sh_org | 32 ++++++++++++++++ .../cam/exevs_cam_rrfs_chem_grid2obs_prep.sh | 18 +++++++++ 6 files changed, 102 insertions(+), 14 deletions(-) create mode 100644 dev/modulefiles/cam/cam_plots.sh_org create mode 100644 dev/modulefiles/cam/cam_stats.sh_org diff --git a/dev/modulefiles/cam/cam_plots.sh b/dev/modulefiles/cam/cam_plots.sh index 42d7ba0f8e..91bffcae00 100644 --- a/dev/modulefiles/cam/cam_plots.sh +++ b/dev/modulefiles/cam/cam_plots.sh @@ -16,16 +16,16 @@ module load libjpeg/${libjpeg_ver} module load libpng/${libpng_ver} module load zlib/${zlib_ver} module load jasper/${jasper_ver} -module load udunits/${udunits_ver} +##module load udunits/${udunits_ver} module load gsl/${gsl_ver} -module load netcdf/${netcdf_ver} -module load nco/${nco_ver} +## module load netcdf/${netcdf_ver} +##module load nco/${nco_ver} module load prod_util/${prod_util_ver} -module load cdo/${cdo_ver} +##module load cdo/${cdo_ver} module load grib_util/${grib_util_ver} module load wgrib2/${wgrib2_ver} -module load proj/${proj_ver} -module load geos/${geos_ver} +##module load proj/${proj_ver} +##module load geos/${geos_ver} module load imagemagick/${imagemagick_ver} module load met/${met_ver} module load metplus/${metplus_ver} diff --git a/dev/modulefiles/cam/cam_plots.sh_org b/dev/modulefiles/cam/cam_plots.sh_org new file mode 100644 index 0000000000..42d7ba0f8e --- /dev/null +++ b/dev/modulefiles/cam/cam_plots.sh_org @@ -0,0 +1,38 @@ +#!/bin/bash +# modulefile for EVS cam component, plots step + +set -x + +module use /apps/ops/para/libs/modulefiles/compiler/intel/${intel_ver} +export HPC_OPT=/apps/ops/para/libs +module use /apps/dev/modulefiles +module load PrgEnv-intel/${PrgEnvintel_ver} +module load intel/${intel_ver} +module load ve/evs/${ve_evs_ver} +module load cray-mpich/${craympich_ver} +module load cray-pals/${craypals_ver} +module load cfp/${cfp_ver} +module load libjpeg/${libjpeg_ver} +module load libpng/${libpng_ver} +module load zlib/${zlib_ver} +module load jasper/${jasper_ver} +module load udunits/${udunits_ver} +module load gsl/${gsl_ver} +module load netcdf/${netcdf_ver} +module load nco/${nco_ver} +module load prod_util/${prod_util_ver} +module load cdo/${cdo_ver} +module load grib_util/${grib_util_ver} +module load wgrib2/${wgrib2_ver} +module load proj/${proj_ver} +module load geos/${geos_ver} +module load imagemagick/${imagemagick_ver} +module load met/${met_ver} +module load metplus/${metplus_ver} + +export SIPHONROOT=${UTILROOT}/fakedbn +export DBNROOT=$SIPHONROOT + +module list +set -x + diff --git a/dev/modulefiles/cam/cam_prep.sh b/dev/modulefiles/cam/cam_prep.sh index 8673616c93..385d33a9c0 100644 --- a/dev/modulefiles/cam/cam_prep.sh +++ b/dev/modulefiles/cam/cam_prep.sh @@ -16,12 +16,12 @@ module load libjpeg/${libjpeg_ver} module load libpng/${libpng_ver} module load zlib/${zlib_ver} module load jasper/${jasper_ver} -module load udunits/${udunits_ver} +##module load udunits/${udunits_ver} module load gsl/${gsl_ver} module load netcdf/${netcdf_ver} -module load nco/${nco_ver} +##module load nco/${nco_ver} module load prod_util/${prod_util_ver} -module load cdo/${cdo_ver} +##module load cdo/${cdo_ver} module load grib_util/${grib_util_ver} module load wgrib2/${wgrib2_ver} module load met/${met_ver} diff --git a/dev/modulefiles/cam/cam_stats.sh b/dev/modulefiles/cam/cam_stats.sh index a3178a8867..0286aafa1a 100644 --- a/dev/modulefiles/cam/cam_stats.sh +++ b/dev/modulefiles/cam/cam_stats.sh @@ -16,17 +16,17 @@ module load libjpeg/${libjpeg_ver} module load libpng/${libpng_ver} module load zlib/${zlib_ver} module load jasper/${jasper_ver} -module load udunits/${udunits_ver} +##module load udunits/${udunits_ver} module load gsl/${gsl_ver} -module load netcdf/${netcdf_ver} -module load nco/${nco_ver} +## module load netcdf/${netcdf_ver} +##module load nco/${nco_ver} module load prod_util/${prod_util_ver} -module load cdo/${cdo_ver} +##module load cdo/${cdo_ver} module load grib_util/${grib_util_ver} module load wgrib2/${wgrib2_ver} module load met/${met_ver} module load metplus/${metplus_ver} -module load bufr/${bufr_ver} +##module load bufr/${bufr_ver} module list set -x diff --git a/dev/modulefiles/cam/cam_stats.sh_org b/dev/modulefiles/cam/cam_stats.sh_org new file mode 100644 index 0000000000..a3178a8867 --- /dev/null +++ b/dev/modulefiles/cam/cam_stats.sh_org @@ -0,0 +1,32 @@ +#!/bin/bash +# modulefile for EVS cam component, stats step + +set -x + +module use /apps/ops/para/libs/modulefiles/compiler/intel/${intel_ver} +export HPC_OPT=/apps/ops/para/libs +module use /apps/dev/modulefiles +module load PrgEnv-intel/${PrgEnvintel_ver} +module load intel/${intel_ver} +module load ve/evs/${ve_evs_ver} +module load cray-mpich/${craympich_ver} +module load cray-pals/${craypals_ver} +module load cfp/${cfp_ver} +module load libjpeg/${libjpeg_ver} +module load libpng/${libpng_ver} +module load zlib/${zlib_ver} +module load jasper/${jasper_ver} +module load udunits/${udunits_ver} +module load gsl/${gsl_ver} +module load netcdf/${netcdf_ver} +module load nco/${nco_ver} +module load prod_util/${prod_util_ver} +module load cdo/${cdo_ver} +module load grib_util/${grib_util_ver} +module load wgrib2/${wgrib2_ver} +module load met/${met_ver} +module load metplus/${metplus_ver} +module load bufr/${bufr_ver} + +module list +set -x diff --git a/scripts/prep/cam/exevs_cam_rrfs_chem_grid2obs_prep.sh b/scripts/prep/cam/exevs_cam_rrfs_chem_grid2obs_prep.sh index 6b2991f04f..4f253b2de7 100755 --- a/scripts/prep/cam/exevs_cam_rrfs_chem_grid2obs_prep.sh +++ b/scripts/prep/cam/exevs_cam_rrfs_chem_grid2obs_prep.sh @@ -193,11 +193,13 @@ match_pm25_1="MASSDEN" match_pm25_2="8 m above ground" match_pm25_3="aerosol=Missing" match_pm25_4="aerosol_size <2.5e-06" +match_pm25_sp="aerosol=Total Aerosol" # match_pm10_1="MASSDEN" match_pm10_2="8 m above ground" match_pm10_3="aerosol=Missing" match_pm10_4="aerosol_size <1e-05" +match_pm10_sp="aerosol=Total Aerosol" # declare -a cyc_opt=( 00 06 12 18 ) let inc=1 @@ -249,7 +251,23 @@ for mdl_cyc in "${cyc_opt[@]}"; do if [ -e extract_pm10 ]; then /bin/rm -rf extract_pm10; fi wgrib2 -match "${match_aod_1}" -match "${match_aod_2}" ${check_full_file} -grib extract_aod wgrib2 -match "${match_pm25_1}" -match "${match_pm25_2}" -match "${match_pm25_3}" -match "${match_pm25_4}" ${check_full_file} -grib extract_pm25 + wgrib2 extract_pm25 > extract_pm25_rec + number_of_record=$(wc -l extract_pm25_rec | awk -F" " '{print $1}') + if [ "${number_of_record}" == "0" ]; then + wgrib2 -match "${match_pm25_1}" -match "${match_pm25_2}" -match "${match_pm25_sp}" -match "${match_pm25_4}" ${check_full_file} -grib extract_pm25 + fi + wgrib2 extract_pm25 > extract_pm25_rec + number_of_record=$(wc -l extract_pm25_rec | awk -F" " '{print $1}') + echo "DEBUG: Number of extracted record is ${number_of_record} for file extract_pm25" wgrib2 -match "${match_pm10_1}" -match "${match_pm10_2}" -match "${match_pm10_3}" -match "${match_pm10_4}" ${check_full_file} -grib extract_pm10 + wgrib2 extract_pm10 > extract_pm10_rec + number_of_record=$(wc -l extract_pm10_rec | awk -F" " '{print $1}') + if [ "${number_of_record}" == "0" ]; then + wgrib2 -match "${match_pm10_1}" -match "${match_pm10_2}" -match "${match_pm10_sp}" -match "${match_pm10_4}" ${check_full_file} -grib extract_pm10 + fi + wgrib2 extract_pm10 > extract_pm10_rec + number_of_record=$(wc -l extract_pm10_rec | awk -F" " '{print $1}') + echo "DEBUG: Number of extracted record is ${number_of_record} for file extract_pm10" cat extract_pm25 extract_pm10 extract_aod > ${reduced_rec_grib2} if [ ${SENDCOM} = "YES" ]; then cp -v ${reduced_rec_grib2} ${prep_rrfs} From 371e9699b52ffaf54d1b1af8f2b6dd0a0cac9795 Mon Sep 17 00:00:00 2001 From: Ho-Chun Huang Date: Fri, 13 Jun 2025 20:25:40 +0000 Subject: [PATCH 05/11] incremental changes 06/13/2025 2 --- dev/modulefiles/cam/cam_plots.sh_org | 38 ---------------------------- dev/modulefiles/cam/cam_stats.sh_org | 32 ----------------------- 2 files changed, 70 deletions(-) delete mode 100644 dev/modulefiles/cam/cam_plots.sh_org delete mode 100644 dev/modulefiles/cam/cam_stats.sh_org diff --git a/dev/modulefiles/cam/cam_plots.sh_org b/dev/modulefiles/cam/cam_plots.sh_org deleted file mode 100644 index 42d7ba0f8e..0000000000 --- a/dev/modulefiles/cam/cam_plots.sh_org +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# modulefile for EVS cam component, plots step - -set -x - -module use /apps/ops/para/libs/modulefiles/compiler/intel/${intel_ver} -export HPC_OPT=/apps/ops/para/libs -module use /apps/dev/modulefiles -module load PrgEnv-intel/${PrgEnvintel_ver} -module load intel/${intel_ver} -module load ve/evs/${ve_evs_ver} -module load cray-mpich/${craympich_ver} -module load cray-pals/${craypals_ver} -module load cfp/${cfp_ver} -module load libjpeg/${libjpeg_ver} -module load libpng/${libpng_ver} -module load zlib/${zlib_ver} -module load jasper/${jasper_ver} -module load udunits/${udunits_ver} -module load gsl/${gsl_ver} -module load netcdf/${netcdf_ver} -module load nco/${nco_ver} -module load prod_util/${prod_util_ver} -module load cdo/${cdo_ver} -module load grib_util/${grib_util_ver} -module load wgrib2/${wgrib2_ver} -module load proj/${proj_ver} -module load geos/${geos_ver} -module load imagemagick/${imagemagick_ver} -module load met/${met_ver} -module load metplus/${metplus_ver} - -export SIPHONROOT=${UTILROOT}/fakedbn -export DBNROOT=$SIPHONROOT - -module list -set -x - diff --git a/dev/modulefiles/cam/cam_stats.sh_org b/dev/modulefiles/cam/cam_stats.sh_org deleted file mode 100644 index a3178a8867..0000000000 --- a/dev/modulefiles/cam/cam_stats.sh_org +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# modulefile for EVS cam component, stats step - -set -x - -module use /apps/ops/para/libs/modulefiles/compiler/intel/${intel_ver} -export HPC_OPT=/apps/ops/para/libs -module use /apps/dev/modulefiles -module load PrgEnv-intel/${PrgEnvintel_ver} -module load intel/${intel_ver} -module load ve/evs/${ve_evs_ver} -module load cray-mpich/${craympich_ver} -module load cray-pals/${craypals_ver} -module load cfp/${cfp_ver} -module load libjpeg/${libjpeg_ver} -module load libpng/${libpng_ver} -module load zlib/${zlib_ver} -module load jasper/${jasper_ver} -module load udunits/${udunits_ver} -module load gsl/${gsl_ver} -module load netcdf/${netcdf_ver} -module load nco/${nco_ver} -module load prod_util/${prod_util_ver} -module load cdo/${cdo_ver} -module load grib_util/${grib_util_ver} -module load wgrib2/${wgrib2_ver} -module load met/${met_ver} -module load metplus/${metplus_ver} -module load bufr/${bufr_ver} - -module list -set -x From d5c8e32d6e943f83075a361c10e381817f1bfb05 Mon Sep 17 00:00:00 2001 From: Ho-Chun Huang Date: Mon, 23 Jun 2025 16:38:33 +0000 Subject: [PATCH 06/11] update for new wgrib2 encoding --- scripts/prep/cam/exevs_cam_rrfs_chem_grid2obs_prep.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/prep/cam/exevs_cam_rrfs_chem_grid2obs_prep.sh b/scripts/prep/cam/exevs_cam_rrfs_chem_grid2obs_prep.sh index 4f253b2de7..de3c0411aa 100755 --- a/scripts/prep/cam/exevs_cam_rrfs_chem_grid2obs_prep.sh +++ b/scripts/prep/cam/exevs_cam_rrfs_chem_grid2obs_prep.sh @@ -191,13 +191,13 @@ match_aod_2="entire atmosphere" # match_pm25_1="MASSDEN" match_pm25_2="8 m above ground" -match_pm25_3="aerosol=Missing" +match_pm25_3="aerosol=Total Aerosol" match_pm25_4="aerosol_size <2.5e-06" match_pm25_sp="aerosol=Total Aerosol" # match_pm10_1="MASSDEN" match_pm10_2="8 m above ground" -match_pm10_3="aerosol=Missing" +match_pm10_3="aerosol=Total Aerosol" match_pm10_4="aerosol_size <1e-05" match_pm10_sp="aerosol=Total Aerosol" # From aa6b80ccac86446ba9305984824d9f4c032f2327 Mon Sep 17 00:00:00 2001 From: Ho-Chun Huang Date: Tue, 2 Sep 2025 18:34:32 +0000 Subject: [PATCH 07/11] change cpreq and others --- .../stats/cam/jevs_cam_rrfs_chem_grid2obs_aeronet_aod_stats.sh | 3 --- .../stats/cam/jevs_cam_rrfs_chem_grid2obs_airnow_pm10_stats.sh | 3 --- .../stats/cam/jevs_cam_rrfs_chem_grid2obs_airnow_pm25_stats.sh | 3 --- scripts/stats/cam/exevs_cam_rrfs_chem_grid2obs_stats.sh | 2 +- 4 files changed, 1 insertion(+), 10 deletions(-) diff --git a/dev/drivers/scripts/stats/cam/jevs_cam_rrfs_chem_grid2obs_aeronet_aod_stats.sh b/dev/drivers/scripts/stats/cam/jevs_cam_rrfs_chem_grid2obs_aeronet_aod_stats.sh index af14ffb296..db7ec49399 100755 --- a/dev/drivers/scripts/stats/cam/jevs_cam_rrfs_chem_grid2obs_aeronet_aod_stats.sh +++ b/dev/drivers/scripts/stats/cam/jevs_cam_rrfs_chem_grid2obs_aeronet_aod_stats.sh @@ -48,9 +48,6 @@ export RUN=${RUN:-chem} export VERIF_CASE=${VERIF_CASE:-grid2obs} export MODELNAME=${MODELNAME:-rrfs} -export VDATE=${VDATE:-$(date --date="3 days ago" +%Y%m%d)} -echo "VDATE=${VDATE}" - export DATA_TYPE=aeronet_aod export COMIN=/lfs/h2/emc/vpppg/noscrub/$USER/$NET/${evs_ver_2d} diff --git a/dev/drivers/scripts/stats/cam/jevs_cam_rrfs_chem_grid2obs_airnow_pm10_stats.sh b/dev/drivers/scripts/stats/cam/jevs_cam_rrfs_chem_grid2obs_airnow_pm10_stats.sh index 0d7a8478ff..ca6541970b 100755 --- a/dev/drivers/scripts/stats/cam/jevs_cam_rrfs_chem_grid2obs_airnow_pm10_stats.sh +++ b/dev/drivers/scripts/stats/cam/jevs_cam_rrfs_chem_grid2obs_airnow_pm10_stats.sh @@ -49,9 +49,6 @@ export RUN=${RUN:-chem} export VERIF_CASE=${VERIF_CASE:-grid2obs} export MODELNAME=${MODELNAME:-rrfs} -export VDATE=${VDATE:-$(date --date="3 days ago" +%Y%m%d)} -echo "VDATE=${VDATE}" - export DATA_TYPE=airnow_pm10 export COMIN=/lfs/h2/emc/vpppg/noscrub/$USER/$NET/${evs_ver_2d} diff --git a/dev/drivers/scripts/stats/cam/jevs_cam_rrfs_chem_grid2obs_airnow_pm25_stats.sh b/dev/drivers/scripts/stats/cam/jevs_cam_rrfs_chem_grid2obs_airnow_pm25_stats.sh index 364256bee3..a498bb2b46 100755 --- a/dev/drivers/scripts/stats/cam/jevs_cam_rrfs_chem_grid2obs_airnow_pm25_stats.sh +++ b/dev/drivers/scripts/stats/cam/jevs_cam_rrfs_chem_grid2obs_airnow_pm25_stats.sh @@ -49,9 +49,6 @@ export RUN=${RUN:-chem} export VERIF_CASE=${VERIF_CASE:-grid2obs} export MODELNAME=${MODELNAME:-rrfs} -export VDATE=${VDATE:-$(date --date="3 days ago" +%Y%m%d)} -echo "VDATE=${VDATE}" - export DATA_TYPE=airnow_pm25 export COMIN=/lfs/h2/emc/vpppg/noscrub/$USER/$NET/${evs_ver_2d} diff --git a/scripts/stats/cam/exevs_cam_rrfs_chem_grid2obs_stats.sh b/scripts/stats/cam/exevs_cam_rrfs_chem_grid2obs_stats.sh index a0bc329219..3c0fed31c3 100755 --- a/scripts/stats/cam/exevs_cam_rrfs_chem_grid2obs_stats.sh +++ b/scripts/stats/cam/exevs_cam_rrfs_chem_grid2obs_stats.sh @@ -173,7 +173,7 @@ for ObsType in ${grid2obs_list}; do if [ "${vhr}" == "23" ]; then stat_file_count=$(find ${COMOUTsmall} -name "*${OutputId}*" | wc -l) if [ ${stat_file_count} -ne 0 ]; then - cpreq ${COMOUTsmall}/*${OutputId}* ${finalstat} + cp -v ${COMOUTsmall}/*${OutputId}* ${finalstat} cd ${finalstat} run_metplus.py ${stat_analysis_conf_file} ${config_common} export err=$?; err_chk From 6178cff7ad9cb2fb610ff63443e9e54422860196 Mon Sep 17 00:00:00 2001 From: Ho-Chun Huang Date: Fri, 12 Sep 2025 13:54:16 +0000 Subject: [PATCH 08/11] update threshold values for plotting variables --- .../chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM10.conf | 4 ++-- .../chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM25.conf | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM10.conf b/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM10.conf index 0a9873f3dd..79e853b43c 100755 --- a/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM10.conf +++ b/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM10.conf @@ -61,8 +61,8 @@ OBS_VAR1_NAME= PM10 OBS_VAR1_LEVELS= A1 OBS_VAR1_OPTIONS = message_type = "AIRNOW_HOURLY_AQOBS"; censor_thresh = [ <=0.0 ]; censor_val=[ 0.0 ]; set_attr_units = "10^-6g/m^3"; -## POINT_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [ >5, >10, >15, >20, >25, >30, >35, >40, >45, >50, >55, >60, >65 ]; duplicate_flag = UNIQUE; obs_summary = NEAREST; -POINT_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [ >5, >10, >15, >20, >25, >30, >35, >40, >45, >50, >55, >60, >65 ]; duplicate_flag = UNIQUE; +## POINT_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [ >10, >20, >30, >40, >60, >80, >100, >200 ]; duplicate_flag = UNIQUE; obs_summary = NEAREST; +POINT_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [ >10, >20, >30, >40, >60, >80, >100, >200 ]; duplicate_flag = UNIQUE; ## POINT_STAT_MET_CONFIG_OVERRIDES = duplicate_flag = UNIQUE; obs_summary = NEAREST; {ENV[met_config_overrides]} diff --git a/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM25.conf b/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM25.conf index d7b991286e..d5b2db43eb 100755 --- a/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM25.conf +++ b/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM25.conf @@ -61,8 +61,8 @@ OBS_VAR1_NAME= PM25 OBS_VAR1_LEVELS= A1 OBS_VAR1_OPTIONS = message_type = "AIRNOW_HOURLY_AQOBS"; censor_thresh = [ <=0.0 ]; censor_val=[ 0.0 ]; set_attr_units = "10^-6g/m^3"; -## POINT_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [ >5, >10, >15, >20, >25, >30, >35, >40, >45, >50, >55, >60, >65 ]; duplicate_flag = UNIQUE; obs_summary = NEAREST; -POINT_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [ >5, >10, >15, >20, >25, >30, >35, >40, >45, >50, >55, >60, >65 ]; duplicate_flag = UNIQUE; +## POINT_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [ >5, >10, >15, >20, >25, >35, >40, >50, >60, >80, >100 ]; duplicate_flag = UNIQUE; obs_summary = NEAREST; +POINT_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [ >5, >10, >15, >20, >25, >35, >40, >50, >60, >80, >100 ]; duplicate_flag = UNIQUE; ## POINT_STAT_MET_CONFIG_OVERRIDES = duplicate_flag = UNIQUE; obs_summary = NEAREST; {ENV[met_config_overrides]} From 18cd1e307d3cf0ba5e020627fd6e16eaf47634c0 Mon Sep 17 00:00:00 2001 From: Ho-Chun Huang Date: Tue, 21 Oct 2025 11:41:09 +0000 Subject: [PATCH 09/11] separate prep mdl and obs --- jobs/JEVS_CAM_PREP | 5 ++-- jobs/JEVS_CAM_STATS | 2 +- ...PointStat_fcstRRFSAero_obsAERONET_AOD.conf | 11 +++----- ...PointStat_fcstRRFSAero_obsAIRNOW_PM10.conf | 16 +++--------- ...PointStat_fcstRRFSAero_obsAIRNOW_PM25.conf | 16 +++--------- .../cam/exevs_cam_rrfs_chem_grid2obs_prep.sh | 21 +++++++++------ .../cam/exevs_cam_rrfs_chem_grid2obs_stats.sh | 26 +++++++++---------- 7 files changed, 38 insertions(+), 59 deletions(-) diff --git a/jobs/JEVS_CAM_PREP b/jobs/JEVS_CAM_PREP index c563c54e7c..2757778cfe 100755 --- a/jobs/JEVS_CAM_PREP +++ b/jobs/JEVS_CAM_PREP @@ -122,9 +122,8 @@ export COMINrrfs=${COMINrrfs:-$(compath.py ${envir}/com/rrfs/${rrfs_ver})} if [ "${RUN}" == "chem" ]; then export DCOMINairnow=${DCOMINairnow:-${DCOMROOT}} export DCOMINaeronet=${DCOMINaeronet:-${DCOMROOT}} - export COMOUTprep=${COMOUTprep:-${COMOUT}/${RUN}.${INITDATE}/${MODELNAME}} - - mkdir -p ${COMOUTprep} + export COMOUTprepmdl=${COMOUTprepmdl:-${COMOUT}/${RUN}.${INITDATE}/${MODELNAME}} + export COMOUTprepobs=${COMOUTprepobs:-${COMOUT}/${RUN}.${INITDATE}/obs} # alternative choice is an option different from aqobs export airnow_hourly_type=${airnow_hourly_type:-aqobs} diff --git a/jobs/JEVS_CAM_STATS b/jobs/JEVS_CAM_STATS index 21d280349f..94b6f11d81 100755 --- a/jobs/JEVS_CAM_STATS +++ b/jobs/JEVS_CAM_STATS @@ -142,7 +142,7 @@ else fi if [ "${RUN}" == "chem" ]; then - export EVSINrrfs=${EVSINrrfs:-${COMIN}/prep/${COMPONENT}} + export EVSINprep=${EVSINprep:-${COMIN}/prep/${COMPONENT}} export DATA_TYPE=${DATA_TYPE:-aeronet_aod} # alternative choice is an option different from aqobs export airnow_hourly_type=${airnow_hourly_type:-aqobs} diff --git a/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAERONET_AOD.conf b/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAERONET_AOD.conf index 25cc3c68f4..3434eb4a76 100755 --- a/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAERONET_AOD.conf +++ b/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAERONET_AOD.conf @@ -1,9 +1,9 @@ [dir] OUTPUT_BASE = {ENV[DATA]} -FCST_POINT_STAT_INPUT_DIR = {ENV[EVSINrrfs]} +FCST_POINT_STAT_INPUT_DIR = {ENV[EVSINprep]} -OBS_POINT_STAT_INPUT_DIR = {ENV[EVSINrrfs]} +OBS_POINT_STAT_INPUT_DIR = {ENV[EVSINprep]} POINT_STAT_OUTPUT_DIR = {ENV[RUNTIME_STATS]} @@ -17,8 +17,6 @@ LOOP_BY = VALID POINT_STAT_CONFIG_FILE = {PARM_BASE}/met_config/PointStatConfig_wrapped -## METPLUS_FIX = {METPLUS_BASE}/fix - METPLUS_CONF = {OUTPUT_BASE}/config/{ENV[COMPONENT]}/metplus_final_{ENV[MODELNAME]}_{ENV[RUN]}_{ENV[ObsType]}.conf LOG_METPLUS = {LOG_DIR}/metplus_{ENV[MODELNAME]}_{ENV[RUN]}_{ENV[ObsType]}.log.{LOG_TIMESTAMP_TEMPLATE} @@ -37,14 +35,11 @@ VALID_INCREMENT = 1H OBS_POINT_STAT_WINDOW_BEGIN = -1800 OBS_POINT_STAT_WINDOW_END = 1800 -#OBS_WINDOW_BEGIN = -3600 -#OBS_WINDOW_END = 3600 POINT_STAT_DESC = NA FCST_VAR1_NAME = AOTK FCST_VAR1_LEVELS = L0 -## FCST_VAR1_OPTIONS = GRIB2_ipdtmpl_index = [ 2, 10 ]; GRIB2_ipdtmpl_val = [ 62000, 545 ] OBS_VAR1_NAME = AOD OBS_VAR1_LEVELS = Z550 @@ -77,6 +72,6 @@ POINT_STAT_MESSAGE_TYPE = [filename_templates] FCST_POINT_STAT_INPUT_TEMPLATE = {ENV[RUN]}.{init?fmt=%Y%m%d}/{ENV[MODELNAME]}/{ENV[mdl_cyc]}/{ENV[MODELNAME]}.t{ENV[mdl_cyc]}z.prslev.f{lead?fmt=%3H}.reduced.grib2 -OBS_POINT_STAT_INPUT_TEMPLATE = {ENV[RUN]}.{valid?fmt=%Y%m%d}/{ENV[MODELNAME]}/{ENV[ObsSrc]}_All_{valid?fmt=%Y%m%d}_lev15.nc +OBS_POINT_STAT_INPUT_TEMPLATE = {ENV[RUN]}.{valid?fmt=%Y%m%d}/obs/{ENV[ObsSrc]}_All_{valid?fmt=%Y%m%d}_lev15.nc POINT_STAT_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}.stat diff --git a/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM10.conf b/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM10.conf index 79e853b43c..f0e8d6300f 100755 --- a/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM10.conf +++ b/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM10.conf @@ -2,9 +2,9 @@ OUTPUT_BASE = {ENV[DATA]} -FCST_POINT_STAT_INPUT_DIR = {ENV[EVSINrrfs]} +FCST_POINT_STAT_INPUT_DIR = {ENV[EVSINprep]} -OBS_POINT_STAT_INPUT_DIR = {ENV[EVSINrrfs]} +OBS_POINT_STAT_INPUT_DIR = {ENV[EVSINprep]} POINT_STAT_OUTPUT_DIR = {ENV[RUNTIME_STATS]} @@ -54,18 +54,13 @@ POINT_STAT_MASK_POLY = {ENV[MASKS]}/Bukovsky_G{ENV[gridspec_conus]}_CONUS.nc, {E FCST_VAR1_NAME = MASSDEN FCST_VAR1_LEVELS = Z8 FCST_VAR1_OPTIONS = convert(x) = x * 10^9; GRIB2_ipdtmpl_index = [ 5 ]; GRIB2_ipdtmpl_val = [ 100 ]; set_attr_units = "10^-6g/m^3"; set_attr_name = "PMTC"; -## Use direct record numbe such as record #2 -## FCST_VAR1_LEVELS = [R02] OBS_VAR1_NAME= PM10 OBS_VAR1_LEVELS= A1 OBS_VAR1_OPTIONS = message_type = "AIRNOW_HOURLY_AQOBS"; censor_thresh = [ <=0.0 ]; censor_val=[ 0.0 ]; set_attr_units = "10^-6g/m^3"; -## POINT_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [ >10, >20, >30, >40, >60, >80, >100, >200 ]; duplicate_flag = UNIQUE; obs_summary = NEAREST; POINT_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [ >10, >20, >30, >40, >60, >80, >100, >200 ]; duplicate_flag = UNIQUE; -## POINT_STAT_MET_CONFIG_OVERRIDES = duplicate_flag = UNIQUE; obs_summary = NEAREST; {ENV[met_config_overrides]} - POINT_STAT_OUTPUT_FLAG_CNT = STAT POINT_STAT_OUTPUT_FLAG_CTC = STAT POINT_STAT_OUTPUT_FLAG_CTS = STAT @@ -73,22 +68,17 @@ POINT_STAT_OUTPUT_FLAG_SL1L2 = STAT POINT_STAT_INTERP_TYPE_METHOD = BILIN POINT_STAT_INTERP_TYPE_WIDTH = 2 -## for NEAREST option -## POINT_STAT_INTERP_TYPE_WIDTH = 1 POINT_STAT_REGRID_TO_GRID = NONE POINT_STAT_REGRID_METHOD = BILIN POINT_STAT_REGRID_WIDTH = 2 -## for NEAREST option -## POINT_STAT_REGRID_WIDTH = 1 POINT_STAT_OUTPUT_PREFIX = {ENV[OutputId]} -POINT_STAT_MESSAGE_TYPE = [filename_templates] FCST_POINT_STAT_INPUT_TEMPLATE = {ENV[RUN]}.{init?fmt=%Y%m%d}/{ENV[MODELNAME]}/{ENV[mdl_cyc]}/{ENV[MODELNAME]}.t{ENV[mdl_cyc]}z.prslev.f{lead?fmt=%3H}.reduced.grib2 -OBS_POINT_STAT_INPUT_TEMPLATE = {ENV[RUN]}.{valid?fmt=%Y%m%d?shift=-3600}/{ENV[MODELNAME]}/{ENV[ObsSrc]}_{ENV[HOURLY_INPUT_TYPE]}_{valid?fmt=%Y%m%d%H?shift=-3600}.nc +OBS_POINT_STAT_INPUT_TEMPLATE = {ENV[RUN]}.{valid?fmt=%Y%m%d?shift=-3600}/obs/{ENV[ObsSrc]}_{ENV[HOURLY_INPUT_TYPE]}_{valid?fmt=%Y%m%d%H?shift=-3600}.nc POINT_STAT_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}.stat diff --git a/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM25.conf b/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM25.conf index d5b2db43eb..e487b7941f 100755 --- a/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM25.conf +++ b/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM25.conf @@ -2,9 +2,9 @@ OUTPUT_BASE = {ENV[DATA]} -FCST_POINT_STAT_INPUT_DIR = {ENV[EVSINrrfs]} +FCST_POINT_STAT_INPUT_DIR = {ENV[EVSINprep]} -OBS_POINT_STAT_INPUT_DIR = {ENV[EVSINrrfs]} +OBS_POINT_STAT_INPUT_DIR = {ENV[EVSINprep]} POINT_STAT_OUTPUT_DIR = {ENV[RUNTIME_STATS]} @@ -54,18 +54,13 @@ POINT_STAT_MASK_POLY = {ENV[MASKS]}/Bukovsky_G{ENV[gridspec_conus]}_CONUS.nc, {E FCST_VAR1_NAME = MASSDEN FCST_VAR1_LEVELS = Z8 FCST_VAR1_OPTIONS = convert(x) = x * 10^9; GRIB2_ipdtmpl_index = [ 5 ]; GRIB2_ipdtmpl_val = [ 25 ]; set_attr_units = "10^-6g/m^3"; set_attr_name = "PMTF"; -## Use direct record numbe such as record #1 -## FCST_VAR1_LEVELS = [R01] OBS_VAR1_NAME= PM25 OBS_VAR1_LEVELS= A1 OBS_VAR1_OPTIONS = message_type = "AIRNOW_HOURLY_AQOBS"; censor_thresh = [ <=0.0 ]; censor_val=[ 0.0 ]; set_attr_units = "10^-6g/m^3"; -## POINT_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [ >5, >10, >15, >20, >25, >35, >40, >50, >60, >80, >100 ]; duplicate_flag = UNIQUE; obs_summary = NEAREST; POINT_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [ >5, >10, >15, >20, >25, >35, >40, >50, >60, >80, >100 ]; duplicate_flag = UNIQUE; -## POINT_STAT_MET_CONFIG_OVERRIDES = duplicate_flag = UNIQUE; obs_summary = NEAREST; {ENV[met_config_overrides]} - POINT_STAT_OUTPUT_FLAG_CNT = STAT POINT_STAT_OUTPUT_FLAG_CTC = STAT POINT_STAT_OUTPUT_FLAG_CTS = STAT @@ -73,22 +68,17 @@ POINT_STAT_OUTPUT_FLAG_SL1L2 = STAT POINT_STAT_INTERP_TYPE_METHOD = BILIN POINT_STAT_INTERP_TYPE_WIDTH = 2 -## for NEAREST option -## POINT_STAT_INTERP_TYPE_WIDTH = 1 POINT_STAT_REGRID_TO_GRID = NONE POINT_STAT_REGRID_METHOD = BILIN POINT_STAT_REGRID_WIDTH = 2 -## for NEAREST option -## POINT_STAT_REGRID_WIDTH = 1 POINT_STAT_OUTPUT_PREFIX = {ENV[OutputId]} -POINT_STAT_MESSAGE_TYPE = [filename_templates] FCST_POINT_STAT_INPUT_TEMPLATE = {ENV[RUN]}.{init?fmt=%Y%m%d}/{ENV[MODELNAME]}/{ENV[mdl_cyc]}/{ENV[MODELNAME]}.t{ENV[mdl_cyc]}z.prslev.f{lead?fmt=%3H}.reduced.grib2 -OBS_POINT_STAT_INPUT_TEMPLATE = {ENV[RUN]}.{valid?fmt=%Y%m%d?shift=-3600}/{ENV[MODELNAME]}/{ENV[ObsSrc]}_{ENV[HOURLY_INPUT_TYPE]}_{valid?fmt=%Y%m%d%H?shift=-3600}.nc +OBS_POINT_STAT_INPUT_TEMPLATE = {ENV[RUN]}.{valid?fmt=%Y%m%d?shift=-3600}/obs/{ENV[ObsSrc]}_{ENV[HOURLY_INPUT_TYPE]}_{valid?fmt=%Y%m%d%H?shift=-3600}.nc POINT_STAT_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}.stat diff --git a/scripts/prep/cam/exevs_cam_rrfs_chem_grid2obs_prep.sh b/scripts/prep/cam/exevs_cam_rrfs_chem_grid2obs_prep.sh index de3c0411aa..85c10eabdf 100755 --- a/scripts/prep/cam/exevs_cam_rrfs_chem_grid2obs_prep.sh +++ b/scripts/prep/cam/exevs_cam_rrfs_chem_grid2obs_prep.sh @@ -45,7 +45,7 @@ for OBTTYPE in ${obstype}; do if [ "${OBTTYPE}" == "aeronet" ]; then flag_process_ascii_aeronet="YES" if [ "${check_restart}" == "YES" ]; then ## Check ASCII2NC AERONET AOD file for RESTART ability - checkfile=${COMOUTprep}/${OBTTYPE}_All_${INITDATE}_lev15.nc + checkfile=${COMOUTprepobs}/${OBTTYPE}_All_${INITDATE}_lev15.nc if [ -s ${checkfile} ]; then msg=$(ncdump -h ${checkfile} 1> /dev/null 2>&1 ; err=$? ; echo ${err} ) if [ ${msg} -eq 0 ]; then flag_process_ascii_aeronet="NO"; fi @@ -68,7 +68,8 @@ for OBTTYPE in ${obstype}; do if [ ${SENDCOM} = "YES" ]; then cpfile=${finalprep}/${OBTTYPE}_All_${INITDATE}_lev15.nc if [ -e ${cpfile} ]; then - cp -v ${cpfile} ${COMOUTprep} + mkdir -p ${COMOUTprepobs} + cp -v ${cpfile} ${COMOUTprepobs} fi fi fi @@ -110,13 +111,13 @@ for OBTTYPE in ${obstype}; do if [ "${check_restart}" == "YES" ]; then ## Check ASCII2NC AIRNOW files for RESTART ability checkfile="${OBTTYPE}_${HOURLY_OUTPUT_TYPE}_*.nc" - obs_file_count=$(find ${COMOUTprep} -name ${checkfile} | wc -l ) + obs_file_count=$(find ${COMOUTprepobs} -name ${checkfile} | wc -l ) if [ ${obs_file_count} -eq 0 ]; then let ic=0 elif [ ${obs_file_count} -eq ${total_num_file} ]; then ## check corrupted ASCII2NC file vldhr=$(printf %2.2d ${endvhr}) - checkfile="${COMOUTprep}/${OBTTYPE}_${HOURLY_OUTPUT_TYPE}_${INITDATE}${vldhr}.nc" + checkfile="${COMOUTprepobs}/${OBTTYPE}_${HOURLY_OUTPUT_TYPE}_${INITDATE}${vldhr}.nc" msg=$(ncdump -h ${checkfile} 1> /dev/null 2>&1 ; err=$? ; echo ${err} ) if [ ${msg} -eq 0 ]; then let ic=${endvhr}+1 ## skip current AIRNOW Processing @@ -151,7 +152,10 @@ for OBTTYPE in ${obstype}; do export err=$?; err_chk if [ ${SENDCOM} = "YES" ]; then cpfile=${finalprep}/airnow_hourly_aqobs_${INITDATE}${VHOUR}.nc - if [ -e ${cpfile} ]; then cp -v ${cpfile} ${COMOUTprep}; fi + if [ -e ${cpfile} ]; then + mkdir -p ${COMOUTprepobs} + cp -v ${cpfile} ${COMOUTprepobs} + fi fi fi else @@ -206,13 +210,12 @@ let inc=1 for mdl_cyc in "${cyc_opt[@]}"; do com_rrfs=${COMINrrfs}/${MODELNAME}.${INITDATE}/${mdl_cyc} if [ -d ${com_rrfs} ]; then - prep_rrfs=${COMOUTprep}/${mdl_cyc} - if [ ! -d ${prep_rrfs} ]; then mkdir -p ${prep_rrfs}; fi + prep_rrfs=${COMOUTprepmdl}/${mdl_cyc} let hour_now=1 let max_hour=84 let total_num_file=${max_hour} - if [ "${check_restart}" == "YES" ]; then ## Check RRFS reduced grib2 files for RESTART ability + if [ "${check_restart}" == "YES" ] && [ -d ${prep_rrfs} ]; then ## Check RRFS reduced grib2 files for RESTART ability checkfile="${MODELNAME}.t${mdl_cyc}z.prslev.f*.reduced.grib2" mdl_file_count=$(find ${prep_rrfs} -name ${checkfile} | wc -l ) if [ ${mdl_file_count} -eq 0 ]; then @@ -243,6 +246,7 @@ for mdl_cyc in "${cyc_opt[@]}"; do if [ -s ${check_reduced_file} ]; then echo "Found file ${check_reduced_file}" if [ ${SENDCOM} = "YES" ]; then + if [ ! -d ${prep_rrfs} ]; then mkdir -p ${prep_rrfs}; fi cp -v ${check_reduced_file} ${prep_rrfs} fi elif [ -s ${check_full_file} ]; then @@ -270,6 +274,7 @@ for mdl_cyc in "${cyc_opt[@]}"; do echo "DEBUG: Number of extracted record is ${number_of_record} for file extract_pm10" cat extract_pm25 extract_pm10 extract_aod > ${reduced_rec_grib2} if [ ${SENDCOM} = "YES" ]; then + if [ ! -d ${prep_rrfs} ]; then mkdir -p ${prep_rrfs}; fi cp -v ${reduced_rec_grib2} ${prep_rrfs} fi else diff --git a/scripts/stats/cam/exevs_cam_rrfs_chem_grid2obs_stats.sh b/scripts/stats/cam/exevs_cam_rrfs_chem_grid2obs_stats.sh index 3c0fed31c3..1e76c2df75 100755 --- a/scripts/stats/cam/exevs_cam_rrfs_chem_grid2obs_stats.sh +++ b/scripts/stats/cam/exevs_cam_rrfs_chem_grid2obs_stats.sh @@ -78,7 +78,7 @@ for ObsType in ${grid2obs_list}; do stat_analysis_conf_file=${CONFIGevs}/Statanalysis_fcstRRFSAero_obs${OBSTYPE}.conf if [ "${ObsSrc}" == "aeronet" ]; then - check_file=${EVSINrrfs}/${RUN}.${VDATE}/${MODELNAME}/${ObsSrc}_All_${VDATE}_lev15.nc + check_file=${EVSINprep}/${RUN}.${VDATE}/obs/${ObsSrc}_All_${VDATE}_lev15.nc num_obs_found=0 if [ -s ${check_file} ]; then num_obs_found=1 @@ -91,7 +91,7 @@ for ObsType in ${grid2obs_list}; do vld_date=$(${NDATE} -1 ${cdate} | cut -c1-8) vld_time=$(${NDATE} -1 ${cdate} | cut -c1-10) - check_file=${EVSINrrfs}/${RUN}.${vld_date}/${MODELNAME}/${ObsSrc}_${HOURLY_INPUT_TYPE}_${vld_time}.nc + check_file=${EVSINprep}/${RUN}.${vld_date}/obs/${ObsSrc}_${HOURLY_INPUT_TYPE}_${vld_time}.nc num_obs_found=0 if [ -s ${check_file} ]; then num_obs_found=1 @@ -118,7 +118,7 @@ for ObsType in ${grid2obs_list}; do aday=`echo ${adate} |cut -c1-8` acyc=`echo ${adate} |cut -c9-10` if [ "${acyc}" == "${mdl_cyc}" ]; then - fcst_file=${EVSINrrfs}/${RUN}.${aday}/${MODELNAME}/${acyc}/${MODELNAME}.t${acyc}z.prslev.f${filehr}.reduced.grib2 + fcst_file=${EVSINprep}/${RUN}.${aday}/${MODELNAME}/${acyc}/${MODELNAME}.t${acyc}z.prslev.f${filehr}.reduced.grib2 if [ -s ${fcst_file} ]; then if [ "${check_restart}" == "YES" ]; then point_stat_file="${COMOUTsmall}/point_stat_${OutputId}_${fhr}0000L_${VDATE}_${vhr}0000V.stat" @@ -158,18 +158,18 @@ for ObsType in ${grid2obs_list}; do echo "DEBUG: There is no pre-processed ${ObsVar} ${CMODEL}-smoke and dust ${mdl_cyc} cycle forecast output validated at ${vhr}Z, the metplus stats process will be skipped" fi fi - if [ "${SENDCOM}" == "YES" ]; then - if [ -d ${RUNTIME_STATS}/${VDATE}.stat ]; then ## does not exist if run_metplus.py did not execute - stat_file_count=$(find ${RUNTIME_STATS}/${VDATE}.stat -name "*${OutputId}*" | wc -l) - if [ ${stat_file_count} -ne 0 ]; then - mkdir -p ${COMOUTsmall} - cp -v ${RUNTIME_STATS}/${VDATE}.stat/*${OutputId}* ${COMOUTsmall} - else - echo "DEBUG: NO stats file *${OutputId}* found in ${RUNTIME_STATS}/${VDATE}.stat" - fi + done ## init hour loop + if [ "${SENDCOM}" == "YES" ]; then + if [ -d ${RUNTIME_STATS}/${VDATE}.stat ]; then ## does not exist if run_metplus.py did not execute + stat_file_count=$(find ${RUNTIME_STATS}/${VDATE}.stat -name "*${OutputId}*" | wc -l) + if [ ${stat_file_count} -ne 0 ]; then + mkdir -p ${COMOUTsmall} + cp -v ${RUNTIME_STATS}/${VDATE}.stat/*${OutputId}* ${COMOUTsmall} + else + echo "DEBUG: NO stats file *${OutputId}* found in ${RUNTIME_STATS}/${VDATE}.stat" fi fi - done ## init hour loop + fi if [ "${vhr}" == "23" ]; then stat_file_count=$(find ${COMOUTsmall} -name "*${OutputId}*" | wc -l) if [ ${stat_file_count} -ne 0 ]; then From a8133a2291dbdfbb3a188b8c08d10b07c9f7368c Mon Sep 17 00:00:00 2001 From: Ho-Chun Huang Date: Wed, 24 Dec 2025 19:19:58 +0000 Subject: [PATCH 10/11] update for pre obs and mdl sub dir --- jobs/JEVS_CAM_PREP | 7 ++--- jobs/JEVS_CAM_STATS | 4 +-- ...PointStat_fcstRRFSAero_obsAERONET_AOD.conf | 9 +++++-- ...PointStat_fcstRRFSAero_obsAIRNOW_PM10.conf | 14 ++++++++-- ...PointStat_fcstRRFSAero_obsAIRNOW_PM25.conf | 14 ++++++++-- ...h => exevs_prep_cam_rrfs_chem_grid2obs.sh} | 9 +++---- versions/run.ver | 27 ++++++++++--------- 7 files changed, 55 insertions(+), 29 deletions(-) rename scripts/prep/cam/{exevs_cam_rrfs_chem_grid2obs_prep.sh => exevs_prep_cam_rrfs_chem_grid2obs.sh} (97%) diff --git a/jobs/JEVS_CAM_PREP b/jobs/JEVS_CAM_PREP index 2757778cfe..d60342f33a 100755 --- a/jobs/JEVS_CAM_PREP +++ b/jobs/JEVS_CAM_PREP @@ -122,8 +122,9 @@ export COMINrrfs=${COMINrrfs:-$(compath.py ${envir}/com/rrfs/${rrfs_ver})} if [ "${RUN}" == "chem" ]; then export DCOMINairnow=${DCOMINairnow:-${DCOMROOT}} export DCOMINaeronet=${DCOMINaeronet:-${DCOMROOT}} - export COMOUTprepmdl=${COMOUTprepmdl:-${COMOUT}/${RUN}.${INITDATE}/${MODELNAME}} - export COMOUTprepobs=${COMOUTprepobs:-${COMOUT}/${RUN}.${INITDATE}/obs} + export COMOUT=${COMOUT:-$(compath.py -o ${NET}/${evs_ver})} + export COMOUTprepmdl=${COMOUTprepmdl:-${COMOUT}/${STEP}/${COMPONENT}/${RUN}.${INITDATE}/${MODELNAME}} + export COMOUTprepobs=${COMOUTprepobs:-${COMOUT}/${STEP}/${COMPONENT}/${RUN}.${INITDATE}/obs} # alternative choice is an option different from aqobs export airnow_hourly_type=${airnow_hourly_type:-aqobs} @@ -155,7 +156,7 @@ fi # Execute the script #################################### if [ "${RUN}" == "chem" ]; then - ${HOMEevs}/scripts/${STEP}/${COMPONENT}/exevs_${COMPONENT}_${MODELNAME}_${RUN}_${VERIF_CASE}_${STEP}.sh + ${HOMEevs}/scripts/${STEP}/${COMPONENT}/exevs_${STEP}_${COMPONENT}_${MODELNAME}_${RUN}_${VERIF_CASE}.sh else $HOMEevs/scripts/$STEP/$COMPONENT/exevs_${MODELNAME}_${VERIF_CASE}_${STEP}.sh fi diff --git a/jobs/JEVS_CAM_STATS b/jobs/JEVS_CAM_STATS index 94b6f11d81..0293bdc887 100755 --- a/jobs/JEVS_CAM_STATS +++ b/jobs/JEVS_CAM_STATS @@ -142,7 +142,7 @@ else fi if [ "${RUN}" == "chem" ]; then - export EVSINprep=${EVSINprep:-${COMIN}/prep/${COMPONENT}} + export EVSINrrfs=${EVSINrrfs:-${COMIN}/prep/${COMPONENT}} export DATA_TYPE=${DATA_TYPE:-aeronet_aod} # alternative choice is an option different from aqobs export airnow_hourly_type=${airnow_hourly_type:-aqobs} @@ -158,7 +158,7 @@ if [ $VERIF_CASE = radar ] || [ $VERIF_CASE = severe ]; then elif [ $MODELNAME = nam_firewxnest ] || [ $MODELNAME = rrfs_firewxnest ]; then $HOMEevs/scripts/${STEP}/${COMPONENT}/exevs_${COMPONENT}_${MODELNAME}_${VERIF_CASE}_${STEP}.sh elif [ "${RUN}" == "chem" ]; then - ${HOMEevs}/scripts/${STEP}/${COMPONENT}/exevs_${COMPONENT}_${MODELNAME}_${RUN}_${VERIF_CASE}_${STEP}.sh + ${HOMEevs}/scripts/${STEP}/${COMPONENT}/exevs_${STEP}_${COMPONENT}_${MODELNAME}_${RUN}_${VERIF_CASE}.sh else $HOMEevs/scripts/${STEP}/${COMPONENT}/exevs_${MODELNAME}_${VERIF_CASE}_${STEP}.sh fi diff --git a/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAERONET_AOD.conf b/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAERONET_AOD.conf index 3434eb4a76..87d1416259 100755 --- a/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAERONET_AOD.conf +++ b/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAERONET_AOD.conf @@ -1,9 +1,9 @@ [dir] OUTPUT_BASE = {ENV[DATA]} -FCST_POINT_STAT_INPUT_DIR = {ENV[EVSINprep]} +FCST_POINT_STAT_INPUT_DIR = {ENV[EVSINrrfs]} -OBS_POINT_STAT_INPUT_DIR = {ENV[EVSINprep]} +OBS_POINT_STAT_INPUT_DIR = {ENV[EVSINrrfs]} POINT_STAT_OUTPUT_DIR = {ENV[RUNTIME_STATS]} @@ -17,6 +17,8 @@ LOOP_BY = VALID POINT_STAT_CONFIG_FILE = {PARM_BASE}/met_config/PointStatConfig_wrapped +## METPLUS_FIX = {METPLUS_BASE}/fix + METPLUS_CONF = {OUTPUT_BASE}/config/{ENV[COMPONENT]}/metplus_final_{ENV[MODELNAME]}_{ENV[RUN]}_{ENV[ObsType]}.conf LOG_METPLUS = {LOG_DIR}/metplus_{ENV[MODELNAME]}_{ENV[RUN]}_{ENV[ObsType]}.log.{LOG_TIMESTAMP_TEMPLATE} @@ -35,11 +37,14 @@ VALID_INCREMENT = 1H OBS_POINT_STAT_WINDOW_BEGIN = -1800 OBS_POINT_STAT_WINDOW_END = 1800 +#OBS_WINDOW_BEGIN = -3600 +#OBS_WINDOW_END = 3600 POINT_STAT_DESC = NA FCST_VAR1_NAME = AOTK FCST_VAR1_LEVELS = L0 +## FCST_VAR1_OPTIONS = GRIB2_ipdtmpl_index = [ 2, 10 ]; GRIB2_ipdtmpl_val = [ 62000, 545 ] OBS_VAR1_NAME = AOD OBS_VAR1_LEVELS = Z550 diff --git a/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM10.conf b/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM10.conf index f0e8d6300f..3b2ce3b702 100755 --- a/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM10.conf +++ b/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM10.conf @@ -2,9 +2,9 @@ OUTPUT_BASE = {ENV[DATA]} -FCST_POINT_STAT_INPUT_DIR = {ENV[EVSINprep]} +FCST_POINT_STAT_INPUT_DIR = {ENV[EVSINrrfs]} -OBS_POINT_STAT_INPUT_DIR = {ENV[EVSINprep]} +OBS_POINT_STAT_INPUT_DIR = {ENV[EVSINrrfs]} POINT_STAT_OUTPUT_DIR = {ENV[RUNTIME_STATS]} @@ -54,13 +54,18 @@ POINT_STAT_MASK_POLY = {ENV[MASKS]}/Bukovsky_G{ENV[gridspec_conus]}_CONUS.nc, {E FCST_VAR1_NAME = MASSDEN FCST_VAR1_LEVELS = Z8 FCST_VAR1_OPTIONS = convert(x) = x * 10^9; GRIB2_ipdtmpl_index = [ 5 ]; GRIB2_ipdtmpl_val = [ 100 ]; set_attr_units = "10^-6g/m^3"; set_attr_name = "PMTC"; +## Use direct record numbe such as record #2 +## FCST_VAR1_LEVELS = [R02] OBS_VAR1_NAME= PM10 OBS_VAR1_LEVELS= A1 OBS_VAR1_OPTIONS = message_type = "AIRNOW_HOURLY_AQOBS"; censor_thresh = [ <=0.0 ]; censor_val=[ 0.0 ]; set_attr_units = "10^-6g/m^3"; +## POINT_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [ >10, >20, >30, >40, >60, >80, >100, >200 ]; duplicate_flag = UNIQUE; obs_summary = NEAREST; POINT_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [ >10, >20, >30, >40, >60, >80, >100, >200 ]; duplicate_flag = UNIQUE; +## POINT_STAT_MET_CONFIG_OVERRIDES = duplicate_flag = UNIQUE; obs_summary = NEAREST; {ENV[met_config_overrides]} + POINT_STAT_OUTPUT_FLAG_CNT = STAT POINT_STAT_OUTPUT_FLAG_CTC = STAT POINT_STAT_OUTPUT_FLAG_CTS = STAT @@ -68,12 +73,17 @@ POINT_STAT_OUTPUT_FLAG_SL1L2 = STAT POINT_STAT_INTERP_TYPE_METHOD = BILIN POINT_STAT_INTERP_TYPE_WIDTH = 2 +## for NEAREST option +## POINT_STAT_INTERP_TYPE_WIDTH = 1 POINT_STAT_REGRID_TO_GRID = NONE POINT_STAT_REGRID_METHOD = BILIN POINT_STAT_REGRID_WIDTH = 2 +## for NEAREST option +## POINT_STAT_REGRID_WIDTH = 1 POINT_STAT_OUTPUT_PREFIX = {ENV[OutputId]} +POINT_STAT_MESSAGE_TYPE = [filename_templates] diff --git a/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM25.conf b/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM25.conf index e487b7941f..ae80c44ed3 100755 --- a/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM25.conf +++ b/parm/metplus_config/stats/cam/chem_grid2obs/PointStat_fcstRRFSAero_obsAIRNOW_PM25.conf @@ -2,9 +2,9 @@ OUTPUT_BASE = {ENV[DATA]} -FCST_POINT_STAT_INPUT_DIR = {ENV[EVSINprep]} +FCST_POINT_STAT_INPUT_DIR = {ENV[EVSINrrfs]} -OBS_POINT_STAT_INPUT_DIR = {ENV[EVSINprep]} +OBS_POINT_STAT_INPUT_DIR = {ENV[EVSINrrfs]} POINT_STAT_OUTPUT_DIR = {ENV[RUNTIME_STATS]} @@ -54,13 +54,18 @@ POINT_STAT_MASK_POLY = {ENV[MASKS]}/Bukovsky_G{ENV[gridspec_conus]}_CONUS.nc, {E FCST_VAR1_NAME = MASSDEN FCST_VAR1_LEVELS = Z8 FCST_VAR1_OPTIONS = convert(x) = x * 10^9; GRIB2_ipdtmpl_index = [ 5 ]; GRIB2_ipdtmpl_val = [ 25 ]; set_attr_units = "10^-6g/m^3"; set_attr_name = "PMTF"; +## Use direct record numbe such as record #1 +## FCST_VAR1_LEVELS = [R01] OBS_VAR1_NAME= PM25 OBS_VAR1_LEVELS= A1 OBS_VAR1_OPTIONS = message_type = "AIRNOW_HOURLY_AQOBS"; censor_thresh = [ <=0.0 ]; censor_val=[ 0.0 ]; set_attr_units = "10^-6g/m^3"; +## POINT_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [ >5, >10, >15, >20, >25, >35, >40, >50, >60, >80, >100 ]; duplicate_flag = UNIQUE; obs_summary = NEAREST; POINT_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [ >5, >10, >15, >20, >25, >35, >40, >50, >60, >80, >100 ]; duplicate_flag = UNIQUE; +## POINT_STAT_MET_CONFIG_OVERRIDES = duplicate_flag = UNIQUE; obs_summary = NEAREST; {ENV[met_config_overrides]} + POINT_STAT_OUTPUT_FLAG_CNT = STAT POINT_STAT_OUTPUT_FLAG_CTC = STAT POINT_STAT_OUTPUT_FLAG_CTS = STAT @@ -68,12 +73,17 @@ POINT_STAT_OUTPUT_FLAG_SL1L2 = STAT POINT_STAT_INTERP_TYPE_METHOD = BILIN POINT_STAT_INTERP_TYPE_WIDTH = 2 +## for NEAREST option +## POINT_STAT_INTERP_TYPE_WIDTH = 1 POINT_STAT_REGRID_TO_GRID = NONE POINT_STAT_REGRID_METHOD = BILIN POINT_STAT_REGRID_WIDTH = 2 +## for NEAREST option +## POINT_STAT_REGRID_WIDTH = 1 POINT_STAT_OUTPUT_PREFIX = {ENV[OutputId]} +POINT_STAT_MESSAGE_TYPE = [filename_templates] diff --git a/scripts/prep/cam/exevs_cam_rrfs_chem_grid2obs_prep.sh b/scripts/prep/cam/exevs_prep_cam_rrfs_chem_grid2obs.sh similarity index 97% rename from scripts/prep/cam/exevs_cam_rrfs_chem_grid2obs_prep.sh rename to scripts/prep/cam/exevs_prep_cam_rrfs_chem_grid2obs.sh index 85c10eabdf..6db9296313 100755 --- a/scripts/prep/cam/exevs_cam_rrfs_chem_grid2obs_prep.sh +++ b/scripts/prep/cam/exevs_prep_cam_rrfs_chem_grid2obs.sh @@ -152,9 +152,9 @@ for OBTTYPE in ${obstype}; do export err=$?; err_chk if [ ${SENDCOM} = "YES" ]; then cpfile=${finalprep}/airnow_hourly_aqobs_${INITDATE}${VHOUR}.nc - if [ -e ${cpfile} ]; then + if [ -e ${cpfile} ]; then mkdir -p ${COMOUTprepobs} - cp -v ${cpfile} ${COMOUTprepobs} + cp -v ${cpfile} ${COMOUTprepobs} fi fi fi @@ -211,11 +211,12 @@ for mdl_cyc in "${cyc_opt[@]}"; do com_rrfs=${COMINrrfs}/${MODELNAME}.${INITDATE}/${mdl_cyc} if [ -d ${com_rrfs} ]; then prep_rrfs=${COMOUTprepmdl}/${mdl_cyc} + if [ ! -d ${prep_rrfs} ]; then mkdir -p ${prep_rrfs}; fi let hour_now=1 let max_hour=84 let total_num_file=${max_hour} - if [ "${check_restart}" == "YES" ] && [ -d ${prep_rrfs} ]; then ## Check RRFS reduced grib2 files for RESTART ability + if [ "${check_restart}" == "YES" ]; then ## Check RRFS reduced grib2 files for RESTART ability checkfile="${MODELNAME}.t${mdl_cyc}z.prslev.f*.reduced.grib2" mdl_file_count=$(find ${prep_rrfs} -name ${checkfile} | wc -l ) if [ ${mdl_file_count} -eq 0 ]; then @@ -246,7 +247,6 @@ for mdl_cyc in "${cyc_opt[@]}"; do if [ -s ${check_reduced_file} ]; then echo "Found file ${check_reduced_file}" if [ ${SENDCOM} = "YES" ]; then - if [ ! -d ${prep_rrfs} ]; then mkdir -p ${prep_rrfs}; fi cp -v ${check_reduced_file} ${prep_rrfs} fi elif [ -s ${check_full_file} ]; then @@ -274,7 +274,6 @@ for mdl_cyc in "${cyc_opt[@]}"; do echo "DEBUG: Number of extracted record is ${number_of_record} for file extract_pm10" cat extract_pm25 extract_pm10 extract_aod > ${reduced_rec_grib2} if [ ${SENDCOM} = "YES" ]; then - if [ ! -d ${prep_rrfs} ]; then mkdir -p ${prep_rrfs}; fi cp -v ${reduced_rec_grib2} ${prep_rrfs} fi else diff --git a/versions/run.ver b/versions/run.ver index dba09a90d1..de6d34801c 100644 --- a/versions/run.ver +++ b/versions/run.ver @@ -1,33 +1,33 @@ export evs_ver=v2.0.0 export bacio_ver=2.4.1 -export bufr_ver=12.0.1 +export bufr_ver=12.2.0 export cdo_ver=1.9.8 export cfp_ver=2.0.4 export craympich_ver=8.1.19 -export craypals_ver=1.2.2 +export craypals_ver=1.3.2 export craype_ver=2.7.17 export g2_ver=3.4.5 -export g2c_ver=1.7.0 +export g2c_ver=2.2.0 export geos_ver=3.8.1 export grib_util_ver=1.2.4 export gsl_ver=2.7 export hdf5_ver=1.12.2 export imagemagick_ver=7.0.8-7 export intel_ver=19.1.3.304 -export ip_ver=4.0.0 +export ip_ver=5.2.0 export jasper_ver=2.0.25 -export libfabric_ver=1.11.0.0. +export libfabric_ver=1.20.1 export libjpeg_ver=9c export libpng_ver=1.6.37 -export met_ver=12.0.0-rc1 -export metcalcpy_ver=3.0.0-rc1 -export metdataio_ver=3.0.0-rc1 -export metplotpy_ver=3.0.0-rc1 -export metplus_ver=6.0.0-rc1 -export nco_ver=5.0.6 +export met_ver=12.0.1 +export metcalcpy_ver=3.0.0 +export metdataio_ver=3.0.0 +export metplotpy_ver=3.0.0 +export metplus_ver=6.0.0 +export nco_ver=5.2.4 export netcdf_ver=4.7.4 -export PrgEnvintel_ver=8.3.3 +export PrgEnvintel_ver=8.5.0 export prod_envir_ver=2.0.6 export prod_util_ver=2.0.14 export proj_ver=7.1.0 @@ -62,9 +62,10 @@ export obsproc_ver=v1.2 export rap_ver=v5.1 export rrfs_ver=na export rtma_ver=v2.10 -export rtofs_ver=v2.4 +export rtofs_ver=v2.5 export sref_ver=v7.1 export st4_ver=v4.1 +export ukmet_ver=v2.2 export urma_ver=v2.10 export wafs_ver=v7.0 From 3fbd9454c8e2d7d8983ecc1e580629af3cd4181f Mon Sep 17 00:00:00 2001 From: Ho-Chun Huang Date: Wed, 24 Dec 2025 19:33:58 +0000 Subject: [PATCH 11/11] renam exevs_stats_cam_rrfs_chem_grid2obs.sh --- ... => exevs_stats_cam_rrfs_chem_grid2obs.sh} | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) rename scripts/stats/cam/{exevs_cam_rrfs_chem_grid2obs_stats.sh => exevs_stats_cam_rrfs_chem_grid2obs.sh} (91%) diff --git a/scripts/stats/cam/exevs_cam_rrfs_chem_grid2obs_stats.sh b/scripts/stats/cam/exevs_stats_cam_rrfs_chem_grid2obs.sh similarity index 91% rename from scripts/stats/cam/exevs_cam_rrfs_chem_grid2obs_stats.sh rename to scripts/stats/cam/exevs_stats_cam_rrfs_chem_grid2obs.sh index 1e76c2df75..bb8c8740cb 100755 --- a/scripts/stats/cam/exevs_cam_rrfs_chem_grid2obs_stats.sh +++ b/scripts/stats/cam/exevs_stats_cam_rrfs_chem_grid2obs.sh @@ -78,7 +78,7 @@ for ObsType in ${grid2obs_list}; do stat_analysis_conf_file=${CONFIGevs}/Statanalysis_fcstRRFSAero_obs${OBSTYPE}.conf if [ "${ObsSrc}" == "aeronet" ]; then - check_file=${EVSINprep}/${RUN}.${VDATE}/obs/${ObsSrc}_All_${VDATE}_lev15.nc + check_file=${EVSINrrfs}/${RUN}.${VDATE}/obs/${ObsSrc}_All_${VDATE}_lev15.nc num_obs_found=0 if [ -s ${check_file} ]; then num_obs_found=1 @@ -91,7 +91,7 @@ for ObsType in ${grid2obs_list}; do vld_date=$(${NDATE} -1 ${cdate} | cut -c1-8) vld_time=$(${NDATE} -1 ${cdate} | cut -c1-10) - check_file=${EVSINprep}/${RUN}.${vld_date}/obs/${ObsSrc}_${HOURLY_INPUT_TYPE}_${vld_time}.nc + check_file=${EVSINrrfs}/${RUN}.${vld_date}/obs/${ObsSrc}_${HOURLY_INPUT_TYPE}_${vld_time}.nc num_obs_found=0 if [ -s ${check_file} ]; then num_obs_found=1 @@ -118,7 +118,7 @@ for ObsType in ${grid2obs_list}; do aday=`echo ${adate} |cut -c1-8` acyc=`echo ${adate} |cut -c9-10` if [ "${acyc}" == "${mdl_cyc}" ]; then - fcst_file=${EVSINprep}/${RUN}.${aday}/${MODELNAME}/${acyc}/${MODELNAME}.t${acyc}z.prslev.f${filehr}.reduced.grib2 + fcst_file=${EVSINrrfs}/${RUN}.${aday}/${MODELNAME}/${acyc}/${MODELNAME}.t${acyc}z.prslev.f${filehr}.reduced.grib2 if [ -s ${fcst_file} ]; then if [ "${check_restart}" == "YES" ]; then point_stat_file="${COMOUTsmall}/point_stat_${OutputId}_${fhr}0000L_${VDATE}_${vhr}0000V.stat" @@ -158,18 +158,18 @@ for ObsType in ${grid2obs_list}; do echo "DEBUG: There is no pre-processed ${ObsVar} ${CMODEL}-smoke and dust ${mdl_cyc} cycle forecast output validated at ${vhr}Z, the metplus stats process will be skipped" fi fi - done ## init hour loop - if [ "${SENDCOM}" == "YES" ]; then - if [ -d ${RUNTIME_STATS}/${VDATE}.stat ]; then ## does not exist if run_metplus.py did not execute - stat_file_count=$(find ${RUNTIME_STATS}/${VDATE}.stat -name "*${OutputId}*" | wc -l) - if [ ${stat_file_count} -ne 0 ]; then - mkdir -p ${COMOUTsmall} - cp -v ${RUNTIME_STATS}/${VDATE}.stat/*${OutputId}* ${COMOUTsmall} - else - echo "DEBUG: NO stats file *${OutputId}* found in ${RUNTIME_STATS}/${VDATE}.stat" + if [ "${SENDCOM}" == "YES" ]; then + if [ -d ${RUNTIME_STATS}/${VDATE}.stat ]; then ## does not exist if run_metplus.py did not execute + stat_file_count=$(find ${RUNTIME_STATS}/${VDATE}.stat -name "*${OutputId}*" | wc -l) + if [ ${stat_file_count} -ne 0 ]; then + mkdir -p ${COMOUTsmall} + cp -v ${RUNTIME_STATS}/${VDATE}.stat/*${OutputId}* ${COMOUTsmall} + else + echo "DEBUG: NO stats file *${OutputId}* found in ${RUNTIME_STATS}/${VDATE}.stat" + fi fi fi - fi + done ## init hour loop if [ "${vhr}" == "23" ]; then stat_file_count=$(find ${COMOUTsmall} -name "*${OutputId}*" | wc -l) if [ ${stat_file_count} -ne 0 ]; then