diff --git a/.github/workflows/ci_unit_tests.yaml b/.github/workflows/ci_unit_tests.yaml index 2198947bf90..281f10f461e 100644 --- a/.github/workflows/ci_unit_tests.yaml +++ b/.github/workflows/ci_unit_tests.yaml @@ -63,7 +63,7 @@ jobs: sudo chmod -R a+w /scratch3/NCEPDEV source rocoto-${{ env.ROCOTO_VERSION }}/rocoto_path.sh cd global-workflow/sorc - git submodule update --init -j 2 ufs_model.fd gsi_monitor.fd wxflow + git submodule update --init -j 2 ufs_model.fd wxflow ./link_workflow.sh export PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${PWD}/wxflow/src" # Create test data directory for unit tests diff --git a/.gitmodules b/.gitmodules index 461697b559e..191f76aadae 100644 --- a/.gitmodules +++ b/.gitmodules @@ -8,13 +8,12 @@ [submodule "sorc/gfs_utils.fd"] path = sorc/gfs_utils.fd url = https://github.com/NOAA-EMC/gfs-utils +[submodule "sorc/gdas.cd"] + path = sorc/gdas.cd + url = https://github.com/NOAA-EMC/GDASApp.git [submodule "sorc/ufs_utils.fd"] path = sorc/ufs_utils.fd url = https://github.com/ufs-community/UFS_UTILS.git -[submodule "sorc/gdas.cd"] - path = sorc/gcdas.cd - url = https://github.com/NOAA-EMC/GDASApp.git - ignore = dirty [submodule "sorc/gsi_utils.fd"] path = sorc/gsi_utils.fd url = https://github.com/NOAA-EMC/GSI-Utils.git diff --git a/dev/jobs/JGCDAS_PREPARE_OBS b/dev/jobs/JGCDAS_PREPARE_OBS new file mode 100755 index 00000000000..1d5dd7c0f24 --- /dev/null +++ b/dev/jobs/JGCDAS_PREPARE_OBS @@ -0,0 +1,44 @@ +#! /usr/bin/env bash + +source "${HOMEgcafs}/ush/preamble.sh" +source "${HOMEgcafs}/ush/jjob_header.sh" -e "prepareobs" -c "base prepareobs" + +############################################## +# Set variables used in the script +############################################## +export COMOUT_OBS="${COMROOT}/${RUN}.${PDY}/${cyc}/obs/chem" + +############################################## +# Begin JOB SPECIFIC work +############################################## + +############################################################### +# Run relevant script + +EXSCRIPT=${DUMPAODPY:-${HOMEgcafs}/scripts/exgcdas_prepare_obs.py} +${EXSCRIPT} +status=$? +if [[ ${status} -ne 0 ]]; then + exit "${status}" +fi + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]]; then + cat "${pgmout}" +fi + +########################################## +# Remove the Temporary working directory +########################################## +cd "${DATAROOT}" || exit +if [[ "${KEEPDATA}" == "NO" ]]; then + rm -rf "${DATA}" +fi + +exit 0 diff --git a/dev/jobs/JGLOBAL_OFFLINE_ATMOS_ANALYSIS b/dev/jobs/JGLOBAL_OFFLINE_ATMOS_ANALYSIS index 4e457b185d7..cb3b956ca47 100755 --- a/dev/jobs/JGLOBAL_OFFLINE_ATMOS_ANALYSIS +++ b/dev/jobs/JGLOBAL_OFFLINE_ATMOS_ANALYSIS @@ -22,7 +22,8 @@ export GPREFIX="${GDUMP}.t${gcyc}z." RUN="gdas" DUMP="gdas" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMIN_OBSPROC:COM_OBSPROC_TMPL \ - COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL + COMINgfs:COM_GFS_TMPL +export COMINgfs_ATMOS_ANALYSIS=${COMINgfs}/atmos/ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMOUT_OBS:COM_OBS_TMPL \ diff --git a/dev/parm/config/gcafs/config.prepareobs b/dev/parm/config/gcafs/config.prepareobs new file mode 100644 index 00000000000..a172e3df9bd --- /dev/null +++ b/dev/parm/config/gcafs/config.prepareobs @@ -0,0 +1,9 @@ +#!/bin/bash -x + +########## config.prepareobs ########## + +echo "BEGIN: config.prepareobs" + +# Get task specific resources +source "${EXPDIR}/config.resources" prepareobs +echo "END: config.prepareobs" diff --git a/dev/parm/config/gcafs/config.resources b/dev/parm/config/gcafs/config.resources index 5ef6de40077..03dc6bb3084 100644 --- a/dev/parm/config/gcafs/config.resources +++ b/dev/parm/config/gcafs/config.resources @@ -18,6 +18,7 @@ if (( $# != 1 )); then echo "snowanl esnowanl" echo "prep_emissions" echo "aeroanlinit aeroanlvar aeroanlfinal aeroanlgenb" + echo "prepareobs" echo "offlineanl" echo "anal sfcanl analcalc analdiag anlstat fcst echgres" echo "upp atmos_products" @@ -363,6 +364,15 @@ case ${step} in tasks_per_node=$(( max_tasks_per_node / threads_per_task )) ;; + "prepareobs") + ntasks=80 + walltime="00:20:00" + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + memory="3072M" + ;; + + "aeroanlinit") # below lines are for creating JEDI YAML case "${CASE}" in diff --git a/dev/scripts/exgcdas_prepare_obs.py b/dev/scripts/exgcdas_prepare_obs.py new file mode 100755 index 00000000000..978a788ba4f --- /dev/null +++ b/dev/scripts/exgcdas_prepare_obs.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +# exgcdas_prepare_obs.py +# This script will collect and preprocess +# aerosol optical depth observations for +# global aerosol assimilation +import os + +from wxflow import AttrDict, Logger, cast_strdict_as_dtypedict, parse_j2yaml +from pyobsforge.task.aero_prepobs import AerosolObsPrep + +# Initialize root logger +logger = Logger(level='DEBUG', colored_log=True) + + +if __name__ == '__main__': + + # Take configuration from environment and cast it as python dictionary + config_env = cast_strdict_as_dtypedict(os.environ) + # Take configuration from YAML file to augment/append config dict + config_yaml = parse_j2yaml(os.path.join(config_env['HOMEgcafs'], 'parm', 'chem', 'prepare_obs.yaml'), config_env) + # Combine configs together + config = AttrDict(**config_env, **config_yaml['aoddump']) + + aeroObs = AerosolObsPrep(config) + aeroObs.initialize() + aeroObs.execute() + aeroObs.finalize() diff --git a/dev/ush/setup_gcafs_for_nco.py b/dev/ush/setup_gcafs_for_nco.py index 1fe719c2f0d..14c7b592e96 100755 --- a/dev/ush/setup_gcafs_for_nco.py +++ b/dev/ush/setup_gcafs_for_nco.py @@ -18,17 +18,63 @@ # which is assumed to be two directories up from the current file global_workflow_dir = os.path.abspath(os.path.join(current_dir_path, "../..")) +gcafs_ex_scripts = { + "exgcafs_forecast.sh": "exglobal_forecast.sh", + "exgcafs_prep_emissions.py": "exglobal_prep_emissions.py", + "exgcafs_atmos_post_manager.sh": "exglobal_atmos_pmgr.sh", + "exgcafs_atmos_products.sh": "exglobal_atmos_products.sh", +} +gcdas_ex_scripts = { + "exgcdas_forecast.sh": "exglobal_forecast.sh", + "exgcdas_prep_emissions.py": "exglobal_prep_emissions.py", + "exgcdas_atmos_post_manager.sh": "exglobal_atmos_pmgr.sh", + "exgcdas_atmos_products.sh": "exglobal_atmos_products.sh", + "exgcdas_atmos_initialize.py": "exglobal_offline_atmos_analysis.py", + "exgcdas_surface_initialize.sh": "exglobal_atmos_sfcanl.sh", + "exgcdas_aero_analysis_initialize.py": "exglobal_aero_analysis_initialize.py", + "exgcdas_aero_analysis_variational.py": "exglobal_aero_analysis_variational.py", + "exgcdas_aero_analysis_finalize.py": "exglobal_aero_analysis_finalize.py", + "exgcdas_aero_analysis_calc.sh": "exglobal_atmos_analysis_calc.sh", + "exgcdas_aero_analysis_stats.py": "exglobal_analysis_stats.py", + "exgcdas_aero_analysis_generate_bmatrix.py": "exgdas_aero_analysis_generate_bmatrix.py", + "exgcdas_prepare_obs.py": "exgcdas_prepare_obs.py", + # need to add something here for the post job once Yaping's PR is in ? +} + +gcafs_jobs = { + "JGCAFS_FORECAST": "JGLOBAL_FORECAST", + "JGCAFS_PREP_EMISSIONS": "JGLOBAL_PREP_EMISSIONS", + "JGCAFS_ATMOS_POST_MANAGER": "JGLOBAL_ATMOS_POST_MANAGER", + "JGCAFS_ATMOS_PRODUCTS": "JGLOBAL_ATMOS_PRODUCTS", +} +gcdas_jobs = { + "JGCDAS_FORECAST": "JGLOBAL_FORECAST", + "JGCDAS_PREP_EMISSIONS": "JGLOBAL_PREP_EMISSIONS", + "JGCDAS_ATMOS_POST_MANAGER": "JGLOBAL_ATMOS_POST_MANAGER", + "JGCDAS_ATMOS_PRODUCTS": "JGLOBAL_ATMOS_PRODUCTS", + "JGCDAS_ATMOS_INITIALIZE": "JGLOBAL_OFFLINE_ATMOS_ANALYSIS", + "JGCDAS_SURFACE_INITIALIZE": "JGLOBAL_ATMOS_SFCANL", + "JGCDAS_AERO_ANALYSIS_INITIALIZE": "JGLOBAL_AERO_ANALYSIS_INITIALIZE", + "JGCDAS_AERO_ANALYSIS_VARIATIONAL": "JGLOBAL_AERO_ANALYSIS_VARIATIONAL", + "JGCDAS_AERO_ANALYSIS_FINALIZE": "JGLOBAL_AERO_ANALYSIS_FINALIZE", + "JGCDAS_AERO_ANALYSIS_CALC": "JGLOBAL_ATMOS_ANALYSIS_CALC", + "JGCDAS_AERO_ANALYSIS_STATS": "JGLOBAL_ANALYSIS_STATS", + "JGCDAS_AERO_ANALYSIS_GENERATE_BMATRIX": "JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX", + "JGCDAS_PREPARE_OBS": "JGCDAS_PREPARE_OBS", + # need to add something here for the post job once Yaping's PR is in +} + def replace_gfs_with_gcafs(input_file): """ Replace all instances of FOOgfs with FOOgcafs in the given input file. This matches patterns like HOMEgfs -> HOMEgcafs, USHgfs -> USHgcafs, etc. - + Parameters ---------- input_file : str Path to the file to modify - + Returns ------- int @@ -36,71 +82,50 @@ def replace_gfs_with_gcafs(input_file): """ if not os.path.exists(input_file): raise FileNotFoundError(f"File not found: {input_file}") - + # Read the file content with open(input_file, 'r') as f: content = f.read() - + # Count and replace all instances of FOOgfs with FOOgcafs # This will match patterns like: HOMEgfs, USHgfs, PARMgfs, etc. # Does NOT match standalone "gfs" or quoted "gfs" import re - # Match word characters followed by "gfs" at word boundary, but ensure prefix has at least 2 chars - # This ensures we match variable names like HOMEgfs but not just "gfs" or "Xgfs" - pattern = r'(\w{2,})gfs\b' - + # Match word characters followed by "gfs" at word boundary, but ensure the character + # immediately before "gfs" is a capital letter (e.g., HOMEgfs, not pygfs) + pattern = r'(\w*[A-Z])gfs\b' + replacement_count = 0 + def replace_func(match): nonlocal replacement_count replacement_count += 1 prefix = match.group(1) return f"{prefix}gcafs" - + modified_content = re.sub(pattern, replace_func, content) - + # Write the modified content back to the file with open(input_file, 'w') as f: f.write(modified_content) - + return replacement_count def copy_job_files(global_workflow_dir): """ Copy job files from dev/jobs to jobs directory with appropriate renaming. - + Parameters ---------- global_workflow_dir : str Path to the global workflow directory - + Returns ------- list List of tuples containing (src_path, dest_path) for copied files """ - gcafs_jobs = { - "JGCAFS_FORECAST": "JGLOBAL_FORECAST", - "JGCAFS_PREP_EMISSIONS": "JGLOBAL_PREP_EMISSIONS", - "JGCAFS_ATMOS_POST_MANAGER": "JGLOBAL_ATMOS_POST_MANAGER", - "JGCAFS_ATMOS_PRODUCTS": "JGLOBAL_ATMOS_PRODUCTS", - } - gcdas_jobs = { - "JGCDAS_FORECAST": "JGLOBAL_FORECAST", - "JGCDAS_PREP_EMISSIONS": "JGLOBAL_PREP_EMISSIONS", - "JGCDAS_ATMOS_POST_MANAGER": "JGLOBAL_ATMOS_POST_MANAGER", - "JGCDAS_ATMOS_PRODUCTS": "JGLOBAL_ATMOS_PRODUCTS", - "JGCDAS_ATMOS_INITIALIZE": "JGLOBAL_OFFLINE_ATMOS_ANALYSIS", - "JGCDAS_SURFACE_INITIALIZE": "JGLOBAL_ATMOS_SFCANL", - "JGCDAS_AERO_ANALYSIS_INITIALIZE": "JGLOBAL_AERO_ANALYSIS_INITIALIZE", - "JGCDAS_AERO_ANALYSIS_VARIATIONAL": "JGLOBAL_AERO_ANALYSIS_VARIATIONAL", - "JGCDAS_AERO_ANALYSIS_FINALIZE": "JGLOBAL_AERO_ANALYSIS_FINALIZE", - "JGCDAS_AERO_ANALYSIS_CALC": "JGLOBAL_ATMOS_ANALYSIS_CALC", - "JGCDAS_AERO_ANALYSIS_STATS": "JGLOBAL_ANALYSIS_STATS", - "JGCDAS_AERO_ANALYSIS_GENERATE_BMATRIX": "JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX", - # JGCDAS_PREPARE_OBS is taken from ObsForge for v1, not in global-workflow, do this manually!! - # need to add something here for the post job once Yaping's PR is in - } job_file_copy_list = [] for dest_job, src_job in {**gcafs_jobs, **gcdas_jobs}.items(): @@ -115,46 +140,24 @@ def copy_job_files(global_workflow_dir): } # Execute the file operations FileHandler(job_file_handler).sync() - + return job_file_copy_list def copy_script_files(global_workflow_dir): """ Copy script files from dev/scripts to scripts directory with appropriate renaming. - + Parameters ---------- global_workflow_dir : str Path to the global workflow directory - + Returns ------- list List of tuples containing (src_path, dest_path) for copied files """ - gcafs_ex_scripts = { - "exgcafs_forecast.sh": "exglobal_forecast.sh", - "exgcafs_prep_emissions.sh": "exglobal_prep_emissions.py", - "exgcafs_atmos_post_manager.sh": "exglobal_atmos_pmgr.sh", - "exgcafs_atmos_products.sh": "exglobal_atmos_products.sh", - } - gcdas_ex_scripts = { - "exgcdas_forecast.sh": "exglobal_forecast.sh", - "exgcdas_prep_emissions.sh": "exglobal_prep_emissions.py", - "exgcdas_atmos_post_manager.sh": "exglobal_atmos_pmgr.sh", - "exgcdas_atmos_products.sh": "exglobal_atmos_products.sh", - "exgcdas_atmos_initialize.py": "exglobal_offline_atmos_analysis.py", - "exgcdas_surface_initialize.sh": "exglobal_atmos_sfcanl.sh", - "exgcdas_aero_analysis_initialize.py": "exglobal_aero_analysis_initialize.py", - "exgcdas_aero_analysis_variational.py": "exglobal_aero_analysis_variational.py", - "exgcdas_aero_analysis_finalize.py": "exglobal_aero_analysis_finalize.py", - "exgcdas_aero_analysis_calc.sh": "exglobal_atmos_analysis_calc.sh", - "exgcdas_aero_analysis_stats.py": "exglobal_analysis_stats.py", - "exgcdas_aero_analysis_generate_bmatrix.py": "exgdas_aero_analysis_generate_bmatrix.py", - # exgcdas_prepare_obs is taken from ObsForge for v1, not in global-workflow, do this manually!! - # need to add something here for the post job once Yaping's PR is in - } # if the scripts directory exists as a symlink, remove it first scripts_dir = os.path.join(global_workflow_dir, 'scripts') @@ -169,97 +172,66 @@ def copy_script_files(global_workflow_dir): # Create a FileHandler dictionary for scripts ex_script_file_handler = { 'mkdir': [os.path.join(global_workflow_dir, 'scripts')], - 'copy': ex_script_file_copy_list, + 'copy_opt': ex_script_file_copy_list, } # Execute the file operations for scripts FileHandler(ex_script_file_handler).sync() - + return ex_script_file_copy_list def remove_unused_executables(global_workflow_dir): """ Remove unused executables from the exec directory. - + Parameters ---------- global_workflow_dir : str Path to the global workflow directory - + Returns ------- list List of files that were successfully removed """ - unused_executables = [ - "gdas_apply_incr.x", - "gdas_fv3jedi_correction_increment.x", - "gdas_fv3jedi_ensemble_add_increment.x", - "gdas_fv3jedi_fv3inc.x", - "gdas_fv3jedi_land_ensrecenter.x", - "gdas_fv3jedi_scf_to_ioda.x", - "gdas_ioda_mean.x", - "gdas_soca_anpproc.x", - "gdas_soca_diagb.x", - "gdas_soca_diagnostics.x", - "gdas_soca_ens_handler.x", - "gdas_soca_error_covariance_toolbox.x", - "gdas_soca_gridgen.x", - "gdas_soca_hybridweights.x", - "gdas_soca_incr_handler.x", - "gdas_soca_obsstats.x", - "gdas_soca_setcorscales.x", - "gdas_soca_to_fv3.x", - "emcsfc_snow2mdl", - "emcsfc_ice_blend", - "calc_increment_ens.x", - "ensadd.x", - "ensppf.x", - "ensstat.x", - "fbwndgfs.x", - "fregrid", - "getsfcensmeanp.x", - "getsigensmeanp_smooth.x", - "getsigensstatp.x", - "gfs_bufr.x", - "mkgfsawps.x", - "ocnicepost.x", - "overgridid.x", - "oznmon_horiz.x", - "oznmon_time.x", - "radmon_angle.x", - "radmon_bcoef.x", - "radmon_bcor.x", - "rdbfmsua.x", - "radmon_time.x", - "recentersigp.x", - "regridStates.x", - "supvit.x", - "syndat_getjtbul.x", - "syndat_maksynrc.x", - "syndat_qctropcy.x", - "tave.x", - "tocsbufr.x", - "vint.x", - "wave_stat.x", - "webtitle.x" + desired_executables = [ + "calc_analysis.x", + "calc_increment_ens_ncio.x", + "enkf_chgres_recenter_nc.x", + "gaussian_sfcanl.x", + "gcafs_model.x", + "gdas_fv3jedi_chem_diagb.x", + "gdas_fv3jedi_error_covariance_toolbox.x", + "gdas_ioda-stats.x", + "gdas_obsprovider2ioda.x", + "gdas.x", + "global_cycle", + "interp_inc.x", + "nexus.x", + "tref_calc.x", + "upp.x" ] - + exec_dir = os.path.join(global_workflow_dir, 'exec') removed_files = [] - - for executable in unused_executables: - executable_path = os.path.join(exec_dir, executable) - if os.path.exists(executable_path): - try: - os.remove(executable_path) - removed_files.append(executable) - print(f"Removed unused executable: {executable}") - except OSError as e: - print(f"Error removing {executable}: {e}") - else: - print(f"Executable not found (already removed?): {executable}") - + + # Get all files in exec_dir + if os.path.exists(exec_dir): + all_files = [f for f in os.listdir(exec_dir) if os.path.isfile(os.path.join(exec_dir, f))] + + # Remove all files except those in desired_executables + for filename in all_files: + if filename not in desired_executables: + file_path = os.path.join(exec_dir, filename) + try: + os.remove(file_path) + removed_files.append(filename) + print(f"Removed unused executable: {filename}") + except OSError as e: + print(f"Error removing {filename}: {e}") + else: + print(f"Exec directory not found: {exec_dir}") + return removed_files @@ -278,7 +250,99 @@ def setup_gcafs_for_nco(): for file_path in all_copied_files: num_replacements = replace_gfs_with_gcafs(file_path) print(f"Modified {file_path}: {num_replacements} replacements made.") - - + + # Go through the copied job files and replace the scripts they call as appropriate + jobs_dir = os.path.join(global_workflow_dir, 'jobs') + for job_name in list(gcafs_jobs.keys()) + list(gcdas_jobs.keys()): + job_file_path = os.path.join(jobs_dir, job_name) + if os.path.exists(job_file_path): + # Determine which script mapping to use based on job name + if job_name.startswith('JGCAFS_'): + script_mapping = gcafs_ex_scripts + elif job_name.startswith('JGCDAS_'): + script_mapping = gcdas_ex_scripts + else: + continue + + # Read the job file content + with open(job_file_path, 'r') as f: + content = f.read() + + # Replace script calls based on the mapping + modified = False + for new_script, old_script in script_mapping.items(): + if old_script in content: + content = content.replace(old_script, new_script) + modified = True + print(f"In {job_name}: Replaced {old_script} with {new_script}") + + # Write back the modified content if changes were made + if modified: + with open(job_file_path, 'w') as f: + f.write(content) + + # Go through the ush directory and replace FOOgfs with FOOgcafs in all scripts + ush_dir = os.path.join(global_workflow_dir, 'ush') + for root, _, files in os.walk(ush_dir): + for file in files: + # Only process .sh and .py files + if file.endswith(('.sh', '.py')): + file_path = os.path.join(root, file) + num_replacements = replace_gfs_with_gcafs(file_path) + if num_replacements > 0: + print(f"Modified {file_path}: {num_replacements} replacements made.") + + # Go through the dev/ush and dev/workflow directories and replace FOOgfs with FOOgcafs in all scripts + # and YAMLs in dev/ci/cases + for subdir in ['dev/ush', 'dev/workflow', 'dev/ci/cases']: + dir_path = os.path.join(global_workflow_dir, subdir) + for root, _, files in os.walk(dir_path): + for file in files: + # Skip the current script to avoid self-modification + if file == 'setup_gcafs_for_nco.py': + continue + # Only process .sh and .py scripts and YAML files + if file.endswith(('.sh', '.py', '.yaml')): + file_path = os.path.join(root, file) + num_replacements = replace_gfs_with_gcafs(file_path) + if num_replacements > 0: + print(f"Modified {file_path}: {num_replacements} replacements made.") + + # Go through the dev/parm/config/gcafs directory and replace FOOgfs with FOOgcafs in all config files + config_gcafs_dir = os.path.join(global_workflow_dir, 'dev', 'parm', 'config', 'gcafs') + for root, _, files in os.walk(config_gcafs_dir): + for file in files: + # Only process config files + if file.startswith('config.'): + file_path = os.path.join(root, file) + num_replacements = replace_gfs_with_gcafs(file_path) + if num_replacements > 0: + print(f"Modified {file_path}: {num_replacements} replacements made.") + + # Go through parm/ directory and replace FOOgfs with FOOgcafs in all files + parm_dir = os.path.join(global_workflow_dir, 'parm') + for root, _, files in os.walk(parm_dir): + for file in files: + file_path = os.path.join(root, file) + try: + num_replacements = replace_gfs_with_gcafs(file_path) + if num_replacements > 0: + print(f"Modified {file_path}: {num_replacements} replacements made.") + except: # noqa + print(f"Skipping: {file_path}") + + # Go through sorc/gdas.cd/parm directory and replace FOOgcafs with FOOgcafs in all files + gcdas_parm_dir = os.path.join(global_workflow_dir, 'sorc', 'gdas.cd', 'parm') + for root, _, files in os.walk(gcdas_parm_dir): + for file in files: + file_path = os.path.join(root, file) + try: + num_replacements = replace_gfs_with_gcafs(file_path) + if num_replacements > 0: + print(f"Modified {file_path}: {num_replacements} replacements made (gcafs).") + except: # noqa + print(f"Skipping: {file_path}") + + if __name__ == "__main__": setup_gcafs_for_nco() diff --git a/ecf/defs/gcafs_v1.def b/ecf/defs/gcafs_v1.def index 36a9b484093..a8fc40f977e 100644 --- a/ecf/defs/gcafs_v1.def +++ b/ecf/defs/gcafs_v1.def @@ -7,8 +7,11 @@ suite gcafs_v1 edit QUEUE_ARCH 'dev_transfer' edit PROJENVIR 'DEV' edit MACHINE_SITE 'development' - edit ENVIR 'prod' - edit EXPDIR '%PACKAGEHOME%/dev/parm/config/gcafs' + edit ENVIR 'para' + edit EXPDIR '%PACKAGEHOME%/parm/config/gcafs' + edit COMROOT '/lfs/h2/emc/ptmp/cory.r.martin/gcafs/com' + edit DATAROOT '/lfs/h2/emc/ptmp/cory.r.martin/gcafs/dataroot' + edit ECF_INCLUDE '/lfs/h2/emc/da/noscrub/cory.r.martin/gcafs/git/release/gcafs.v1.0.0/ecf/include' family 00 edit CYC '00' family gcafs diff --git a/ecf/include/head.h b/ecf/include/head.h index 487c7509924..0c1074b8f72 100644 --- a/ecf/include/head.h +++ b/ecf/include/head.h @@ -1,3 +1,4 @@ +echo "HELLO CRM" date hostname set -xe # print commands as they are executed and enable signal trapping @@ -59,6 +60,12 @@ if [ -d /apps/ops/prod ]; then # On WCOSS2 echo "Listing modules from head.h:" module list set -x + if [[ ! " ops.prod ops.para " =~ " $(whoami) " ]]; then + echo "Allow over-riding defaults for developers" + if [ -n "%COMROOT:%" ]; then export COMROOT="%COMROOT:%"; fi + if [ -n "%DATAROOT:%" ]; then export DATAROOT="%DATAROOT:%"; fi + if [ -n "%DCOMROOT:%" ]; then export DCOMROOT="%DCOMROOT:%"; fi + fi fi timeout 300 ecflow_client --init=${ECF_RID} diff --git a/ecf/scripts/gcafs/atmos/post/jgcafs_atmos_post_manager.ecf b/ecf/scripts/gcafs/atmos/post/jgcafs_atmos_post_manager.ecf index a3c9153d05d..56937d28c90 100755 --- a/ecf/scripts/gcafs/atmos/post/jgcafs_atmos_post_manager.ecf +++ b/ecf/scripts/gcafs/atmos/post/jgcafs_atmos_post_manager.ecf @@ -25,6 +25,7 @@ module list export cyc=%CYC% export cycle=t%CYC%z +export EXPDIR=%EXPDIR% ############################################################ # CALL executable job script here diff --git a/ecf/scripts/gcafs/atmos/post/jgcafs_atmos_post_master.ecf b/ecf/scripts/gcafs/atmos/post/jgcafs_atmos_post_master.ecf index 6afb9f7c109..3550e7cfe54 100755 --- a/ecf/scripts/gcafs/atmos/post/jgcafs_atmos_post_master.ecf +++ b/ecf/scripts/gcafs/atmos/post/jgcafs_atmos_post_master.ecf @@ -45,6 +45,7 @@ export FHRGRP=%FHRGRP% export FHRLST=%FHRLST% export cyc=%CYC% export cycle=t%CYC%z +export EXPDIR=%EXPDIR% export USE_CFP=YES export g2tmpl_ver=v${g2tmpl_ver} diff --git a/ecf/scripts/gcafs/chem/jgcafs_prep_emissions.ecf b/ecf/scripts/gcafs/chem/jgcafs_prep_emissions.ecf index 9c4cc6a7dd7..9e6fcb6ab91 100755 --- a/ecf/scripts/gcafs/chem/jgcafs_prep_emissions.ecf +++ b/ecf/scripts/gcafs/chem/jgcafs_prep_emissions.ecf @@ -62,6 +62,7 @@ module list ############################################################# export cyc=%CYC% export cycle=t%CYC%z +export EXPDIR=%EXPDIR% if [[ -d "${HOMEgcafs}/sorc/wxflow/src" ]]; then PYTHONPATH="${HOMEgcafs}/sorc/wxflow/src${PYTHONPATH:+:${PYTHONPATH}}" diff --git a/ecf/scripts/gcafs/jgcafs_forecast.ecf b/ecf/scripts/gcafs/jgcafs_forecast.ecf index 0f0ebd9fb5e..b954503d08a 100755 --- a/ecf/scripts/gcafs/jgcafs_forecast.ecf +++ b/ecf/scripts/gcafs/jgcafs_forecast.ecf @@ -60,6 +60,7 @@ module list ############################################################# export cyc=%CYC% export cycle=t%CYC%z +export EXPDIR=%EXPDIR% if [[ -d "${HOMEgcafs}/sorc/wxflow/src" ]]; then PYTHONPATH="${HOMEgcafs}/sorc/wxflow/src${PYTHONPATH:+:${PYTHONPATH}}" diff --git a/ecf/scripts/gcdas/atmos/init/jgcdas_atmos_initialize.ecf b/ecf/scripts/gcdas/atmos/init/jgcdas_atmos_initialize.ecf index a4a87f7c2d2..8e6b3afd0a3 100755 --- a/ecf/scripts/gcdas/atmos/init/jgcdas_atmos_initialize.ecf +++ b/ecf/scripts/gcdas/atmos/init/jgcdas_atmos_initialize.ecf @@ -62,6 +62,7 @@ module list ############################################################# export cyc=%CYC% export cycle=t%CYC%z +export EXPDIR=%EXPDIR% if [[ -d "${HOMEgcafs}/sorc/wxflow/src" ]]; then PYTHONPATH="${HOMEgcafs}/sorc/wxflow/src${PYTHONPATH:+:${PYTHONPATH}}" @@ -72,7 +73,7 @@ export PYTHONPATH ############################################################ # CALL executable job script here ############################################################ -${HOMEgcafs}/jobs/JGLOBAL_OFFLINE_ATMOS_ANALYSIS +${HOMEgcafs}/jobs/JGCDAS_ATMOS_INITIALIZE if [ $? -ne 0 ]; then ecflow_client --msg="***JOB ${ECF_NAME} ERROR RUNNING J-SCRIPT ***" diff --git a/ecf/scripts/gcdas/atmos/init/jgcdas_surface_initialize.ecf b/ecf/scripts/gcdas/atmos/init/jgcdas_surface_initialize.ecf index 9106ab5e9ec..cd24d004d09 100755 --- a/ecf/scripts/gcdas/atmos/init/jgcdas_surface_initialize.ecf +++ b/ecf/scripts/gcdas/atmos/init/jgcdas_surface_initialize.ecf @@ -62,6 +62,7 @@ module list ############################################################# export cyc=%CYC% export cycle=t%CYC%z +export EXPDIR=%EXPDIR% if [[ -d "${HOMEgcafs}/sorc/wxflow/src" ]]; then PYTHONPATH="${HOMEgcafs}/sorc/wxflow/src${PYTHONPATH:+:${PYTHONPATH}}" @@ -72,7 +73,7 @@ export PYTHONPATH ############################################################ # CALL executable job script here ############################################################ -${HOMEgcafs}/jobs/JGLOBAL_ATMOS_SFCANL +${HOMEgcafs}/jobs/JGCDAS_SURFACE_INITIALIZE if [ $? -ne 0 ]; then ecflow_client --msg="***JOB ${ECF_NAME} ERROR RUNNING J-SCRIPT ***" diff --git a/ecf/scripts/gcdas/atmos/post/jgcdas_atmos_post_manager.ecf b/ecf/scripts/gcdas/atmos/post/jgcdas_atmos_post_manager.ecf index a3c9153d05d..56937d28c90 100755 --- a/ecf/scripts/gcdas/atmos/post/jgcdas_atmos_post_manager.ecf +++ b/ecf/scripts/gcdas/atmos/post/jgcdas_atmos_post_manager.ecf @@ -25,6 +25,7 @@ module list export cyc=%CYC% export cycle=t%CYC%z +export EXPDIR=%EXPDIR% ############################################################ # CALL executable job script here diff --git a/ecf/scripts/gcdas/atmos/post/jgcdas_atmos_post_master.ecf b/ecf/scripts/gcdas/atmos/post/jgcdas_atmos_post_master.ecf index 6afb9f7c109..3550e7cfe54 100755 --- a/ecf/scripts/gcdas/atmos/post/jgcdas_atmos_post_master.ecf +++ b/ecf/scripts/gcdas/atmos/post/jgcdas_atmos_post_master.ecf @@ -45,6 +45,7 @@ export FHRGRP=%FHRGRP% export FHRLST=%FHRLST% export cyc=%CYC% export cycle=t%CYC%z +export EXPDIR=%EXPDIR% export USE_CFP=YES export g2tmpl_ver=v${g2tmpl_ver} diff --git a/ecf/scripts/gcdas/chem/analysis/jgcdas_aerosol_analysis_calc.ecf b/ecf/scripts/gcdas/chem/analysis/jgcdas_aerosol_analysis_calc.ecf index 8a6a4a63401..6476ae2254c 100644 --- a/ecf/scripts/gcdas/chem/analysis/jgcdas_aerosol_analysis_calc.ecf +++ b/ecf/scripts/gcdas/chem/analysis/jgcdas_aerosol_analysis_calc.ecf @@ -45,13 +45,14 @@ export FHRGRP=%FHRGRP% export FHRLST=%FHRLST% export cyc=%CYC% export cycle=t%CYC%z +export EXPDIR=%EXPDIR% export USE_CFP=YES export g2tmpl_ver=v${g2tmpl_ver} ############################################################ # CALL executable job script here ############################################################ -echo "Do nothing yet, placeholder" +${HOMEgcafs}/jobs/JGCDAS_AERO_ANALYSIS_CALC if [ $? -ne 0 ]; then ecflow_client --msg="***JOB ${ECF_NAME} ERROR RUNNING J-SCRIPT ***" diff --git a/ecf/scripts/gcdas/chem/analysis/jgcdas_aerosol_analysis_finalize.ecf b/ecf/scripts/gcdas/chem/analysis/jgcdas_aerosol_analysis_finalize.ecf index b7dea625e96..2975d835dee 100755 --- a/ecf/scripts/gcdas/chem/analysis/jgcdas_aerosol_analysis_finalize.ecf +++ b/ecf/scripts/gcdas/chem/analysis/jgcdas_aerosol_analysis_finalize.ecf @@ -23,7 +23,7 @@ export RUN=%RUN% #TODO: Remove LMOD_TMOD_FIND_FIRST line when spack-stack on WCOSS2 export LMOD_TMOD_FIND_FIRST=yes # TODO: Add path to GDASApp libraries and cray-mpich as temporary patches -export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${HOMEgfs}/sorc/gdas.cd/build/lib" +export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${HOMEgcafs}/sorc/gdas.cd/build/lib" # TODO: Remove LD_LIBRARY_PATH line as soon as permanent solution is available export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/cray/pe/mpich/8.1.29/ofi/intel/2022.1/lib" @@ -32,13 +32,9 @@ module use /apps/ops/test/spack-stack-nco-1.9/modulefiles/Core module load stack-oneapi/2024.2.1 module load stack-cray-mpich/8.1.29 module load stack-python/3.11.7 -module load cmake/3.27.9 module load craype/2.7.17 module load cray-pals/1.3.2 -module load git/2.47.0 -module load git-lfs/3.5.1 - module load zstd/1.5.6 module load pigz/2.8 module load tar/1.34 @@ -51,45 +47,18 @@ module load nccmp/1.9.0.1 module load netcdf-fortran/4.6.1 module load nco/5.2.4 module load parallelio/2.6.2 -module load wget/1.21.1 module load boost/1.84.0 -module load ecbuild/3.7.2 -module load eccodes/2.33.0 module load eigen/3.4.0 module load openblas/0.3.24 module load eckit/1.28.3 module load fckit/0.13.2 module load fms/2024.02 -module load python-venv/1.0 -module load py-pyyaml/6.0.2 module load intel-oneapi-runtime/2024.2.1 module load glibc/2.31 -module load esmf/8.8.0 -module load atlas/0.40.0 -module load sp/2.5.0 -module load ip/5.1.0 -module load gsl-lite/0.37.0 -module load libjpeg/2.1.0 -module load krb5/1.21.1-1 -module load libtirpc/1.3.3 -module load hdf/4.2.15 -module load jedi-cmake/1.4.0 -module load libpng/1.6.37 -module load libxt/1.3.0 -module load libxmu/1.2.1 -module load libxpm/3.5.17 -module load libxaw/1.0.16 -module load udunits/2.2.28 -module load ncview/2.1.9 -module load netcdf-cxx4/4.3.1 -module load json/3.11.3 -#module load crtm/v2.4_jedi -module load prod_util/2.0.14 -module load grib-util/1.4.0 +#TODO load python virtual env +module load python-venv/1.0 module load py-numpy/1.26.4 -module load bufr/12.1.0 - module load py-markupsafe/2.1.3 module load py-jinja2/3.1.4 module load py-cftime/1.0.3.4 @@ -100,7 +69,6 @@ module load py-setuptools/63.4.3 module load py-pycodestyle/2.11.0 module load py-pyyaml/6.0.2 module load py-scipy/1.14.1 - module load py-setuptools/63.4.3 module load py-tzdata/2023.3 module load py-pytz/2023.3 @@ -125,6 +93,7 @@ module list ############################################################# export cyc=%CYC% export cycle=t%CYC%z +export EXPDIR=%EXPDIR% if [[ -d "${HOMEgcafs}/sorc/wxflow/src" ]]; then PYTHONPATH="${HOMEgcafs}/sorc/wxflow/src${PYTHONPATH:+:${PYTHONPATH}}" diff --git a/ecf/scripts/gcdas/chem/analysis/jgcdas_aerosol_analysis_generate_bmatrix.ecf b/ecf/scripts/gcdas/chem/analysis/jgcdas_aerosol_analysis_generate_bmatrix.ecf index 2068b6979a5..41e41859b8f 100755 --- a/ecf/scripts/gcdas/chem/analysis/jgcdas_aerosol_analysis_generate_bmatrix.ecf +++ b/ecf/scripts/gcdas/chem/analysis/jgcdas_aerosol_analysis_generate_bmatrix.ecf @@ -23,7 +23,7 @@ export RUN=%RUN% #TODO: Remove LMOD_TMOD_FIND_FIRST line when spack-stack on WCOSS2 export LMOD_TMOD_FIND_FIRST=yes # TODO: Add path to GDASApp libraries and cray-mpich as temporary patches -export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${HOMEgfs}/sorc/gdas.cd/build/lib" +export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${HOMEgcafs}/sorc/gdas.cd/build/lib" # TODO: Remove LD_LIBRARY_PATH line as soon as permanent solution is available export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/cray/pe/mpich/8.1.29/ofi/intel/2022.1/lib" @@ -32,12 +32,9 @@ module use /apps/ops/test/spack-stack-nco-1.9/modulefiles/Core module load stack-oneapi/2024.2.1 module load stack-cray-mpich/8.1.29 module load stack-python/3.11.7 -module load cmake/3.27.9 module load craype/2.7.17 module load cray-pals/1.3.2 -module load git/2.47.0 -module load git-lfs/3.5.1 module load zstd/1.5.6 module load pigz/2.8 @@ -51,10 +48,7 @@ module load nccmp/1.9.0.1 module load netcdf-fortran/4.6.1 module load nco/5.2.4 module load parallelio/2.6.2 -module load wget/1.21.1 module load boost/1.84.0 -module load ecbuild/3.7.2 -module load eccodes/2.33.0 module load eigen/3.4.0 module load openblas/0.3.24 module load eckit/1.28.3 @@ -70,20 +64,13 @@ module load sp/2.5.0 module load ip/5.1.0 module load gsl-lite/0.37.0 module load libjpeg/2.1.0 -module load krb5/1.21.1-1 -module load libtirpc/1.3.3 -module load hdf/4.2.15 -module load jedi-cmake/1.4.0 module load libpng/1.6.37 module load libxt/1.3.0 module load libxmu/1.2.1 module load libxpm/3.5.17 module load libxaw/1.0.16 module load udunits/2.2.28 -module load ncview/2.1.9 module load netcdf-cxx4/4.3.1 -module load json/3.11.3 -#module load crtm/v2.4_jedi module load prod_util/2.0.14 module load grib-util/1.4.0 @@ -125,6 +112,7 @@ module list ############################################################# export cyc=%CYC% export cycle=t%CYC%z +export EXPDIR=%EXPDIR% if [[ -d "${HOMEgcafs}/sorc/wxflow/src" ]]; then PYTHONPATH="${HOMEgcafs}/sorc/wxflow/src${PYTHONPATH:+:${PYTHONPATH}}" diff --git a/ecf/scripts/gcdas/chem/analysis/jgcdas_aerosol_analysis_initialize.ecf b/ecf/scripts/gcdas/chem/analysis/jgcdas_aerosol_analysis_initialize.ecf index c230dc231c4..4c45659bff7 100755 --- a/ecf/scripts/gcdas/chem/analysis/jgcdas_aerosol_analysis_initialize.ecf +++ b/ecf/scripts/gcdas/chem/analysis/jgcdas_aerosol_analysis_initialize.ecf @@ -23,7 +23,7 @@ export RUN=%RUN% #TODO: Remove LMOD_TMOD_FIND_FIRST line when spack-stack on WCOSS2 export LMOD_TMOD_FIND_FIRST=yes # TODO: Add path to GDASApp libraries and cray-mpich as temporary patches -export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${HOMEgfs}/sorc/gdas.cd/build/lib" +export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${HOMEgcafs}/sorc/gdas.cd/build/lib" # TODO: Remove LD_LIBRARY_PATH line as soon as permanent solution is available export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/cray/pe/mpich/8.1.29/ofi/intel/2022.1/lib" @@ -32,13 +32,9 @@ module use /apps/ops/test/spack-stack-nco-1.9/modulefiles/Core module load stack-oneapi/2024.2.1 module load stack-cray-mpich/8.1.29 module load stack-python/3.11.7 -module load cmake/3.27.9 module load craype/2.7.17 module load cray-pals/1.3.2 -module load git/2.47.0 -module load git-lfs/3.5.1 - module load zstd/1.5.6 module load pigz/2.8 module load tar/1.34 @@ -51,45 +47,18 @@ module load nccmp/1.9.0.1 module load netcdf-fortran/4.6.1 module load nco/5.2.4 module load parallelio/2.6.2 -module load wget/1.21.1 module load boost/1.84.0 -module load ecbuild/3.7.2 -module load eccodes/2.33.0 module load eigen/3.4.0 module load openblas/0.3.24 module load eckit/1.28.3 module load fckit/0.13.2 module load fms/2024.02 -module load python-venv/1.0 -module load py-pyyaml/6.0.2 module load intel-oneapi-runtime/2024.2.1 module load glibc/2.31 -module load esmf/8.8.0 -module load atlas/0.40.0 -module load sp/2.5.0 -module load ip/5.1.0 -module load gsl-lite/0.37.0 -module load libjpeg/2.1.0 -module load krb5/1.21.1-1 -module load libtirpc/1.3.3 -module load hdf/4.2.15 -module load jedi-cmake/1.4.0 -module load libpng/1.6.37 -module load libxt/1.3.0 -module load libxmu/1.2.1 -module load libxpm/3.5.17 -module load libxaw/1.0.16 -module load udunits/2.2.28 -module load ncview/2.1.9 -module load netcdf-cxx4/4.3.1 -module load json/3.11.3 -#module load crtm/v2.4_jedi -module load prod_util/2.0.14 -module load grib-util/1.4.0 +#TODO load python virtual env +module load python-venv/1.0 module load py-numpy/1.26.4 -module load bufr/12.1.0 - module load py-markupsafe/2.1.3 module load py-jinja2/3.1.4 module load py-cftime/1.0.3.4 @@ -100,7 +69,6 @@ module load py-setuptools/63.4.3 module load py-pycodestyle/2.11.0 module load py-pyyaml/6.0.2 module load py-scipy/1.14.1 - module load py-setuptools/63.4.3 module load py-tzdata/2023.3 module load py-pytz/2023.3 @@ -125,6 +93,7 @@ module list ############################################################# export cyc=%CYC% export cycle=t%CYC%z +export EXPDIR=%EXPDIR% if [[ -d "${HOMEgcafs}/sorc/wxflow/src" ]]; then PYTHONPATH="${HOMEgcafs}/sorc/wxflow/src${PYTHONPATH:+:${PYTHONPATH}}" @@ -132,6 +101,9 @@ fi PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgcafs}/ush/python" export PYTHONPATH +#TODO fix below line properly +export CRTM_FIX="/lfs/h2/emc/da/noscrub/emc.da/GDASApp/fix/crtm/2.4.0" + ############################################################ # CALL executable job script here ############################################################ diff --git a/ecf/scripts/gcdas/chem/analysis/jgcdas_aerosol_analysis_stats.ecf b/ecf/scripts/gcdas/chem/analysis/jgcdas_aerosol_analysis_stats.ecf index 196bd25311f..8dad9eba8eb 100755 --- a/ecf/scripts/gcdas/chem/analysis/jgcdas_aerosol_analysis_stats.ecf +++ b/ecf/scripts/gcdas/chem/analysis/jgcdas_aerosol_analysis_stats.ecf @@ -23,7 +23,7 @@ export RUN=%RUN% #TODO: Remove LMOD_TMOD_FIND_FIRST line when spack-stack on WCOSS2 export LMOD_TMOD_FIND_FIRST=yes # TODO: Add path to GDASApp libraries and cray-mpich as temporary patches -export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${HOMEgfs}/sorc/gdas.cd/build/lib" +export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${HOMEgcafs}/sorc/gdas.cd/build/lib" # TODO: Remove LD_LIBRARY_PATH line as soon as permanent solution is available export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/cray/pe/mpich/8.1.29/ofi/intel/2022.1/lib" @@ -32,13 +32,9 @@ module use /apps/ops/test/spack-stack-nco-1.9/modulefiles/Core module load stack-oneapi/2024.2.1 module load stack-cray-mpich/8.1.29 module load stack-python/3.11.7 -module load cmake/3.27.9 module load craype/2.7.17 module load cray-pals/1.3.2 -module load git/2.47.0 -module load git-lfs/3.5.1 - module load zstd/1.5.6 module load pigz/2.8 module load tar/1.34 @@ -51,45 +47,18 @@ module load nccmp/1.9.0.1 module load netcdf-fortran/4.6.1 module load nco/5.2.4 module load parallelio/2.6.2 -module load wget/1.21.1 module load boost/1.84.0 -module load ecbuild/3.7.2 -module load eccodes/2.33.0 module load eigen/3.4.0 module load openblas/0.3.24 module load eckit/1.28.3 module load fckit/0.13.2 module load fms/2024.02 -module load python-venv/1.0 -module load py-pyyaml/6.0.2 module load intel-oneapi-runtime/2024.2.1 module load glibc/2.31 -module load esmf/8.8.0 -module load atlas/0.40.0 -module load sp/2.5.0 -module load ip/5.1.0 -module load gsl-lite/0.37.0 -module load libjpeg/2.1.0 -module load krb5/1.21.1-1 -module load libtirpc/1.3.3 -module load hdf/4.2.15 -module load jedi-cmake/1.4.0 -module load libpng/1.6.37 -module load libxt/1.3.0 -module load libxmu/1.2.1 -module load libxpm/3.5.17 -module load libxaw/1.0.16 -module load udunits/2.2.28 -module load ncview/2.1.9 -module load netcdf-cxx4/4.3.1 -module load json/3.11.3 -#module load crtm/v2.4_jedi -module load prod_util/2.0.14 -module load grib-util/1.4.0 +#TODO load python virtual env +module load python-venv/1.0 module load py-numpy/1.26.4 -module load bufr/12.1.0 - module load py-markupsafe/2.1.3 module load py-jinja2/3.1.4 module load py-cftime/1.0.3.4 @@ -100,7 +69,6 @@ module load py-setuptools/63.4.3 module load py-pycodestyle/2.11.0 module load py-pyyaml/6.0.2 module load py-scipy/1.14.1 - module load py-setuptools/63.4.3 module load py-tzdata/2023.3 module load py-pytz/2023.3 @@ -125,6 +93,7 @@ module list ############################################################# export cyc=%CYC% export cycle=t%CYC%z +export EXPDIR=%EXPDIR% if [[ -d "${HOMEgcafs}/sorc/wxflow/src" ]]; then PYTHONPATH="${HOMEgcafs}/sorc/wxflow/src${PYTHONPATH:+:${PYTHONPATH}}" @@ -135,7 +104,7 @@ export PYTHONPATH ############################################################ # CALL executable job script here ############################################################ -${HOMEgcafs}/jobs/JGLOBAL_ANALYSIS_STATS +${HOMEgcafs}/jobs/JGCDAS_AERO_ANALYSIS_STATS if [ $? -ne 0 ]; then ecflow_client --msg="***JOB ${ECF_NAME} ERROR RUNNING J-SCRIPT ***" diff --git a/ecf/scripts/gcdas/chem/analysis/jgcdas_aerosol_analysis_variational.ecf b/ecf/scripts/gcdas/chem/analysis/jgcdas_aerosol_analysis_variational.ecf index fa3e3121176..f70c85a41c2 100755 --- a/ecf/scripts/gcdas/chem/analysis/jgcdas_aerosol_analysis_variational.ecf +++ b/ecf/scripts/gcdas/chem/analysis/jgcdas_aerosol_analysis_variational.ecf @@ -23,7 +23,7 @@ export RUN=%RUN% #TODO: Remove LMOD_TMOD_FIND_FIRST line when spack-stack on WCOSS2 export LMOD_TMOD_FIND_FIRST=yes # TODO: Add path to GDASApp libraries and cray-mpich as temporary patches -export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${HOMEgfs}/sorc/gdas.cd/build/lib" +export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${HOMEgcafs}/sorc/gdas.cd/build/lib" # TODO: Remove LD_LIBRARY_PATH line as soon as permanent solution is available export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/cray/pe/mpich/8.1.29/ofi/intel/2022.1/lib" @@ -32,12 +32,9 @@ module use /apps/ops/test/spack-stack-nco-1.9/modulefiles/Core module load stack-oneapi/2024.2.1 module load stack-cray-mpich/8.1.29 module load stack-python/3.11.7 -module load cmake/3.27.9 module load craype/2.7.17 module load cray-pals/1.3.2 -module load git/2.47.0 -module load git-lfs/3.5.1 module load zstd/1.5.6 module load pigz/2.8 @@ -51,10 +48,7 @@ module load nccmp/1.9.0.1 module load netcdf-fortran/4.6.1 module load nco/5.2.4 module load parallelio/2.6.2 -module load wget/1.21.1 module load boost/1.84.0 -module load ecbuild/3.7.2 -module load eccodes/2.33.0 module load eigen/3.4.0 module load openblas/0.3.24 module load eckit/1.28.3 @@ -70,20 +64,13 @@ module load sp/2.5.0 module load ip/5.1.0 module load gsl-lite/0.37.0 module load libjpeg/2.1.0 -module load krb5/1.21.1-1 -module load libtirpc/1.3.3 -module load hdf/4.2.15 -module load jedi-cmake/1.4.0 module load libpng/1.6.37 module load libxt/1.3.0 module load libxmu/1.2.1 module load libxpm/3.5.17 module load libxaw/1.0.16 module load udunits/2.2.28 -module load ncview/2.1.9 module load netcdf-cxx4/4.3.1 -module load json/3.11.3 -#module load crtm/v2.4_jedi module load prod_util/2.0.14 module load grib-util/1.4.0 @@ -125,6 +112,7 @@ module list ############################################################# export cyc=%CYC% export cycle=t%CYC%z +export EXPDIR=%EXPDIR% if [[ -d "${HOMEgcafs}/sorc/wxflow/src" ]]; then PYTHONPATH="${HOMEgcafs}/sorc/wxflow/src${PYTHONPATH:+:${PYTHONPATH}}" diff --git a/ecf/scripts/gcdas/chem/analysis/jgcdas_prepare_obs.ecf b/ecf/scripts/gcdas/chem/analysis/jgcdas_prepare_obs.ecf index 552e2a9e533..a5560c4b879 100755 --- a/ecf/scripts/gcdas/chem/analysis/jgcdas_prepare_obs.ecf +++ b/ecf/scripts/gcdas/chem/analysis/jgcdas_prepare_obs.ecf @@ -23,7 +23,7 @@ export RUN=%RUN% #TODO: Remove LMOD_TMOD_FIND_FIRST line when spack-stack on WCOSS2 export LMOD_TMOD_FIND_FIRST=yes # TODO: Add path to GDASApp libraries and cray-mpich as temporary patches -export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${HOMEgfs}/sorc/gdas.cd/build/lib" +export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${HOMEgcafs}/sorc/gcdas.cd/build/lib" # TODO: Remove LD_LIBRARY_PATH line as soon as permanent solution is available export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/cray/pe/mpich/8.1.29/ofi/intel/2022.1/lib" @@ -32,64 +32,25 @@ module use /apps/ops/test/spack-stack-nco-1.9/modulefiles/Core module load stack-oneapi/2024.2.1 module load stack-cray-mpich/8.1.29 module load stack-python/3.11.7 -module load cmake/3.27.9 module load craype/2.7.17 module load cray-pals/1.3.2 +module load prod_util/2.0.14 -module load git/2.47.0 -module load git-lfs/3.5.1 - -module load zstd/1.5.6 -module load pigz/2.8 -module load tar/1.34 -module load gettext/0.22.5 -module load curl/8.10.1 module load hdf5/1.14.3 module load parallel-netcdf/1.12.3 module load netcdf-c/4.9.2 -module load nccmp/1.9.0.1 -module load netcdf-fortran/4.6.1 -module load nco/5.2.4 -module load parallelio/2.6.2 -module load wget/1.21.1 module load boost/1.84.0 -module load ecbuild/3.7.2 -module load eccodes/2.33.0 module load eigen/3.4.0 module load openblas/0.3.24 module load eckit/1.28.3 module load fckit/0.13.2 -module load fms/2024.02 -module load python-venv/1.0 -module load py-pyyaml/6.0.2 module load intel-oneapi-runtime/2024.2.1 -module load glibc/2.31 -module load esmf/8.8.0 -module load atlas/0.40.0 -module load sp/2.5.0 -module load ip/5.1.0 -module load gsl-lite/0.37.0 -module load libjpeg/2.1.0 -module load krb5/1.21.1-1 -module load libtirpc/1.3.3 -module load hdf/4.2.15 -module load jedi-cmake/1.4.0 -module load libpng/1.6.37 -module load libxt/1.3.0 -module load libxmu/1.2.1 -module load libxpm/3.5.17 -module load libxaw/1.0.16 module load udunits/2.2.28 -module load ncview/2.1.9 module load netcdf-cxx4/4.3.1 -module load json/3.11.3 -#module load crtm/v2.4_jedi -module load prod_util/2.0.14 -module load grib-util/1.4.0 +#TODO load python virtual env +module load python-venv/1.0 module load py-numpy/1.26.4 -module load bufr/12.1.0 - module load py-markupsafe/2.1.3 module load py-jinja2/3.1.4 module load py-cftime/1.0.3.4 @@ -100,7 +61,6 @@ module load py-setuptools/63.4.3 module load py-pycodestyle/2.11.0 module load py-pyyaml/6.0.2 module load py-scipy/1.14.1 - module load py-setuptools/63.4.3 module load py-tzdata/2023.3 module load py-pytz/2023.3 @@ -114,9 +74,9 @@ module load py-pip/23.1.2 module load py-click/8.1.7 module load py-wheel/0.41.2 -ncdump=$(command -v ncdump) -NETCDF=$(echo "${ncdump}" | cut -d " " -f 3) -export NETCDF +#ncdump=$(command -v ncdump) +#NETCDF=$(echo "${ncdump}" | cut -d " " -f 3) +#export NETCDF module list @@ -125,6 +85,7 @@ module list ############################################################# export cyc=%CYC% export cycle=t%CYC%z +export EXPDIR=%EXPDIR% if [[ -d "${HOMEgcafs}/sorc/wxflow/src" ]]; then PYTHONPATH="${HOMEgcafs}/sorc/wxflow/src${PYTHONPATH:+:${PYTHONPATH}}" diff --git a/ecf/scripts/gcdas/chem/jgcdas_aerosol_analysis_post.ecf b/ecf/scripts/gcdas/chem/jgcdas_aerosol_analysis_post.ecf index b07eaeb4cb7..0fc458c84cc 100644 --- a/ecf/scripts/gcdas/chem/jgcdas_aerosol_analysis_post.ecf +++ b/ecf/scripts/gcdas/chem/jgcdas_aerosol_analysis_post.ecf @@ -45,6 +45,7 @@ export FHRGRP=%FHRGRP% export FHRLST=%FHRLST% export cyc=%CYC% export cycle=t%CYC%z +export EXPDIR=%EXPDIR% export USE_CFP=YES export g2tmpl_ver=v${g2tmpl_ver} diff --git a/ecf/scripts/gcdas/chem/jgcdas_prep_emissions.ecf b/ecf/scripts/gcdas/chem/jgcdas_prep_emissions.ecf index 1a6f363a51b..e0a1bd3a80a 100755 --- a/ecf/scripts/gcdas/chem/jgcdas_prep_emissions.ecf +++ b/ecf/scripts/gcdas/chem/jgcdas_prep_emissions.ecf @@ -62,6 +62,7 @@ module list ############################################################# export cyc=%CYC% export cycle=t%CYC%z +export EXPDIR=%EXPDIR% if [[ -d "${HOMEgcafs}/sorc/wxflow/src" ]]; then PYTHONPATH="${HOMEgcafs}/sorc/wxflow/src${PYTHONPATH:+:${PYTHONPATH}}" @@ -72,7 +73,7 @@ export PYTHONPATH ############################################################ # CALL executable job script here ############################################################ -${HOMEgcafs}/jobs/JGLOBAL_PREP_EMISSIONS +${HOMEgcafs}/jobs/JGCDAS_PREP_EMISSIONS if [ $? -ne 0 ]; then ecflow_client --msg="***JOB ${ECF_NAME} ERROR RUNNING J-SCRIPT ***" diff --git a/ecf/scripts/gcdas/jgcdas_forecast.ecf b/ecf/scripts/gcdas/jgcdas_forecast.ecf index 56506073fc1..9ce423063b9 100755 --- a/ecf/scripts/gcdas/jgcdas_forecast.ecf +++ b/ecf/scripts/gcdas/jgcdas_forecast.ecf @@ -60,6 +60,7 @@ module list ############################################################# export cyc=%CYC% export cycle=t%CYC%z +export EXPDIR=%EXPDIR% if [[ -d "${HOMEgcafs}/sorc/wxflow/src" ]]; then PYTHONPATH="${HOMEgcafs}/sorc/wxflow/src${PYTHONPATH:+:${PYTHONPATH}}" diff --git a/parm/chem/nc2ioda.yaml.j2 b/parm/chem/nc2ioda.yaml.j2 new file mode 100644 index 00000000000..7dc9db25a05 --- /dev/null +++ b/parm/chem/nc2ioda.yaml.j2 @@ -0,0 +1,41 @@ +provider: {{ provider }} +window begin: {{ window_begin | to_isotime }} +window end: {{ window_end | to_isotime }} +output file: {{ output_file }} + +# Superobing configuration +{% if binning_stride is defined%} +binning: + stride: {{ binning_stride }} + min number of obs: {{ binning_min_number_of_obs }} +{% endif %} + {% if binning_cressman_radius is defined%} + cressman radius: {{ binning_cressman_radius }} + {% endif %} + +# Bounds for rudimentary QC +{% if bounds_min is defined and bounds_max is defined %} +bounds: + min: {{ bounds_min }} + max: {{ bounds_max }} +{% endif %} + +# Ocean basin flags +{% if ocean_basin is defined %} +ocean basin: {{ ocean_basin }} +{% endif %} +input files: {{ input_files }} + +# Error adjustment for re-dated observations +{% if error_ratio is defined %} +error ratio: {{ error_ratio }} +{% endif %} + +# Used in JRR_AOD +{% if provider == "VIIRSAOD"%} +variable: aerosolOpticalDepth +thinning: + threshold: {{ thinning_threshold }} +channel: 4 +preqc: {{ preqc }} +{% endif %} diff --git a/parm/chem/prepare_obs.yaml b/parm/chem/prepare_obs.yaml new file mode 100644 index 00000000000..38ca6d29a37 --- /dev/null +++ b/parm/chem/prepare_obs.yaml @@ -0,0 +1,9 @@ +aoddump: + provider: VIIRSAOD + platforms: ['npp', 'j01', 'n21'] # note j01==n20 + binning_stride: 33 + binning_min_number_of_obs: 32 + binning_cressman_radius: 25 + thinning_threshold: 0 + channel: 4 + preqc: 0 diff --git a/parm/config/gcafs/config.aero b/parm/config/gcafs/config.aero new file mode 100644 index 00000000000..cd30b84a4b3 --- /dev/null +++ b/parm/config/gcafs/config.aero @@ -0,0 +1,232 @@ +#! /usr/bin/env bash + +#================================================================================ +# UFS-Aerosols settings +# This configuration file sets up environment variables for aerosol modeling in the UFS (Unified Forecast System) Aerosols component. +# It configures aerosol inputs, diagnostics, emissions, and the NEXUS emissions preprocessor for GCAFS (Global Coupled Aerosol Forecast System). +# Used in GFS (Global Forecast System) workflows for initializing and running aerosol simulations in FV3 (Finite-Volume Cubed-Sphere) dynamical core. +#================================================================================ +echo "BEGIN: config.aero" + +#================================================================================ +# 1. Aerosol settings +#================================================================================ +# General settings for aerosol tracers, diagnostics, and scavenging in the GOCART (Goddard Chemistry Aerosol Radiation and Transport) model. +# These are used in the atmospheric model to handle aerosol transport, chemistry, and interaction with radiation/cloud processes. + +# Base directory for aerosol input data files (e.g., initial conditions, climatologies). +# This path is mounted or staged in the workflow and referenced by the model for reading aerosol fields. +#--------------------------------------------------------------------------------------------------- +export AERO_INPUTS_DIR="${HOMEgcafs}/fix/chem/Emission_data" + +#------------------------------------------------- +# Diag Table and Field Table for GOCART aerosols +#------------------------------------------------- + +# Configuration files defining diagnostic outputs and field registrations for aerosol variables in GOCART. +# diag_table.aero: Specifies which aerosol fields to output and at what frequency (used by FMS diagnostics). +# field_table.aero: Registers prognostic/diagnostic tracers with the FV3 dynamical core (e.g., for advection, diffusion). +#--------------------------------------------------------------------------------------------------- +export AERO_DIAG_TABLE="${PARMgcafs}/ufs/fv3/diag_table.aero" +export AERO_FIELD_TABLE="${PARMgcafs}/ufs/fv3/field_table.aero" + +#================================================================================ +# Aerosol configuration +#================================================================================ + +# Directory containing GOCART-specific namelists, parameters, and runtime configs (e.g., namelist.aero). +# Loaded during model initialization to set aerosol scheme parameters like time steps, vertical levels. +#--------------------------------------------------------------------------------------------------- +export AERO_CONFIG_DIR="${PARMgcafs}/ufs/gocart" + +# Aerosol convective scavenging factors (list of string array elements) +# Element syntax: ':'. Use = * to set default factor for all aerosol tracers +# Scavenging factors represent the fraction of aerosol removed by convective precipitation (wet deposition). +# Used in the convection scheme (e.g., SAS or NF_CONV) to compute in-cloud scavenging rates for each tracer. +# * = default for unspecified tracers; specific gases like SO2 have lower factors due to solubility. +# Scavenging factors are set to 0 (no scavenging) if unset +#--------------------------------------------------------------------------------------------------- +export fscav_aero="'*:0.3','so2:0.0','msa:0.0','dms:0.0','nh3:0.4','nh4:0.6','bc1:0.6','bc2:0.6','oc1:0.4','oc2:0.4','dust1:0.6','dust2:0.6', 'dust3:0.6','dust4:0.6','dust5:0.6','seas1:0.5','seas2:0.5','seas3:0.5','seas4:0.5','seas5:0.5'" + +# Number of diagnostic aerosol tracers (default: 0) +# Specifies how many additional diagnostic (non-prognostic) aerosol tracers to include in the model output. +# Used in GOCART to control verbosity of diagnostics; higher values add more fields for analysis/post-processing. +#--------------------------------------------------------------------------------------------------- +export dnats_aero=2 + +#================================================================================ +# 2. Aerosol emissions settings +#================================================================================ +# Configuration for surface emissions of aerosols and precursors (e.g., from fires, anthropogenic sources). +# These drive the source terms in the GOCART continuity equation for each tracer. +# Biomass burning emission dataset. Choose from: gbbepx, qfed, none +# Dataset for wildfire and biomass burning emissions (e.g., black/organic carbon, CO). +# qfed: Quick Fire Emission Dataset (near-real-time, version specified below). +# gbbepx: Global Biomass Burning Emissions Product (alternative). +# none: Disable fire emissions. +# Used in prep_emissions scripts to fetch/interpolate data to model grid. +#--------------------------------------------------------------------------------------------------- +export AERO_EMIS_FIRE="gbbepx" +# Version of the selected fire emissions dataset (e.g., for QFEDv2.5, version 061). +# Determines which historical or NRT files to load from input directories. +export AERO_EMIS_FIRE_VERSION="004" + +# Flag to enable historical (climatological) fire emissions instead of NRT for testing/spin-up. +# When true, uses fixed-year data; false uses real-time from FIRE_EMIS_NRT_DIR. +# Path to near-real-time (NRT) fire emissions data, updated daily (e.g., from satellites like MODIS). +# On WCOSS2, points to DCOM (Data Communication) root for operational runs; empty for testing. +# Processed by scripts like exglobal_prep_emissions.py to generate input files for GOCART. +#--------------------------------------------------------------------------------------------------- +export AERO_EMIS_FIRE_HIST=0 # Use historical fire emissions | 1 = true 0 = false + +#--------------------------------------------------------------------------------------------------- +export FIRE_EMIS_NRT_DIR="${DCOMROOT}" #TODO: set to DCOM for WCOSS2 "${DCOMROOT}/YYYYMMDD/firewx" # Directory containing NRT fire emissions +export FIRE_EMIS_DIR="${HOMEgcafs}/fix/chem/Emission_data/fires_data/GBBEPx/v4" # Directory containing historical fire emissions + + +#=============================================================================== +# 3. NEXUS settings +#=============================================================================== +# NEXUS (Next-generation Emissions eXchange Utility System) is a preprocessor for anthropogenic/biogenic emissions. +# Generates time-varying, gridded emission inputs for GOCART from inventories like CEDS, HTAP, CAMS. +# Runs offline before the forecast, outputting netCDF files read by the model via AERO_INPUTS_DIR. +# NEXUS aerosol emissions dataset. Choose from: gocart, none +# Specifies the emission species set for NEXUS processing (gocart for GOCART-compatible tracers like SO2, BC, OC, dust). +# none: Skip NEXUS entirely, use other emission sources or zero emissions. + +# NEXUS configuration set +#------------------------- +export NEXUS_CONFIG="gocart" # Options: gocart, none + +# Runtime choice of NEXUS config variant; defaults to gocart for standard aerosol tracers. +# Overrides via Jinja2 templating in workflow (e.g., for different chemistry schemes). +#--------------------------------------------------------------------------------------------------- +export NEXUS_CONFIG_DIR="${PARMgcafs}/chem/nexus/${NEXUS_CONFIG}" # Directory containing NEXUS configuration files + +# NEXUS Inputs +#--------------- +# TODO: when this is merged this will point to AERO_INPUTS_DIR for operations +# export NEXUS_INPUT_DIR="${AERO_INPUTS_DIR}/nexus" +# Directory for static/dynamic input data used by NEXUS (e.g., emission inventories, masks, meteo fields). +# Currently hardcoded for development; will use shared AERO_INPUTS_DIR in production for consistency. +# Specific path for GCAFS external data on this filesystem. +# Contains emission datasets (e.g., CEDS2019/2024, HTAPv2, CAMS) processed by NEXUS. +#--------------------------------------------------------------------------------------------------- +export NEXUS_INPUT_DIR="${HOMEgcafs}/fix/chem/Emission_data/nexus" + + + +#-------------------------- +# NEXUS Time Step (seconds) +#-------------------------- +# Temporal resolution for emission interpolation in NEXUS (e.g., hourly outputs). +# Must align with model coupling time; used in HEMCO time management for diurnal/seasonal scaling. +# 3600s = 1 hour; adjustable for finer/coarser emission updates (e.g., 1800s for sub-hourly). +#--------------------------------------------------------------------------------------------------- +export NEXUS_TSTEP="3600" # Default NEXUS time step in seconds + +#------------------ +# NEXUS Grid +#------------------ +# Defines the emission grid for NEXUS processing (0.5x0.5 degree global lat-lon). +# Emissions are interpolated from this grid to the FV3 cubed-sphere grid during prep. +# Number of longitude points (1440 for 0.25-degree resolution; here 1440 ~0.25deg). +#----------------------------------------------------- +export NEXUS_NX="1440" + +# Number of latitude points (720 for 0.25-degree). +#-------------------------------------------------- +export NEXUS_NY="720" + +# Western boundary longitude (global coverage). +#------------------------------------------------- +export NEXUS_XMIN="-180.0" + +# Eastern boundary longitude. +#-------------------------------------------------- +export NEXUS_XMAX="180.0" + +# Southern boundary latitude. +#-------------------------------------------------- +export NEXUS_YMIN="-90.0" + +# Northern boundary latitude. +#--------------------------------------------------- +export NEXUS_YMAX="90.0" + +# Number of vertical levels (1 for surface emissions; higher for vertical profiles if needed). +#-------------------------------------------------- +export NEXUS_NZ="1" + +#------------------- +# NEXUS Config Files +#------------------- +# HEMCO (Harmonized Emissions Component) runtime configuration files used by NEXUS. +# These define species mappings, time scales, grid alignments, and diagnostic flags. + +# Grid definition file: Specifies emission grid (lat-lon bounds, resolution) and interpolation options to model grid. +export NEXUS_GRID_NAME="HEMCO_sa_Grid.rc" +# Time management file: Defines temporal patterns (diurnal, weekly, monthly) for scaling emissions. +export NEXUS_TIME_NAME="HEMCO_sa_Time.rc" +# Diagnostics file: Controls which emission fields to output for verification (e.g., total SO2, BC emissions). +export NEXUS_DIAG_NAME="HEMCO_sa_Diag.rc" +# Species mapping file: Links emission inventories to GOCART tracers (e.g., CEDS SO2 to model SO2). +export NEXUS_SPEC_NAME="HEMCO_sa_Spec.rc" +# Master config file: Orchestrates all HEMCO components, emission sources, and runtime flags for NEXUS. +export NEXUS_CONFIG_NAME="NEXUS_Config.rc" + +#------------------ +# NEXUS Diagnostics +#------------------ +# Settings for outputting NEXUS-processed emissions for model input and verification. +# Outputs are netCDF files with gridded, time-varying sources read by GOCART at each time step. + +# Base filename prefix for diagnostic output files (e.g., NEXUS_DIAG_YYYYMMDD.nc). +export NEXUS_DIAG_PREFIX="NEXUS_DIAG" +# Frequency of diagnostic emission outputs; Hourly for detailed analysis, coarser for storage efficiency. +export NEXUS_DIAG_FREQ="Hourly" # Options: Hourly, Daily, Monthly + +#------------------ +# NEXUS Logging +#------------------ +# Controls NEXUS execution logs for debugging and monitoring in the workflow. + +# Output log file for NEXUS run; captures errors, warnings, and processing summaries. +# Reviewed in post-processing or if emissions fail to generate. +export NEXUS_LOGFILE="NEXUS.log" + +#------------------ +# NEXUS Emissions +#------------------ +# Flags to enable/disable specific emission inventories processed by NEXUS. +# Multiple can be true for blended emissions; used in NEXUS_Config.rc to select sources. +# Emissions are scaled by region, sector (e.g., industry, transport), and time. + +# Flag for MEGAN (Model of Emissions of Gases and Aerosols from Nature) biogenic VOC/PM emissions. +# Currently disabled; future integration for isoprene, terpenes affecting secondary organic aerosols. +export NEXUS_DO_MEGAN=.false # TODO: Add MEGAN biogenic emissions in the furture + +# Enable Community Emissions Data System 2019 inventory for anthropogenic aerosols/gases (e.g., SO2, NOx, PM2.5). +# Global, gridded data for 1750-2019; used for historical and recent baseline emissions. +export NEXUS_DO_CEDS2019=.false. # Use CEDS2019 emissions + +# Enable newer CEDS 2024 update (if available); mutually exclusive with 2019 for consistency. +export NEXUS_DO_CEDS2024=.true. # Use CEDS2024 emissions + +# Hemispheric Transport of Air Pollution version 2: Regional anthropogenic emissions for Europe/Asia/N. America. +# Focuses on transboundary pollution; supplements CEDS for finer regional detail. +export NEXUS_DO_HTAPv2=.true. # Use HTAPv2 emissions + +# HTAP version 3 flag; disabled pending updates to datasets and NEXUS compatibility. +export NEXUS_DO_HTAPv3=.false. # TODO: Currently only uses HTAPv2 for this. + +# Copernicus Atmosphere Monitoring Service global reanalysis emissions. +# Alternative to CEDS/HTAP for consistent meteo-coupled emissions; disabled here. +export NEXUS_DO_CAMS=.false. # Use CAMS global emissions + +# CAMS temporal disaggregation (e.g., hourly profiles for CAMS data). +# Enables time-varying scaling when CAMS is active. +export NEXUS_DO_CAMSTEMPO=.true. # Use CAMS temporal emissions + +#================================================================================ +echo "END: config.aero" diff --git a/parm/config/gcafs/config.aeroanl b/parm/config/gcafs/config.aeroanl new file mode 100644 index 00000000000..445711389e5 --- /dev/null +++ b/parm/config/gcafs/config.aeroanl @@ -0,0 +1,32 @@ +#!/bin/bash -x + +########## config.aeroanl ########## +# configuration common to all aero analysis tasks + +echo "BEGIN: config.aeroanl" + +# define analysis resolution based on deterministic res +case ${CASE} in + "C1152" | "C768" | "C384") + CASE_ANL="C384" + ;; + "C192" | "C96" | "C48") + CASE_ANL=${CASE} + ;; + *) + echo "FATAL ERROR: Aerosol DA not supported at ${CASE} resolution" + exit 4 +esac +export CASE_ANL + +export STATICB_TYPE='diffusion' + +export TASK_CONFIG_YAML="${PARMgcafs}/gdas/aero/aero_det_config.yaml.j2" +export OBS_LIST_YAML="${PARMgcafs}/gdas/aero/aero_obs_list.yaml.j2" + +export io_layout_x="1" +export io_layout_y="1" + +export aero_bkg_times="3,6,9" + +echo "END: config.aeroanl" \ No newline at end of file diff --git a/parm/config/gcafs/config.aeroanlfinal b/parm/config/gcafs/config.aeroanlfinal new file mode 100644 index 00000000000..34e5d8f1164 --- /dev/null +++ b/parm/config/gcafs/config.aeroanlfinal @@ -0,0 +1,10 @@ +#!/bin/bash -x + +########## config.aeroanlfinal ########## +# Post Aero Analysis specific + +echo "BEGIN: config.aeroanlfinal" + +# Get task specific resources +source "${EXPDIR}/config.resources" aeroanlfinal +echo "END: config.aeroanlfinal" diff --git a/parm/config/gcafs/config.aeroanlgenb b/parm/config/gcafs/config.aeroanlgenb new file mode 100644 index 00000000000..2eb1a42130f --- /dev/null +++ b/parm/config/gcafs/config.aeroanlgenb @@ -0,0 +1,30 @@ +#!/bin/bash -x + +########## config.aeroanlgenb ########## +# Aerosol Variance specific + +echo "BEGIN: config.aeroanlgenb" + +# Get task specific resources +source "${EXPDIR}/config.resources" aeroanlgenb + +export TASK_CONFIG_YAML="${PARMgcafs}/gdas/aero/aero_bmat_config.yaml.j2" +export OBS_LIST_YAML="${PARMgcafs}/gdas/aero/aero_obs_list.yaml.j2" + +export aero_diffusion_iter=200 +export aero_diffusion_horiz_len=300e3 +export aero_diffusion_fixed_val=20.0 +export npx_clim_b=97 +export npy_clim_b=97 +export aero_diagb_weight=1.0 +export aero_staticb_rescaling_factor=2.0 +export aero_diagb_n_halo=4 +export aero_diagb_n_neighbors=16 +aero_diagb_smooth_horiz_iter=0 +export aero_diagb_smooth_vert_iter=0 +if [[ "${CASE_ANL}" == "C384" ]]; then + aero_diagb_smooth_horiz_iter=200 +fi +export aero_diagb_smooth_horiz_iter + +echo "END: config.aeroanlgenb" diff --git a/parm/config/gcafs/config.aeroanlinit b/parm/config/gcafs/config.aeroanlinit new file mode 100644 index 00000000000..7036d3d27b8 --- /dev/null +++ b/parm/config/gcafs/config.aeroanlinit @@ -0,0 +1,10 @@ +#!/bin/bash -x + +########## config.aeroanlinit ########## +# Pre Aero Analysis specific + +echo "BEGIN: config.aeroanlinit" + +# Get task specific resources +source "${EXPDIR}/config.resources" aeroanlinit +echo "END: config.aeroanlinit" diff --git a/parm/config/gcafs/config.aeroanlvar b/parm/config/gcafs/config.aeroanlvar new file mode 100644 index 00000000000..4282b6c840b --- /dev/null +++ b/parm/config/gcafs/config.aeroanlvar @@ -0,0 +1,11 @@ +#!/bin/bash -x + +########## config.aeroanlvar ########## +# Aerosol Analysis specific + +echo "BEGIN: config.aeroanlvar" + +# Get task specific resources +source "${EXPDIR}/config.resources" aeroanlvar + +echo "END: config.aeroanlvar" diff --git a/parm/config/gcafs/config.aerosol_init b/parm/config/gcafs/config.aerosol_init new file mode 100644 index 00000000000..d4787ea96ef --- /dev/null +++ b/parm/config/gcafs/config.aerosol_init @@ -0,0 +1,10 @@ +#! /usr/bin/env bash + +########## config.aerosol_init ########## + +echo "BEGIN: config.aerosol_init" + +# Get task specific resources +source "${EXPDIR}/config.resources" aerosol_init + +echo "END: config.aerosol_init" diff --git a/parm/config/gcafs/config.anal b/parm/config/gcafs/config.anal new file mode 100644 index 00000000000..67a69d9502a --- /dev/null +++ b/parm/config/gcafs/config.anal @@ -0,0 +1,176 @@ +#! /usr/bin/env bash + +########## config.anal ########## +# Analysis specific + +echo "BEGIN: config.anal" + +# Get task specific resources +source "${EXPDIR}/config.resources" anal + +if [[ ${DONST} = "YES" ]]; then + source "${EXPDIR}/config.nsst" +fi + +if [[ "${RUN}" == "gfs" ]] ; then + export USE_RADSTAT="NO" # This can be only used when bias correction is not-zero. + export GENDIAG="NO" + export SETUP='diag_rad=.false.,diag_pcp=.false.,diag_conv=.false.,diag_ozone=.false.,write_diag(3)=.false.,niter(2)=100,' + export DIAG_TARBALL="YES" +fi + +# Set parameters specific to L127 +if [[ ${LEVS} = "128" ]]; then + export GRIDOPTS="nlayers(63)=1,nlayers(64)=1," + export SETUP="gpstop=55,nsig_ext=45,${SETUP:-}" +fi + +# Set namelist option for LETKF +export lobsdiag_forenkf=".false." # anal does not need to write out jacobians + # set to .true. in config.eobs and config.eupd + +# Reduce number of iterations for testing mode +if [[ ${DO_TEST_MODE} = "YES" ]]; then + export SETUP="${SETUP:-}niter(1)=5,niter(2)=5," +fi + +# Do not process the following datasets +export GSNDBF=${GSNDBF:-/dev/null} +export AMSREBF=${AMSREBF:-/dev/null} +export SSMITBF=${SSMITBF:-/dev/null} +export AMSR2BF=${AMSR2BF:-/dev/null} + + +# Set default values for info files and observation error +# NOTE: Remember to set PRVT in config.prep as OBERROR is set below +export CONVINFO=${FIXgcafs}/gsi/global_convinfo.txt +export OZINFO=${FIXgcafs}/gsi/global_ozinfo.txt +export SATINFO=${FIXgcafs}/gsi/global_satinfo.txt +export OBERROR=${FIXgcafs}/gsi/prepobs_errtable.global + +# Use 2m diagnostic for screen level obs +export hofx_2m_sfcfile=".true." + +if [[ ${DO_GSISOILDA} = "YES" ]]; then + export reducedgrid=".false." # not possible for sfc analysis, Jeff Whitaker says it's not useful anyway + # NOTE: convinfo here will be over-written by date-specific files below. + export CONVINFO=${FIXgcafs}/gsi/global_convinfo_2mObs.txt + export ANAVINFO=${FIXgcafs}/gsi/global_anavinfo_soilanal.l127.txt +fi + +# Use experimental dumps in EMC GFS v16 parallels +if [[ ${RUN_ENVIR} == "emc" ]]; then + # Set info files and prepobs.errtable.global for GFS v16 retrospective parallels + if [[ "${PDY}${cyc}" -ge "2019021900" && "${PDY}${cyc}" -lt "2019110706" ]]; then + export CONVINFO=${FIXgcafs}/gsi/gfsv16_historical/global_convinfo.txt.2019021900 + export OBERROR=${FIXgcafs}/gsi/gfsv16_historical/prepobs_errtable.global.2019021900 + fi + + # Place GOES-15 AMVs in monitor, assimilate GOES-17 AMVs, assimilate KOMPSAT-5 gps + if [[ "${PDY}${cyc}" -ge "2019110706" && "${PDY}${cyc}" -lt "2020040718" ]]; then + export CONVINFO=${FIXgcafs}/gsi/gfsv16_historical/global_convinfo.txt.2019110706 + export OBERROR=${FIXgcafs}/gsi/gfsv16_historical/prepobs_errtable.global.2019110706 + fi + + # Assimilate 135 (T) & 235 (uv) Canadian AMDAR observations + if [[ "${PDY}${cyc}" -ge "2020040718" && "${PDY}${cyc}" -lt "2020052612" ]]; then + export CONVINFO=${FIXgcafs}/gsi/gfsv16_historical/global_convinfo.txt.2020040718 + export OBERROR=${FIXgcafs}/gsi/gfsv16_historical/prepobs_errtable.global.2020040718 + fi + + # Assimilate COSMIC-2 + if [[ "${PDY}${cyc}" -ge "2020052612" && "${PDY}${cyc}" -lt "2020082412" ]]; then + export CONVINFO=${FIXgcafs}/gsi/gfsv16_historical/global_convinfo.txt.2020052612 + export OBERROR=${FIXgcafs}/gsi/gfsv16_historical/prepobs_errtable.global.2020040718 + fi + + # Assimilate HDOB + if [[ "${PDY}${cyc}" -ge "2020082412" && "${PDY}${cyc}" -lt "2020091612" ]]; then + export CONVINFO=${FIXgcafs}/gsi/gfsv16_historical/global_convinfo.txt.2020082412 + fi + + # Assimilate Metop-C GNSSRO + if [[ "${PDY}${cyc}" -ge "2020091612" && "${PDY}${cyc}" -lt "2021031712" ]]; then + export CONVINFO=${FIXgcafs}/gsi/gfsv16_historical/global_convinfo.txt.2020091612 + fi + + # Assimilate DO-2 GeoOptics + if [[ "${PDY}${cyc}" -ge "2021031712" && "${PDY}${cyc}" -lt "2021091612" ]]; then + export CONVINFO=${FIXgcafs}/gsi/gfsv16_historical/global_convinfo.txt.2021031712 + fi + + # NOTE: + # As of 2021110312, gfsv16_historical/global_convinfo.txt.2021110312 is + # identical to ../global_convinfo.txt. Thus, the logic below is not + # needed at this time. + # Assimilate COSMIC-2 GPS + # if [[ "${PDY}${cyc}" -ge "2021110312" && "${PDY}${cyc}" -lt "YYYYMMDDHH" ]]; then + # export CONVINFO=${FIXgcafs}/gsi/gfsv16_historical/global_convinfo.txt.2021110312 + # fi + + # Turn off assmilation of OMPS during period of bad data + if [[ "${PDY}${cyc}" -ge "2020011600" && "${PDY}${cyc}" -lt "2020011806" ]]; then + export OZINFO=${FIXgcafs}/gsi/gfsv16_historical/global_ozinfo.txt.2020011600 + fi + + + # Set satinfo for start of GFS v16 parallels + if [[ "${PDY}${cyc}" -ge "2019021900" && "${PDY}${cyc}" -lt "2019110706" ]]; then + export SATINFO=${FIXgcafs}/gsi/gfsv16_historical/global_satinfo.txt.2019021900 + fi + + # Turn on assimilation of Metop-C AMSUA and MHS + if [[ "${PDY}${cyc}" -ge "2019110706" && "${PDY}${cyc}" -lt "2020022012" ]]; then + export SATINFO=${FIXgcafs}/gsi/gfsv16_historical/global_satinfo.txt.2019110706 + fi + + # Turn off assimilation of Metop-A MHS + if [[ "${PDY}${cyc}" -ge "2020022012" && "${PDY}${cyc}" -lt "2021052118" ]]; then + export SATINFO=${FIXgcafs}/gsi/gfsv16_historical/global_satinfo.txt.2020022012 + fi + + # Turn off assimilation of S-NPP CrIS + if [[ "${PDY}${cyc}" -ge "2021052118" && "${PDY}${cyc}" -lt "2021092206" ]]; then + export SATINFO=${FIXgcafs}/gsi/gfsv16_historical/global_satinfo.txt.2021052118 + fi + + # Turn off assimilation of MetOp-A IASI + if [[ "${PDY}${cyc}" -ge "2021092206" && "${PDY}${cyc}" -lt "2021102612" ]]; then + export SATINFO=${FIXgcafs}/gsi/gfsv16_historical/global_satinfo.txt.2021092206 + fi + + # NOTE: + # As of 2021110312, gfsv16_historical/global_satinfo.txt.2021110312 is + # identical to ../global_satinfo.txt. Thus, the logic below is not + # needed at this time + # + # Turn off assmilation of all Metop-A MHS + # if [[ "${PDY}${cyc}" -ge "2021110312" && "${PDY}${cyc}" -lt "YYYYMMDDHH" ]]; then + # export SATINFO=${FIXgcafs}/gsi/gfsv16_historical/global_satinfo.txt.2021110312 + # fi +fi + +if [[ ${USE_BUILD_GSINFO} == "YES" ]]; then + # these will be set based on date in scripts/exglobal_atmos_analysis.sh + export SATINFO="generate" + export CONVINFO="generate" + export OZINFO="generate" + # this can be over-ridden, for example to use NASA's netcdf ozone files use obs_input_reanl_nasa_ozone.txt. + export OBS_INPUT="${BUILD_GSINFO_DIR}/obs_input/obs_input_reanl_ncep_ozone.txt" + + # Use 2m observations when constructing conventional obs info file + # can use 2m obs without enabling soil DA by setting USE_2M_OBS=YES in config.base + export USE_2M_OBS="${USE_2M_OBS:-${DO_GSISOILDA}}" +else + export OBS_INPUT="${BUILD_GSINFO_DIR}/obs_input/obs_input_ops.txt" +fi +# path to "fixed" crtm coefficient files (needed for crtm < 3) +export HIRS_FIX="${BUILD_GSINFO_DIR}/hirs_fix" + +# Flag to turn on (.true.) or off (.false.) the infrared cloud and aerosol detection software +# for AIRS, CrIS, and IASI. Default is .false. +export AIRS_CADS=".false." +export CRIS_CADS=".false." +export IASI_CADS=".false." + +echo "END: config.anal" diff --git a/parm/config/gcafs/config.analcalc b/parm/config/gcafs/config.analcalc new file mode 100644 index 00000000000..d01c1653ef7 --- /dev/null +++ b/parm/config/gcafs/config.analcalc @@ -0,0 +1,11 @@ +#! /usr/bin/env bash + +########## config.analcalc ########## +# GFS post-anal specific (non-diag) + +echo "BEGIN: config.analcalc" + +# Get task specific resources +source "${EXPDIR}/config.resources" analcalc + +echo "END: config.analcalc" diff --git a/parm/config/gcafs/config.analcalc_fv3jedi b/parm/config/gcafs/config.analcalc_fv3jedi new file mode 100644 index 00000000000..e8307f41b26 --- /dev/null +++ b/parm/config/gcafs/config.analcalc_fv3jedi @@ -0,0 +1,25 @@ +#! /usr/bin/env bash + +########## config.analcalc_fv3jedi ########## +# Diagnostic amospheric analysis calculation specific + +# Ignore possible spelling error (nothing is misspelled) +# shellcheck disable=SC2153 + +echo "BEGIN: config.analcalc_fv3jedi" + +export layout_x_analcalc_fv3jedi=2 +export layout_y_analcalc_fv3jedi=2 + +# Get task specific resources +source "${EXPDIR}/config.resources" analcalc_fv3jedi + +export TASK_CONFIG_YAML="${PARMgcafs}/gdas/analcalc/analcalc_config.yaml.j2" + +if [[ ${DOHYBVAR} = "YES" ]]; then + export CASE_ANL=${CASE_ENS} +else + export CASE_ANL=${CASE} +fi + +echo "END: config.analcalc_fv3jedi" diff --git a/parm/config/gcafs/config.anlstat b/parm/config/gcafs/config.anlstat new file mode 100644 index 00000000000..99222d8b409 --- /dev/null +++ b/parm/config/gcafs/config.anlstat @@ -0,0 +1,13 @@ +#!/bin/bash -x + +########## config.anlstat ########## +# Analysis Stat + +echo "BEGIN: config.anlstat" + +# Get task specific resources +source "${EXPDIR}/config.resources" anlstat + +export TASK_CONFIG_YAML="${PARMgcafs}/gdas/anlstat/anlstat_config.yaml.j2" + +echo "END: config.anlstat" diff --git a/parm/config/gcafs/config.arch_tars b/parm/config/gcafs/config.arch_tars new file mode 100644 index 00000000000..422f4b7b564 --- /dev/null +++ b/parm/config/gcafs/config.arch_tars @@ -0,0 +1,29 @@ +#! /usr/bin/env bash + +########## config.arch_tars ########## +# Archive specific + +echo "BEGIN: config.arch_tars" + +# Get task specific resources +source "${EXPDIR}/config.resources" arch_tars + +export ARCH_CYC=00 # Archive data at this cycle for warm start and/or forecast-only capabilities +export ARCH_WARMICFREQ=4 # Archive frequency in days for warm start capability +export ARCH_FCSTICFREQ=1 # Archive frequency in days for gdas and gfs forecast-only capability +export ARCH_EXPDIR='YES' # Archive the EXPDIR configs, XML, and database +export ARCH_EXPDIR_FREQ=0 # How often to archive the EXPDIR in hours or 0 for first and last cycle only +export ARCH_HASHES='YES' # Archive the hashes of the GW and submodules and 'git status' for each; requires ARCH_EXPDIR +export ARCH_DIFFS='NO' # Archive the output of 'git diff' for the GW; requires ARCH_EXPDIR + +export ARCH_GAUSSIAN="YES" +export ARCH_GAUSSIAN_FHMAX=${FHMAX_GFS:-} +export ARCH_GAUSSIAN_FHINC=${FHOUT_GFS:-} + +# If we are running globus archiving, create tarballs in a temporary location +if [[ "${ARCHCOM_TO}" == "globus_hpss" ]]; then + export ATARDIR="${DATAROOT}/archive_rotdir/${RUN}" + export ARCHCOM_TO="local" +fi + +echo "END: config.arch_tars" diff --git a/parm/config/gcafs/config.arch_vrfy b/parm/config/gcafs/config.arch_vrfy new file mode 100644 index 00000000000..420a269056f --- /dev/null +++ b/parm/config/gcafs/config.arch_vrfy @@ -0,0 +1,11 @@ +#! /usr/bin/env bash + +########## config.arch_vrfy ########## +# Archive specific + +echo "BEGIN: config.arch_vrfy" + +# Get task specific resources +. "${EXPDIR}/config.resources" "arch_vrfy" + +echo "END: config.arch_vrfy" diff --git a/parm/config/gcafs/config.atmos_products b/parm/config/gcafs/config.atmos_products new file mode 100644 index 00000000000..2a8fb4cf179 --- /dev/null +++ b/parm/config/gcafs/config.atmos_products @@ -0,0 +1,36 @@ +#! /usr/bin/env bash + +########## config.atmos_products ########## +# atmosphere grib2 products specific + +echo "BEGIN: config.atmos_products" + +# Get task specific resources +source "${EXPDIR}/config.resources" atmos_products + +## Maximum number of rocoto tasks per member +export MAX_TASKS=25 + +if [[ "${RUN:-}" == "gcdas" ]]; then + export downset=2 + export FHOUT_PGBS=${FHOUT:-1} # Output frequency of supplemental gfs pgb file at 1.0 and 0.5 deg + export FLXGF="NO" # Create interpolated sflux.1p00 file + export WGNE="NO" # WGNE products are created for first FHMAX_WGNE forecast hours + export FHMAX_WGNE=0 +elif [[ "${RUN:-}" == "gcafs" ]]; then + export downset=2 + export FHOUT_PGBS=${FHOUT_GFS:-3} # Output frequency of supplemental gfs pgb file at 1.0 and 0.5 deg + export FLXGF="NO" # Create interpolated sflux.1p00 file + export WGNE="NO" # WGNE products are created for first FHMAX_WGNE forecast hours + export FHMAX_WGNE=180 +fi + +# paramlist files for the different forecast hours and downsets +export paramlista="${PARMgcafs}/product/gcafs.fFFF.paramlist.a.txt" +export paramlista_anl="${PARMgcafs}/product/gcafs.anl.paramlist.a.txt" +export paramlista_f000="${PARMgcafs}/product/gcafs.f000.paramlist.a.txt" +export paramlistb="${PARMgcafs}/product/gcafs.fFFF.paramlist.b.txt" +export paramlistb_anl="${PARMgcafs}/product/gcafs.anl.paramlist.b.txt" +export paramlistb_f000="${PARMgcafs}/product/gcafs.f000.paramlist.b.txt" + +echo "END: config.atmos_products" diff --git a/parm/config/gcafs/config.base b/parm/config/gcafs/config.base new file mode 100644 index 00000000000..647d262f9eb --- /dev/null +++ b/parm/config/gcafs/config.base @@ -0,0 +1,518 @@ +#! /usr/bin/env bash + +########## config.base ########## +# Common to all steps + +echo "BEGIN: config.base" + +# Machine environment +export machine="WCOSS2" + +# EMC parallel or NCO production +export RUN_ENVIR="emc" + +# Account, queue, etc. +export ACCOUNT="GFS-DEV" +export QUEUE="dev" +export QUEUE_SERVICE="dev_transfer" +export QUEUE_DTN="${QUEUE_SERVICE}" +export PARTITION_BATCH="{{ PARTITION_BATCH }}" +export PARTITION_SERVICE="" +export PARTITION_DTN="" +export RESERVATION="" +export CLUSTERS="" +export CLUSTERS_SERVICE="${CLUSTERS}" +export CLUSTERS_DTN="${CLUSTERS_SERVICE}" + +# Project to use in mass store: +export HPSS_PROJECT="emc-global" + +# Directories relative to installation areas: +#export HOMEgcafs="/lfs/h2/emc/da/noscrub/cory.r.martin/gcafs/global-workflow" +export EXECgcafs="${HOMEgcafs}/exec" +export FIXgcafs="${HOMEgcafs}/fix" +export PARMgcafs="${HOMEgcafs}/parm" +export SCRgcafs="${HOMEgcafs}/scripts" +export USHgcafs="${HOMEgcafs}/ush" + +export FIXam="${FIXgcafs}/am" +export FIXaer="${FIXgcafs}/aer" +export FIXcpl="${FIXgcafs}/cpl" +export FIXlut="${FIXgcafs}/lut" +export FIXcice="${FIXgcafs}/cice" +export FIXmom="${FIXgcafs}/mom6" +export FIXreg2grb2="${FIXgcafs}/reg2grb2" +export FIXgdas="${FIXgcafs}/gdas" + +######################################################################## + +# GLOBAL static environment parameters +export PACKAGEROOT="${PACKAGEROOT:-"/lfs/h1/ops/prod/packages"}" # TODO: set via prod_envir in Ops +#export COMROOT="/lfs/h2/emc/da/noscrub/cory.r.martin/gcafs/runtests2/COMROOT" # TODO: set via prod_envir in Ops +export COMINsyn="/lfs/h1/ops/prod/com/gfs/v16.3/syndat" +export DMPDIR="/lfs/h2/emc/dump/noscrub/dump" +export IODADIR="/lfs/h2/emc/dump/noscrub/dump_ioda" +export USE_IODADIR="YES" + +# Gempak from external models +# Default locations are to dummy locations for testing +export COMINecmwf="/lfs/h2/emc/global/noscrub/emc.global/data/external_gempak/ecmwf" +export COMINnam="/lfs/h2/emc/global/noscrub/emc.global/data/external_gempak/nam" +export COMINukmet="/lfs/h2/emc/global/noscrub/emc.global/data/external_gempak/ukmet" + +# USER specific paths +export HOMEDIR="/lfs/h2/emc/global/noscrub/${USER}" +export STMP="/lfs/h2/emc/stmp/${USER}" +export PTMP="/lfs/h2/emc/ptmp/${USER}" +export NOSCRUB="${HOMEDIR}" + +# Base directories for various builds +export BASE_GIT="/lfs/h2/emc/global/save/emc.global/git" + +# Base directory for staged data +export BASE_DATA="/lfs/h2/emc/global/noscrub/emc.global/data" + +# Toggle to turn on/off GFS downstream processing. +export DO_GOES="NO" # GOES products +export DO_BUFRSND="NO" # BUFR sounding products +export DO_GEMPAK="NO" # GEMPAK products +export DO_AWIPS="NO" # AWIPS products +export DO_NPOESS="NO" # NPOESS products +export DO_TRACKER="NO" # Hurricane track verification +export DO_GENESIS="NO" # Cyclone genesis verification +export DO_GENESIS_FSU="NO" # Cyclone genesis verification (FSU) +export DO_VERFOZN="YES" # Ozone data assimilation monitoring +export DO_VERFRAD="YES" # Radiance data assimilation monitoring +export DO_VMINMON="YES" # GSI minimization monitoring +export DO_ANLSTAT="NO" # JEDI-based analysis statistics +export DO_MOS="NO" # GFS Model Output Statistics - Only supported on WCOSS2 + +# NO for retrospective parallel; YES for real-time parallel +# arch.sh uses REALTIME for MOS. Need to set REALTIME=YES +# if want MOS written to HPSS. Should update arch_vrfy.sh and arch_tars to +# use RUNMOS flag +export REALTIME="YES" + +# Experiment mode (cycled or forecast-only) +export MODE="cycled" # cycled/forecast-only +export DO_TEST_MODE="NO" # option to change configuration for automated testing + +#################################################### +# DO NOT ADD MACHINE DEPENDENT STUFF BELOW THIS LINE +# IF YOU HAVE TO MAKE MACHINE SPECIFIC CHANGES BELOW +# FEEL FREE TO MOVE THEM ABOVE THIS LINE TO KEEP IT +# CLEAR +#################################################### +# Build paths relative to $HOMEgcafs +export FIXgsi="${HOMEgcafs}/fix/gsi" +export HOMEpost="${HOMEgcafs}" +export HOMEobsproc="${BASE_GIT:-}/obsproc/v${obsproc_run_ver:-}" + +# CONVENIENT utility scripts and other environment parameters +export NMV="/bin/mv" +export NLN="/bin/ln -sf" +export VERBOSE="YES" +export KEEPDATA="NO" +export DEBUG_POSTSCRIPT="NO" # PBS only; sets debug=true +export CHGRP_RSTPROD="YES" +export CHGRP_CMD="chgrp rstprod" +export NCDUMP="${NETCDF:-${netcdf_c_ROOT:-}}/bin/ncdump" +export NCLEN="${HOMEgcafs}/ush/getncdimlen" + +# Machine environment, jobs, and other utility scripts +export BASE_ENV="${HOMEgcafs}/env" + +# EXPERIMENT specific environment parameters +export SDATE="2021122012" +export EDATE="2021122100" +export EXP_WARM_START=".false." +export assim_freq="6" # Assimilation frequency in hours +export PSLOT="" +#export EXPDIR="/lfs/h2/emc/da/noscrub/cory.r.martin/gcafs/runtests2/EXPDIR/${PSLOT}" +#export ROTDIR="/lfs/h2/emc/da/noscrub/cory.r.martin/gcafs/runtests2/COMROOT/${PSLOT}" +export ROTDIR=${COMROOT} +export DUMP_SUFFIX="" +if [[ "${PDY}${cyc}" -ge "2019092100" && "${PDY}${cyc}" -le "2019110700" ]]; then + export DUMP_SUFFIX="p" # Use dumps from NCO GFS v15.3 parallel +fi +export ARCDIR="${NOSCRUB}/archive/${PSLOT}" +export ATARDIR="/NCEPDEV/${HPSS_PROJECT}/1year/${USER}/${machine}/scratch/${PSLOT}" +export FETCHDIR="{{ FETCHDIR }}" + +# Commonly defined parameters in JJOBS +export envir=${envir:-"prod"} +export NET="gcafs" # NET is defined in the job-card (ecf) +export RUN=${RUN:-"gcafs"} # RUN is defined in the job-card (ecf) + +# Get all the COM path templates +source "${EXPDIR}/config.com" + +export LOGSCRIPT=${LOGSCRIPT:-""} +#export LOGSCRIPT=${LOGSCRIPT:-"startmsg"} + +export SENDECF=${SENDECF:-"NO"} +export SENDSDM=${SENDSDM:-"NO"} +export SENDDBN_NTC=${SENDDBN_NTC:-"NO"} +export SENDDBN=${SENDDBN:-"NO"} +export DBNROOT=${DBNROOT:-${UTILROOT:-}/fakedbn} +export SENDAWIP=${SENDAWIP:-"NO"} + +# APP settings +export APP="ATMA" + +shopt -s extglob +# Adjust APP based on RUN +# If a component (WAVES, etc) needs to be turned off by RUN, set it here +case "${RUN}" in + enkf*) # Turn off aerosols and waves + APP="${APP/%+([WA])}" + ;; + *) # Keep app unchanged + ;; +esac +shopt -u extglob + +# Defaults: +export DO_ATM="YES" +export DO_COUPLED="NO" +export DO_WAVE="NO" +export DO_OCN="NO" +export DO_ICE="NO" +DO_AERO="NO" +export DO_PREP_OBS_AERO="NO" +aero_anl_runs="gcdas" # When to run aerosol analysis: gcdas, gcafs or both +export DO_AERO_FCST="YES" +export DO_AERO_ANL="NO" +export DOBNDPNT_WAVE="NO" +export DO_CHEM_WARMSTART="YES" +export FRAC_GRID=".true." +export DO_NEST="NO" # Whether to run a global-nested domain +if [[ "${DO_NEST:-NO}" == "YES" ]] ; then + export ntiles=7 + export NEST_OUTPUT_GRID="regional_latlon" + export FIXugwd="${FIXgcafs}/ugwd_nest" + export FIXorog="${FIXgcafs}/orog_nest" +else + export ntiles=6 + export FIXugwd="${FIXgcafs}/ugwd" + export FIXorog="${FIXgcafs}/orog" +fi + +# Set operational resolution +export OPS_RES="C768" # Do not change # TODO: Why is this needed and where is it used? + +# Resolution specific parameters +export LEVS=128 +export CASE="C384" +export CASE_ENS="{{ CASE_ENS }}" +export CASE_HIST="${CASE}" +export OCNRES="025" +export ICERES="${OCNRES}" + +# These are the currently recommended grid-combinations +case "${CASE}" in + "C48") + export waveGRD='uglo_100km' + ;; + "C96" | "C192") + export waveGRD='uglo_100km' + ;; + "C384") + export waveGRD='uglo_100km' + ;; + "C768" | "C1152") + export waveGRD='uglo_15km' + ;; + *) + echo "FATAL ERROR: Unrecognized CASE ${CASE}, ABORT!" + exit 2 + ;; +esac + +case "${APP}" in + ATM) + ;; + ATMA) + DO_AERO="YES" + ;; + ATMW) + export DO_COUPLED="YES" + export DO_WAVE="YES" + ;; + NG-GODAS) + export DO_ATM="NO" + export DO_OCN="YES" + export DO_ICE="YES" + ;; + S2S*) + export DO_COUPLED="YES" + export DO_OCN="YES" + export DO_ICE="YES" + + if [[ "${APP}" =~ A$ ]]; then + DO_AERO="YES" + fi + + if [[ "${APP}" =~ ^S2SW ]]; then + export DO_WAVE="YES" + fi + ;; + *) + echo "FATAL ERROR: Unrecognized APP: '${APP}'" + exit 3 + ;; +esac + +# Aerosol forecasts and analyses may be RUN-dependent +if [[ "${DO_AERO}" == "YES" ]]; then + if [[ ${aero_anl_runs} =~ ${RUN} ]]; then + export DO_AERO_ANL="YES" + fi + export DO_AERO_FCST="YES" + export USE_AERO_ANL="YES" # Use aerosol analysis in forecast +fi + +# Surface cycle update frequency +if [[ "${RUN}" =~ "gdas" ]] ; then + export FHCYC=1 + export FTSFS=10 +elif [[ "${RUN}" =~ "gfs" ]] ; then + export FHCYC=24 +fi + +# Output frequency of the forecast model (for cycling) +export FHMIN=0 +export FHMAX="9" +export FHOUT=3 # Will be changed to 1 in config.base if (DOHYBVAR set to NO and l4densvar set to false) +export FHOUT_OCN=3 +export FHOUT_ICE=3 +export FHOUT_AERO="3" + +# Cycle to run EnKF (set to BOTH for both gfs and gdas) +export EUPD_CYC="gfs" + +# GFS cycle info +export INTERVAL_GFS="12" # Frequency of GFS forecast +export SDATE_GFS="2021122018" + +# GFS output and frequency +export FHMIN_GFS=0 +export FHMAX_GFS="120" +# Intermediate times to stop forecast when running in segments +breakpnts="" +export FCST_SEGMENTS="${FHMIN_GFS},${breakpnts:+${breakpnts},}${FHMAX_GFS}" +export FHOUT_GFS=3 # 3 for ops +export FHMAX_HF_GFS="0" +export FHOUT_HF_GFS=1 +export FHOUT_OCN_GFS=6 +export FHOUT_ICE_GFS=6 +export FHMIN_WAV=0 +export FHOUT_WAV=3 +export FHOUT_WAV_GFS=3 +export FHMAX_HF_WAV=120 +export FHOUT_HF_WAV=1 +export FHMAX_WAV=${FHMAX:-9} +export FHMAX_WAV_GFS=${FHMAX_GFS} +export FHMAX_HF_GFS=$(( FHMAX_HF_GFS > FHMAX_GFS ? FHMAX_GFS : FHMAX_HF_GFS )) +export FHMAX_WAV=$(( FHMAX_WAV > FHMAX ? FHMAX : FHMAX_WAV )) +export FHMAX_WAV_GFS=$(( FHMAX_WAV_GFS > FHMAX_GFS ? FHMAX_GFS : FHMAX_WAV_GFS )) +export FHMAX_HF_WAV=$(( FHMAX_HF_WAV > FHMAX_WAV_GFS ? FHMAX_WAV_GFS : FHMAX_HF_WAV )) + +# TODO: Change gempak to use standard out variables (#2348) +export ILPOST=${FHOUT_HF_GFS} # gempak output frequency up to F120 +if (( FHMAX_HF_GFS < 120 )); then + export ILPOST=${FHOUT_GFS} +fi + +# Limit bounds of goes processing +export FHMAX_GOES=180 +export FHOUT_GOES=3 +if (( FHMAX_GOES > FHMAX_GFS )); then + export FHMAX_GOES=${FHMAX_GFS} +fi + +# GFS restart interval in hours +export restart_interval_gcafs=12 +# NOTE: Do not set this to zero. Instead set it to $FHMAX_GFS +# TODO: Remove this variable from config.base and reference from config.fcst +# TODO: rework logic in config.wave and push it to parsing_nameslist_WW3.sh where it is actually used + +export QUILTING=".true." +export OUTPUT_GRID="gaussian_grid" +export WRITE_DOPOST=".true." # WRITE_DOPOST=true, use inline POST +export WRITE_NSFLIP=".true." + +# IAU related parameters +export DOIAU="NO" # Enable 4DIAU for control with 3 increments +export IAUFHRS="3,6,9" +export IAU_FHROT=${IAUFHRS%%,*} +export IAU_DELTHRS=6 +export IAU_OFFSET=6 +export DOIAU_ENKF=${DOIAU:-"NO"} # Enable 4DIAU for EnKF ensemble +export IAUFHRS_ENKF="3,6,9" +export IAU_DELTHRS_ENKF=6 + +# Use Jacobians in eupd and thereby remove need to run eomg +export lobsdiag_forenkf=".true." + +# if [[ "$SDATE" -lt "2019020100" ]]; then # no rtofs in GDA +# export DO_WAVE="NO" +# echo "WARNING: Wave suite turned off due to lack of RTOFS in GDA for SDATE" +# fi + +# Microphysics Options: 99-ZhaoCarr, 8-Thompson; 6-WSM6, 10-MG, 11-GFDL +export imp_physics=8 + +# Shared parameters +# DA engine +export DO_JEDIATMVAR="NO" +export DO_JEDIATMENS="NO" +export DO_JEDIAEROVAR="{{ DO_JEDIAEROVAR }}" +export DO_JEDIOCNVAR="NO" +export DO_JEDISNOWDA="NO" +export DO_MERGENSST="NO" +export DO_STARTMEM_FROM_JEDIICE="NO" + +# Hybrid related +export DOHYBVAR="NO" +export DOHYBVAR_OCN="NO" +export DOLETKF_OCN="NO" +export NMEM_ENS="0" +export SMOOTH_ENKF="NO" +export l4densvar=".true." +export lwrite4danl=".true." +export DO_CALC_INCREMENT="NO" +export USE_BUILD_GSINFO="NO" +export BUILD_GSINFO_DIR="${PARMgcafs}/gsinfo" + +# Early-cycle EnKF parameters +export NMEM_ENS_GFS="{{ NMEM_ENS_GFS }}" +export NMEM_ENS_GFS_OFFSET="{{ NMEM_ENS_GFS_OFFSET }}" +export DO_CALC_INCREMENT_ENKF_GFS="NO" + +# EnKF output frequency +if [[ "${DOHYBVAR}" = "YES" ]]; then + export FHMIN_ENKF=3 + export FHMAX_ENKF=9 + export FHMAX_ENKF_GFS="12" + export FHOUT_ENKF_GFS=3 + if [[ ${l4densvar} = ".true." ]]; then + export FHOUT=1 + export FHOUT_ENKF=1 + else + export FHOUT_ENKF=3 + fi +fi + +# if 3DVAR and IAU +if [[ ${DOHYBVAR} == "NO" && ${DOIAU} == "YES" ]]; then + export IAUFHRS="6," + export IAU_FHROT="3" + export IAU_FILTER_INCREMENTS=".true." + export IAUFHRS_ENKF="6," +fi + +# Generate post-processing ensemble spread files +export ENKF_SPREAD="YES" + +#Soil DA and whether it is (land) IAU +if [[ "${RUN}" == "gdas" || "${RUN}" == "enkfgdas" ]]; then + export DO_GSISOILDA="NO" + export DO_LAND_IAU=".false." +else + export DO_GSISOILDA="NO" + export DO_LAND_IAU=".false." +fi +export LSOIL_INCR=2 + +# Check if cycle is cold starting, DOIAU off, or free-forecast mode +if [[ "${MODE}" = "cycled" && "${SDATE}" = "${PDY}${cyc}" && ${EXP_WARM_START} = ".false." ]] || [[ "${DOIAU}" = "NO" ]] || [[ "${MODE}" = "forecast-only" && ${EXP_WARM_START} = ".false." ]] ; then + export IAU_OFFSET=0 + export IAU_FHROT=0 + export IAUFHRS="6," + export DO_LAND_IAU=".false." +fi + +if [[ "${DOIAU_ENKF}" = "NO" ]]; then export IAUFHRS_ENKF="6,"; fi + +# Determine restart intervals +# For IAU, write restarts at beginning of window also +if [[ "${DOIAU_ENKF:-}" == "YES" ]]; then + export restart_interval_enkfgdas="3" +else + export restart_interval_enkfgdas="6" +fi + +export restart_interval_enkfgcafs=${restart_interval_enkfgdas} + +if [[ "${DOIAU}" == "YES" || "${DO_AERO_ANL}" == "YES" ]]; then + export restart_interval_gdas="3" +else + export restart_interval_gdas="6" +fi + +# turned on nsst in anal and/or fcst steps, and turn off rtgsst +export DONST="YES" +if [[ "${DONST}" = "YES" ]]; then export FNTSFA=" "; fi + +# The switch to apply SST elevation correction or not +export nst_anl=.true. + +# Analysis increments to zero in CALCINCEXEC +export INCREMENTS_TO_ZERO="'liq_wat_inc','icmr_inc','rwmr_inc','snmr_inc','grle_inc'" + +# Stratospheric increments to zero +export INCVARS_ZERO_STRAT="'sphum_inc','liq_wat_inc','icmr_inc','rwmr_inc','snmr_inc','grle_inc'" +export INCVARS_EFOLD="5" + +# Swith to generate netcdf or binary diagnostic files. If not specified, +# script default to binary diagnostic files. Set diagnostic file +# variables here since used in DA job +export netcdf_diag=".true." +export binary_diag=".false." + +# Cellular automata +export DO_CA="YES" + +# Verification options +export DO_METP="NO" # Run METPLUS jobs - set METPLUS settings in config.metp +export DO_FIT2OBS="YES" # Run fit to observations package + +#--online archive of netcdf files for fit2obs verification +export FHMAX_FITS=132 +if [[ "${FHMAX_FITS}" -gt "${FHMAX_GFS}" ]]; then + export FHMAX_FITS=${FHMAX_GFS} +fi + +# IC fetching options +# TODO: replace these options with DO_FETCH and FETCH_FROM +export DO_FETCH_HPSS="NO" # Copy from HPSS (on HPSS-accessible machines) onto COM +export DO_FETCH_LOCAL="NO" # Copy from local disk onto COM + +# Archiving options +export DO_ARCHCOM="NO" # Tar and archive the COM directories +export ARCHCOM_TO="hpss" # Valid options are hpss, globus_hpss, and local + +# Globus UUID for this machine +export CLIENT_GLOBUS_UUID="{{ CLIENT_GLOBUS_UUID }}" + +# The monitor jobs are not yet supported for JEDIATMVAR. +if [[ "${DO_JEDIATMVAR}" = "YES" ]]; then + export DO_FIT2OBS="NO" # Run fit to observations package + export DO_VERFOZN="NO" # Ozone data assimilation monitoring + export DO_VERFRAD="NO" # Radiance data assimilation monitoring + export DO_VMINMON="NO" # GSI minimization monitoring + export DO_ANLSTAT="YES" # JEDI-based analysis statistics +else + if [[ ${DO_AERO_ANL} = "YES" || ${DO_JEDIOCNVAR} = "YES" || ${DO_JEDISNOWDA} = "YES" ]]; then + export DO_ANLSTAT="YES" # JEDI-based analysis statistics + fi +fi + +# If starting ICs that are not at cycle hour +export OFFSET_START_HOUR=0 + +# Number of regional collectives to create soundings for +export NUM_SND_COLLECTIVES=${NUM_SND_COLLECTIVES:-9} + +echo "END: config.base" \ No newline at end of file diff --git a/parm/config/gcafs/config.cleanup b/parm/config/gcafs/config.cleanup new file mode 100644 index 00000000000..a4af66b493d --- /dev/null +++ b/parm/config/gcafs/config.cleanup @@ -0,0 +1,48 @@ +#! /usr/bin/env bash + +########## config.cleanup ########## +echo "BEGIN: config.cleanup" + +# Get task specific resources +source "${EXPDIR}/config.resources" cleanup + +export CLEANUP_COM="YES" # NO=retain ROTDIR. YES default in cleanup.sh + +#--starting and ending hours of previous cycles to be removed from rotating directory +# RUN-based cleanup variables +# Keep all files newer than SELECTIVE_CLEANUP_MIN hours +# Selectively remove files between SELECTIVE_CLEANUP_MAX and SELECTIVE_CLEANUP_MIN hours old, based on exclude_string +# Remove all GEMPAK files older than GEMPAK_CLEANUP hours if DO_GEMPAK is set to YES and only for RUN==gfs +# Remove **all** files and directories older than max(SELECTIVE_CLEANUP_MAX, GEMPAK_CLEANUP, RTOFS_CLEANUP) + +# RTOFS-specific variables (runs every RUN) +# Remove all RTOFS files older than RTOFS_CLEANUP hours + +export SELECTIVE_CLEANUP_MAX=144 +export SELECTIVE_CLEANUP_MIN=24 +export GEMPAK_CLEANUP=240 +export RTOFS_CLEANUP=24 +if [[ ${SELECTIVE_CLEANUP_MIN} -gt ${SELECTIVE_CLEANUP_MAX} ]]; then + echo "FATAL ERROR: Invalid selective cleanup times: " + echo " SELECTIVE_CLEANUP_MIN=${SELECTIVE_CLEANUP_MIN} > SELECTIVE_CLEANUP_MAX=${SELECTIVE_CLEANUP_MAX}" + exit 1 +fi + +# GEMPAK_CLEANUP must not be less than SELECTIVE_CLEANUP_MIN, which acts as the starting point for cleanup +if [[ "${DO_GEMPAK}" == "YES" && ${GEMPAK_CLEANUP} -lt ${SELECTIVE_CLEANUP_MIN} ]]; then + echo "FATAL ERROR: Invalid GEMPAK cleanup time: " + echo " GEMPAK_CLEANUP=${GEMPAK_CLEANUP} < SELECTIVE_CLEANUP_MIN=${SELECTIVE_CLEANUP_MIN}" + exit 2 +fi + +# Specify the list of files to exclude from the first stage of cleanup +# Because arrays cannot be exported, list is a single string of comma- +# separated values. This string is split to form an array at runtime. +export selective_exclude_string="" +case ${RUN} in + gdas | gfs) selective_exclude_string+="*prepbufr*, *cnvstat.tar*, *analysis.atm.a*.nc" ;; + enkf*) selective_exclude_string+="*f006.ens*" ;; + *) selective_exclude_string="" ;; +esac + +echo "END: config.cleanup" diff --git a/parm/config/gcafs/config.com b/parm/config/gcafs/config.com new file mode 100644 index 00000000000..0e5609e42b6 --- /dev/null +++ b/parm/config/gcafs/config.com @@ -0,0 +1,120 @@ +# shellcheck shell=bash +# Ignore shellcheck warnings about variables not being expanded; this is what we want +# shellcheck disable=SC2016 +echo "BEGIN: config.com" + +# These are just templates. All templates must use single quotations so variable +# expansion does not occur when this file is sourced. Substitution happens later +# during runtime. It is recommended to use the helper function `declare_from_tmpl()`, +# to do this substitution, which is defined in `ush/preamble.sh`. +# +# Syntax for declare_from_tmpl(): +# declare_from_tmpl [-rx] $var1[:$tmpl1] [$var2[:$tmpl2]] [...]] +# +# options: +# -r: Make variable read-only (same as `declare -r`) +# -x: Mark variable for declare -rx (same as `declare -x`) +# var1, var2, etc: Variable names whose values will be generated from a template +# and declared +# tmpl1, tmpl2, etc: Specify the template to use (default is "${var}_TMPL") +# +# Examples: +# # Current cycle and RUN +# YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_ATMOS_ANALYSIS +# +# # Previous cycle and gdas +# RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ +# COM_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL +# +# # Current cycle and COM for first member +# MEMDIR='mem001' YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_ATMOS_HISTORY +# + +# +# If any restart, input, or analysis template is updated, `setup_expt.py.fill_ROTDIR_cycled()` +# must correspondingly be updated to match. +# +# shellcheck disable=SC2034 +if [[ "${RUN_ENVIR:-emc}" == "nco" ]]; then + COM_OBSPROC_TMPL=$(compath.py "${envir}/obsproc/${obsproc_ver}")'/${RUN}.${YMD}/${HH}/atmos' + COM_OBSFORGE_TMPL=$(compath.py "${envir}/obsforge/${obsforge_ver}")'/${RUN}.${YMD}/${HH}/' + COM_RTOFS_TMPL=$(compath.py "${envir}/rtofs/${rtofs_ver}") + COM_TCVITAL_TMPL=$(compath.py "${envir}/gfs/${gfs_ver}")'/${RUN}.${YMD}/${HH}/atmos' + COM_GFS_TMPL=$(compath.py "${envir}/gfs/${gfs_ver}")'/${RUN}.${YMD}/${HH}' +else + COM_OBSPROC_TMPL='${DMPDIR}/${RUN}${DUMP_SUFFIX}.${YMD}/${HH}/atmos' + COM_OBSFORGE_TMPL='${IODADIR}/${RUN}${DUMP_SUFFIX}.${YMD}/${HH}' + COM_RTOFS_TMPL='${DMPDIR}' + COM_TCVITAL_TMPL='${DMPDIR}/${RUN}.${YMD}/${HH}/atmos' + #COM_GFS_TMPL='${ROTDIR}/${RUN}.${YMD}/${HH}' + COM_GFS_TMPL='/lfs/h1/ops/prod/com/gfs/v16.3/${RUN}.${YMD}/${HH}' +fi +declare -rx COM_OBS_TMPL='${ROTDIR}/${RUN}.${YMD}/${HH}/obs' +declare -rx COM_OBSPROC_TMPL COM_RTOFS_TMPL COM_OBSFORGE_TMPL + +COM_BASE='${ROTDIR}/${RUN}.${YMD}/${HH}/${MEMDIR}' + +declare -rx COM_TOP_TMPL='${ROTDIR}/${RUN}.${YMD}/${HH}' + +declare -rx COM_CONF_TMPL=${COM_BASE}'/conf' + +declare -rx COM_ATMOS_INPUT_TMPL=${COM_BASE}'/model/atmos/input' +declare -rx COM_ATMOS_RESTART_TMPL=${COM_BASE}'/model/atmos/restart' +declare -rx COM_ATMOS_ANALYSIS_TMPL=${COM_BASE}'/analysis/atmos' +declare -rx COM_SNOW_ANALYSIS_TMPL=${COM_BASE}'/analysis/snow' +declare -rx COM_SNOW_ANLMON_TMPL=${COM_BASE}'/products/snow/anlmon' +declare -rx COM_ATMOS_HISTORY_TMPL=${COM_BASE}'/model/atmos/history' +declare -rx COM_ATMOS_MASTER_TMPL=${COM_BASE}'/model/atmos/master' +declare -rx COM_ATMOS_GRIB_TMPL=${COM_BASE}'/products/atmos/grib2' +declare -rx COM_ATMOS_GRIB_GRID_TMPL=${COM_ATMOS_GRIB_TMPL}'/${GRID}' +declare -rx COM_ATMOS_BUFR_TMPL=${COM_BASE}'/products/atmos/bufr' +declare -rx COM_ATMOS_GEMPAK_TMPL=${COM_BASE}'/products/atmos/gempak/${GRID}' +declare -rx COM_ATMOS_GENESIS_TMPL=${COM_BASE}'/products/atmos/cyclone/genesis_vital' +declare -rx COM_ATMOS_TRACK_TMPL=${COM_BASE}'/products/atmos/cyclone/tracks' +declare -rx COM_ATMOS_GOES_TMPL=${COM_BASE}'/products/atmos/goes_sim' +declare -rx COM_ATMOS_IMAGERY_TMPL=${COM_BASE}'/products/atmos/imagery' +declare -rx COM_ATMOS_OZNMON_TMPL=${COM_BASE}'/products/atmos/oznmon' +declare -rx COM_ATMOS_RADMON_TMPL=${COM_BASE}'/products/atmos/radmon' +declare -rx COM_ATMOS_MINMON_TMPL=${COM_BASE}'/products/atmos/minmon' +declare -rx COM_ATMOS_ANLMON_TMPL=${COM_BASE}'/products/atmos/anlmon' +declare -rx COM_ATMOS_WMO_TMPL=${COM_BASE}'/products/atmos/wmo' + +declare -rx COM_WAVE_RESTART_TMPL=${COM_BASE}'/model/wave/restart' +declare -rx COM_WAVE_PREP_TMPL=${COM_BASE}'/model/wave/prep' +declare -rx COM_WAVE_HISTORY_TMPL=${COM_BASE}'/model/wave/history' +declare -rx COM_WAVE_GRID_TMPL=${COM_BASE}'/products/wave/gridded' +declare -rx COM_WAVE_GRID_RES_TMPL=${COM_BASE}'/products/wave/gridded/${GRDRESNAME}' +declare -rx COM_WAVE_STATION_TMPL=${COM_BASE}'/products/wave/station' +declare -rx COM_WAVE_GEMPAK_TMPL=${COM_BASE}'/products/wave/gempak' +declare -rx COM_WAVE_WMO_TMPL=${COM_BASE}'/products/wave/wmo' + +declare -rx COM_OCEAN_HISTORY_TMPL=${COM_BASE}'/model/ocean/history' +declare -rx COM_OCEAN_RESTART_TMPL=${COM_BASE}'/model/ocean/restart' +declare -rx COM_OCEAN_INPUT_TMPL=${COM_BASE}'/model/ocean/input' +declare -rx COM_OCEAN_ANALYSIS_TMPL=${COM_BASE}'/analysis/ocean' +declare -rx COM_OCEAN_ANLMON_TMPL=${COM_BASE}'/products/ocean/anlmon' +declare -rx COM_OCEAN_LETKF_TMPL=${COM_BASE}'/analysis/ocean/letkf' +declare -rx COM_OCEAN_BMATRIX_TMPL=${COM_BASE}'/bmatrix/ocean' +declare -rx COM_OCEAN_NETCDF_TMPL=${COM_BASE}'/products/ocean/netcdf' +declare -rx COM_OCEAN_GRIB_TMPL=${COM_BASE}'/products/ocean/grib2' +declare -rx COM_OCEAN_GRIB_GRID_TMPL=${COM_OCEAN_GRIB_TMPL}'/${GRID}' + +declare -rx COM_ICE_ANALYSIS_TMPL=${COM_BASE}'/analysis/ice' +declare -rx COM_ICE_LETKF_TMPL=${COM_BASE}'/analysis/ice/letkf' +declare -rx COM_ICE_ANLMON_TMPL=${COM_BASE}'/products/ice/anlmon' +declare -rx COM_ICE_BMATRIX_TMPL=${COM_BASE}'/bmatrix/ice' +declare -rx COM_ICE_INPUT_TMPL=${COM_BASE}'/model/ice/input' +declare -rx COM_ICE_HISTORY_TMPL=${COM_BASE}'/model/ice/history' +declare -rx COM_ICE_RESTART_TMPL=${COM_BASE}'/model/ice/restart' +declare -rx COM_ICE_NETCDF_TMPL=${COM_BASE}'/products/ice/netcdf' +declare -rx COM_ICE_GRIB_TMPL=${COM_BASE}'/products/ice/grib2' +declare -rx COM_ICE_GRIB_GRID_TMPL=${COM_ICE_GRIB_TMPL}'/${GRID}' + +declare -rx COM_CHEM_HISTORY_TMPL=${COM_BASE}'/model/chem/history' +declare -rx COM_CHEM_ANALYSIS_TMPL=${COM_BASE}'/analysis/chem' +declare -rx COM_CHEM_BMAT_TMPL=${COM_CHEM_ANALYSIS_TMPL}'/bmatrix' +declare -rx COM_CHEM_ANLMON_TMPL=${COM_BASE}'/products/chem/anlmon' +declare -rx COM_CHEM_INPUT_TMPL=${COM_BASE}'/model/chem/input' +declare -rx COM_CHEM_RESTART_TMPL=${COM_BASE}'/model/chem/restart' + +declare -rx COM_MED_RESTART_TMPL=${COM_BASE}'/model/med/restart' diff --git a/parm/config/gcafs/config.fcst b/parm/config/gcafs/config.fcst new file mode 100644 index 00000000000..9c8cdec64e4 --- /dev/null +++ b/parm/config/gcafs/config.fcst @@ -0,0 +1,333 @@ +#! /usr/bin/env bash + +########## config.fcst ########## +# Forecast specific + +echo "BEGIN: config.fcst" + +export USE_ESMF_THREADING="YES" # Toggle to use ESMF-managed threading or traditional threading in UFSWM +if [[ "${DO_CHEM_WARMSTART}" == "YES" ]]; then + export COPY_FINAL_RESTARTS="YES" # science configuration requires restarts to be copied + +else + export COPY_FINAL_RESTARTS="NO" +fi + +# Source model specific information that is resolution dependent +string="--fv3 ${CASE}" +if [[ "${DO_OCN}" == "YES" ]]; then + string="${string} --mom6 ${OCNRES}" +fi +if [[ "${DO_ICE}" == "YES" ]]; then + string="${string} --cice6 ${ICERES}" +fi +if [[ "${DO_WAVE}" == "YES" ]]; then + string="${string} --ww3 ${waveGRD}" +fi +if [[ "${DO_AERO_FCST}" == "YES" ]]; then + string="${string} --gocart" +fi +# We are counting on $string being multiple arguments +# shellcheck disable=SC2086 +source "${EXPDIR}/config.ufs" ${string} + +# Forecast length for GFS forecast +case ${RUN} in + gfs|gcafs) + # Convert comma-separated string into bash array + IFS=', ' read -ra segments <<< "${FCST_SEGMENTS}" + # Determine MIN and MAX based on the forecast segment + export FHMIN=${segments[${FCST_SEGMENT}]} + export FHMAX=${segments[${FCST_SEGMENT}+1]} + # Cap other FH variables at FHMAX for the segment + export FHMIN_WAV=$(( FHMIN > FHMIN_WAV ? FHMIN : FHMIN_WAV )) + export FHMAX_HF=$(( FHMAX_HF_GFS > FHMAX ? FHMAX : FHMAX_HF_GFS )) + export FHMAX_WAV=$(( FHMAX_WAV > FHMAX ? FHMAX : FHMAX_WAV )) + # shellcheck disable=SC2153 + export FHOUT=${FHOUT_GFS} + export FHOUT_HF=${FHOUT_HF_GFS} + export FHOUT_OCN=${FHOUT_OCN_GFS} + export FHOUT_ICE=${FHOUT_ICE_GFS} + ;; + *gdas|gcdas) + export FHMAX_HF=0 + export FHOUT_HF=0 + ;; + *) + echo "FATAL ERROR: Unsupported RUN '${RUN}'" + exit 1 +esac + +#use 2014-2023 mean MERRA2 aerosol +export MERRA2_6ym=".false." + +# Get task specific resources +source "${EXPDIR}/config.resources" fcst +export domains_stack_size="16000000" + + +if [[ "${DONST}" == "YES" ]]; then + source "${EXPDIR}/config.nsst" +fi + +export esmf_profile=".false." +export esmf_logkind="ESMF_LOGKIND_MULTI_ON_ERROR" #Options: ESMF_LOGKIND_MULTI_ON_ERROR, ESMF_LOGKIND_MULTI, ESMF_LOGKIND_NONE + + +####################################################################### + +export FORECASTSH="${SCRgcafs}/exglobal_forecast.sh" +#export FORECASTSH="${SCRgcafs}/exglobal_forecast.py" # Temp. while this is worked on +export FCSTEXEC="gcafs_model.x" + +####################################################################### +# Model configuration +export TYPE="nh" +export MONO="non-mono" + +# Use stratosphere h2o physics +export h2o_phys=".true." + +# Options of stratosphere O3 physics reaction coefficients +export new_o3forc="YES" + +export gwd_opt=2 + +# --GFS.v16 uGWD.v0, used for suite FV3_GFS_v16 and UFS p6 etc +# do_ugwp=T: use unified CGWD and OGWD, and turbulent orographic form drag (TOFD) +# do_ugwp=F: use unified CGWD but old OGWD, TOFD is not uded. +if (( gwd_opt == 1 )); then + export knob_ugwp_version=0 + export do_ugwp=".false." + export do_tofd=".false." + launch_level=$(echo "${LEVS}/2.35" |bc) + export launch_level +fi + + +# -- uGWD.v1, for suite FV3_GFS_v17 and FV3_GFS_v17p8b etc +if (( gwd_opt == 2 )); then + + #--used for UFS p7 and p8a + #export knob_ugwp_version=1 + #export do_ugwp=".false." + #export do_tofd=".false." + #export do_ugwp_v0=".false." + #export do_ugwp_v1=".true." + #export do_ugwp_v0_orog_only=".false." + #export do_ugwp_v0_nst_only=".false." + #export do_gsl_drag_ls_bl=".true." + #export do_gsl_drag_ss=".true." + #export do_gsl_drag_tofd=".true." + #export do_ugwp_v1_orog_only=".false." + + #--used for UFS p8 + export knob_ugwp_version=1 + export do_ugwp=".false." + export do_tofd=".false." + export do_ugwp_v0=".false." + export do_ugwp_v1=".true." + export do_ugwp_v0_orog_only=".false." + export do_ugwp_v0_nst_only=".false." + export do_gsl_drag_ls_bl=".true." + export do_gsl_drag_ss=".false." + export do_gsl_drag_tofd=".true." + export do_gwd_opt_psl=".true." + export do_ugwp_v1_orog_only=".false." + launch_level=$(echo "${LEVS}/2.35" |bc) + export launch_level +fi + +# Sponge layer settings +export d2_bg_k1=0.20 +export d2_bg_k2=0.04 +export dz_min=6 +export n_sponge=42 + +# PBL/turbulance schemes +export hybedmf=".false." +if [[ "${CCPP_SUITE}" == "FV3_global_nest"* ]]; then + export satmedmf=".false." +else + export satmedmf=".true." +fi +export isatmedmf=1 +tbf="" +if [[ "${satmedmf}" == ".true." ]]; then tbf="_satmedmf" ; fi + +#Convection schemes +export progsigma=".true." +tbp="" +if [[ "${progsigma}" == ".true." ]]; then tbp="_progsigma" ; fi + +# Radiation options +if [[ "${DO_AERO_FCST}" == "YES" ]]; then + if [[ "${RUN}" =~ "gfs" || "${RUN}" == "gcafs" || "${RUN}" == "gcdas" ]]; then + export IAER=2011 # spectral band mapping method for aerosol optical properties + else + export IAER=1011 + fi +else + export IAER=1011 +fi +export iovr_lw=3 ; #de-correlation length cloud overlap method (Barker, 2008) +export iovr_sw=3 ; #de-correlation length cloud overlap method (Barker, 2008) +export iovr=3 ; #de-correlation length cloud overlap method (Barker, 2008) +export icliq_sw=2 ; #cloud optical coeffs from AER's newer version v3.9-v4.0 for hu and stamnes +export isubc_sw=2 +export isubc_lw=2 + +# RRTMGP radiation scheme +export do_RRTMGP=.false. +export doGP_cldoptics_LUT=.false. +export doGP_lwscat=.false. + +# LSM configuration +# NoahMP only +export iopt_sfc="3" +export iopt_trs="2" + +# Microphysics configuration +export dnats=0 +export cal_pre=".true." +export do_sat_adj=".false." +export random_clds=".true." + +case ${imp_physics} in + 99) # ZhaoCarr + export ncld=1 + export FIELD_TABLE="${PARMgcafs}/ufs/fv3/field_table_zhaocarr${tbf}${tbp}" + export nwat=2 + ;; + 6) # WSM6 + export ncld=2 + export FIELD_TABLE="${PARMgcafs}/ufs/fv3/field_table_wsm6${tbf}${tbp}" + export nwat=6 + ;; + 8) # Thompson + export ncld=2 + export FIELD_TABLE="${PARMgcafs}/ufs/fv3/field_table_thompson_noaero_tke${tbp}" + export nwat=6 + + export cal_pre=".false." + export random_clds=".false." + export effr_in=".true." + export ltaerosol=".false." + export lradar=".true." + export ttendlim="-999" + export dt_inner=$((DELTIM/2)) + export sedi_semi=.true. + if [[ "${sedi_semi}" == .true. ]]; then export dt_inner=${DELTIM} ; fi + if [[ dt_inner -gt 300 ]]; then export dt_inner=300; fi + export decfl=10 + + export hord_mt_nh_nonmono=5 + export hord_xx_nh_nonmono=5 + export hord_dp_nh_nonmono=-5 + export vtdm4_nh_nonmono=0.02 + export nord=2 + export dddmp=0.1 + export d4_bg=0.12 + + if [[ "${CCPP_SUITE}" == "FV3_global_nest"* ]]; then + export FIELD_TABLE="${PARMgcafs}/ufs/fv3/field_table_thompson_aero_tke${tbp}" + export ltaerosol=".true." + export lcnorm=".true." + export do_mynnedmf=".true." + export do_mynnsfclay=".true." + export imfshalcnv=5 + export imfdeepcnv=5 + export betascu=0.5 + export betamcu=1.5 + export betadcu=8.0 + fi + + ;; + 11) # GFDL + export ncld=5 + export FIELD_TABLE="${PARMgcafs}/ufs/fv3/field_table_gfdl${tbf}${tbp}" + export nwat=6 + export dnats=1 + export cal_pre=".false." + export do_sat_adj=".true." + export random_clds=".false." + export lgfdlmprad=".true." + export effr_in=".true." + export reiflag=2 + + export hord_mt_nh_nonmono=5 + export hord_xx_nh_nonmono=5 + export hord_dp_nh_nonmono=-5 + export vtdm4_nh_nonmono=0.02 + export nord=2 + export d4_bg=0.12 + export dddmp=0.1 + ;; + *) echo "Unknown microphysics option, ABORT!" ;; +esac + +# Stochastic physics +export DO_SPPT="NO" +export DO_SKEB="NO" +export DO_SHUM="NO" +export DO_LAND_PERT="NO" +export DO_OCN_SPPT="NO" +export DO_OCN_PERT_EPBL="NO" + +#coupling settings +export cplmode="ufs.frac" +if [[ "${FRAC_GRID:-".true."}" == ".false." ]]; then + export cplmode="ufs.nfrac" +fi +export psm_bc="1" + +export min_lakeice="0.15" +export min_seaice=${min_seaice:-"0.15"} +export use_cice_alb=${use_cice_alb:-".false."} + +export FSICL="0" +export FSICS="0" + +#--------------------------------------------------------------------- +if [[ "${RUN}" =~ "gdas" || "${RUN}" =~ "gcdas" ]] ; then # GDAS cycle specific parameters + + # Variables used in DA cycling + export DIAG_TABLE="${PARMgcafs}/ufs/fv3/diag_table_da" + + # Write gfs restart files to rerun fcst from any break point + export restart_interval=${restart_interval_gdas:-6} + + # Turn on dry mass adjustment in GDAS + export adjust_dry_mass=".true." + +elif [[ "${RUN}" = "gfs" || "${RUN}" = "gcafs" ]] ; then # GFS or GCAFS cycle specific parameters + + # Write more variables to output + export DIAG_TABLE="${PARMgcafs}/ufs/fv3/diag_table" + + # Write gfs restart files to rerun fcst from any break point + export restart_interval=${restart_interval_gcafs:-12} + + # Turn off dry mass adjustment in GFS + export adjust_dry_mass=".false." + + # Write each restart file in 16 small files to save time + if [[ "${CASE}" = C768 ]]; then + export io_layout="4,4" + else + export io_layout="1,1" + fi + +else + echo "FATAL ERROR: Unsupported RUN '${RUN}'" + exit 1 +fi + +# This is used to ensure that the orography in the land restart file matches the orography +export CHECK_LAND_RESTART_OROG="YES" + +# Remember config.efcs will over-ride these values for ensemble forecasts +# if these variables are re-defined there. +# Otherwise, the ensemble forecast will inherit from config.fcst + +echo "END: config.fcst" \ No newline at end of file diff --git a/parm/config/gcafs/config.fetch b/parm/config/gcafs/config.fetch new file mode 100644 index 00000000000..0684c7738ad --- /dev/null +++ b/parm/config/gcafs/config.fetch @@ -0,0 +1,37 @@ +#! /usr/bin/env bash + +########## config.fetch ########## + +echo "BEGIN: config.fetch" + +# Get task specific resources +source "${EXPDIR}/config.resources" fetch + +if [[ "${PDY}${cyc}" -gt "${SDATE}" ]]; then + # determine GDAS version based on date + if [[ "${PDY}${cyc}" -ge "2022112900" ]]; then + gdas_version="v16.3" + elif [[ "${PDY}${cyc}" -ge "2022062700" ]]; then + gdas_version="v16.2" + else + gdas_version="prod" + fi + export gdas_version + if [[ "${machine}" == "ORION" || "${machine}" == "HERCULES" ]]; then + FETCH_YAML_TMPL_LIST="${PARMgcafs}/fetch/${NET}_${APP}_gdas-anl_msu.yaml.j2," + else + FETCH_YAML_TMPL_LIST="${PARMgcafs}/fetch/${NET}_${APP}_gdas-anl.yaml.j2," + fi + export FETCH_YAML_TMPL_LIST +else + # fetch based on first cycle + # Determine start type + if [[ "${EXP_WARM_START}" == ".false." ]]; then + ic_type="cold" + else + ic_type="warm" + fi + export FETCH_YAML_TMPL="${PARMgcafs}/fetch/${NET}_${APP}_${ic_type}_${MODE}.yaml.j2" +fi + +echo "END: config.fetch" diff --git a/parm/config/gcafs/config.globus b/parm/config/gcafs/config.globus new file mode 100644 index 00000000000..22510ddcd87 --- /dev/null +++ b/parm/config/gcafs/config.globus @@ -0,0 +1,35 @@ +#! /usr/bin/env bash + +########## config.globus ########## +# Globus-specific variables for all globus-based jobs + +echo "BEGIN: config.globus" + +# Get task specific resources +. "${EXPDIR}/config.resources" globus + +# Set the globus staging directory populated by the arch jobs +export STAGE_DIR="${DATAROOT}/archive_rotdir/${PSLOT}" + +# Set variables used by the Sven and Doorman services + +# Server name where the doorman will run +export SERVER_NAME="mercury" + +# Username on the server +export SERVER_USERNAME="" +# General delivery location on Mercury (staging area for data on server) +export SERVER_HOME="/collab2/data/${SERVER_USERNAME}/${PSLOT}" + +# Location of the doorman package on Mercury +export DOORMAN_ROOT="/home/David.Huber/doorman" + +# Server globus UUID +# niagara_UUID="1bfd8a79-52b2-4589-88b2-0648e0c0b35d" +mercury_UUID="e24545db-4d02-4b80-8aa0-fda791ddc604" +export SERVER_GLOBUS_UUID="${mercury_UUID}" + +# Sven's dropbox (temporary local directory) +export SVEN_DROPBOX_ROOT="${DATA}/SVEN_DROPBOX" + +echo "END: config.globus" diff --git a/parm/config/gcafs/config.metp b/parm/config/gcafs/config.metp new file mode 100644 index 00000000000..46121c8ee0e --- /dev/null +++ b/parm/config/gcafs/config.metp @@ -0,0 +1,102 @@ +#! /usr/bin/env bash + +########## config.metp ########## +# METplus verification step specific + +echo "BEGIN: config.metp" + +# Get task specific resources +. "${EXPDIR}/config.resources" metp + +export nproc=${tasks_per_node:-1} + +export RUN_GRID2GRID_STEP1="YES" # Run grid-to-grid verification using METplus +export RUN_GRID2OBS_STEP1="YES" # Run grid-to-obs verification using METplus +export RUN_PRECIP_STEP1="YES" # Run precip verification using METplus + + +#---------------------------------------------------------- +# METplus: Verify grid-to-grid, grid-to-obs, precipitation options +#---------------------------------------------------------- +## EMC_VERIF_GLOBAL SETTINGS +export HOMEverif_global=${HOMEgcafs}/sorc/verif-global.fd +export VERIF_GLOBALSH=${HOMEverif_global}/ush/run_verif_global_in_global_workflow.sh +## INPUT DATA SETTINGS +export model=${PSLOT} +export model_file_format="pgbf{lead?fmt=%2H}.${RUN}.{init?fmt=%Y%m%d%H}.grib2" +export model_hpss_dir=${ATARDIR}/.. +export model_dir=${ARCDIR}/.. +export get_data_from_hpss="NO" +export hpss_walltime="10" +## OUTPUT SETTINGS +export model_stat_dir=${ARCDIR}/.. +export make_met_data_by="VALID" +export SENDMETVIEWER="NO" +## DATE SETTINGS +export VRFYBACK_HRS="0" +## METPLUS SETTINGS +export METplus_verbosity="INFO" +export MET_verbosity="2" +export log_MET_output_to_METplus="yes" +# GRID-TO-GRID STEP 1: gfsmetpg2g1 +export g2g1_type_list="anom pres sfc" +export g2g1_anom_truth_name="self_anl" +export g2g1_anom_truth_file_format="pgbanl.${RUN}.{valid?fmt=%Y%m%d%H}.grib2" +export g2g1_anom_fhr_min=${FHMIN_GFS} +export g2g1_anom_fhr_max=${FHMAX_GFS} +export g2g1_anom_grid="G002" +export g2g1_anom_gather_by="VSDB" +export g2g1_pres_truth_name="self_anl" +export g2g1_pres_truth_file_format="pgbanl.${RUN}.{valid?fmt=%Y%m%d%H}.grib2" +export g2g1_pres_fhr_min=${FHMIN_GFS} +export g2g1_pres_fhr_max=${FHMAX_GFS} +export g2g1_pres_grid="G002" +export g2g1_pres_gather_by="VSDB" +export g2g1_sfc_truth_name="self_f00" +export g2g1_sfc_truth_file_format="pgbf00.${RUN}.{valid?fmt=%Y%m%d%H}.grib2" +export g2g1_sfc_fhr_min=${FHMIN_GFS} +export g2g1_sfc_fhr_max=${FHMAX_GFS} +export g2g1_sfc_grid="G002" +export g2g1_sfc_gather_by="VSDB" +export g2g1_mv_database_name="mv_${PSLOT}_grid2grid_metplus" +export g2g1_mv_database_group="NOAA NCEP" +export g2g1_mv_database_desc="Grid-to-grid METplus data for global workflow experiment ${PSLOT}" +# GRID-TO-OBS STEP 1: gfsmetpg2o1 +export g2o1_type_list="upper_air conus_sfc" +export g2o1_upper_air_msg_type_list="ADPUPA" +export g2o1_upper_air_vhr_list="00 06 12 18" +export g2o1_upper_air_fhr_min=${FHMIN_GFS} +export g2o1_upper_air_fhr_max="240" +export g2o1_upper_air_grid="G003" +export g2o1_upper_air_gather_by="VSDB" +export g2o1_conus_sfc_msg_type_list="ONLYSF ADPUPA" +export g2o1_conus_sfc_vhr_list="00 03 06 09 12 15 18 21" +export g2o1_conus_sfc_fhr_min=${FHMIN_GFS} +export g2o1_conus_sfc_fhr_max="240" +export g2o1_conus_sfc_grid="G104" +export g2o1_conus_sfc_gather_by="VSDB" +export g2o1_polar_sfc_msg_type_list="IABP" +export g2o1_polar_sfc_vhr_list="00 03 06 09 12 15 18 21" +export g2o1_polar_sfc_fhr_min=${FHMIN_GFS} +export g2o1_polar_sfc_fhr_max="240" +export g2o1_polar_sfc_grid="G219" +export g2o1_polar_sfc_gather_by="VSDB" +export g2o1_prepbufr_data_run_hpss="NO" +export g2o1_mv_database_name="mv_${PSLOT}_grid2obs_metplus" +export g2o1_mv_database_group="NOAA NCEP" +export g2o1_mv_database_desc="Grid-to-obs METplus data for global workflow experiment ${PSLOT}" +# PRECIP STEP 1: gfsmetppcp1 +export precip1_type_list="ccpa_accum24hr" +export precip1_ccpa_accum24hr_model_bucket="06" +export precip1_ccpa_accum24hr_model_var="APCP" +export precip1_ccpa_accum24hr_model_file_format="pgbf{lead?fmt=%2H}.${RUN}.{init?fmt=%Y%m%d%H}.grib2" +export precip1_ccpa_accum24hr_fhr_min=${FHMIN_GFS} +export precip1_ccpa_accum24hr_fhr_max="180" +export precip1_ccpa_accum24hr_grid="G211" +export precip1_ccpa_accum24hr_gather_by="VSDB" +export precip1_obs_data_run_hpss="NO" +export precip1_mv_database_name="mv_${PSLOT}_precip_metplus" +export precip1_mv_database_group="NOAA NCEP" +export precip1_mv_database_desc="Precip METplus data for global workflow experiment ${PSLOT}" + +echo "END: config.metp" diff --git a/parm/config/gcafs/config.nsst b/parm/config/gcafs/config.nsst new file mode 100644 index 00000000000..7bda81f058b --- /dev/null +++ b/parm/config/gcafs/config.nsst @@ -0,0 +1,39 @@ +#! /usr/bin/env bash + +########## config.nsst ########## +# NSST specific + +echo "BEGIN: config.nsst" + +# NSST parameters contained within nstf_name + +# nstf_name(1) : NST_MODEL (NSST Model) : 0 = OFF, 1 = ON but uncoupled, 2 = ON and coupled +export NST_MODEL=2 + +# Set NST_MODEL for JEDIATMVAR or JEDIATMENS +if [[ "${DO_JEDIATMVAR}" == "YES" || "${DO_JEDIATMENS}" == "YES" ]]; then + export NST_MODEL=1 +fi + +# nstf_name(2) : NST_SPINUP : 0 = OFF, 1 = ON, +export NST_SPINUP=0 +cdate="${PDY}${cyc}" +if (( cdate < 2017072000 )); then + export NST_SPINUP=1 +fi + +# nstf_name(3) : NST_RESV (Reserved, NSST Analysis) : 0 = OFF, 1 = ON +export NST_RESV=0 + +# nstf_name(4,5) : ZSEA1, ZSEA2 the two depths to apply vertical average (bias correction) +export ZSEA1=0 +export ZSEA2=0 + +export NST_GSI=3 # default 0: No NST info at all; + # 1: Input NST info but not used in GSI; + # 2: Input NST info, used in CRTM simulation, no Tr analysis + # 3: Input NST info, used in both CRTM simulation and Tr analysis +export NSTINFO=0 # number of elements added in obs. data array (default = 0) +if (( NST_GSI > 0 )); then export NSTINFO=4; fi + +echo "END: config.nsst" diff --git a/parm/config/gcafs/config.offlineanl b/parm/config/gcafs/config.offlineanl new file mode 100644 index 00000000000..95aa78a5b47 --- /dev/null +++ b/parm/config/gcafs/config.offlineanl @@ -0,0 +1,11 @@ +#! /usr/bin/env bash + +########## config.offlineanl ########## +# Offline analysis specific + +echo "BEGIN: config.offlineanl" + +# Get task specific resources +source "${EXPDIR}/config.resources" offlineanl + +echo "END: config.offlineanl" diff --git a/parm/config/gcafs/config.prep b/parm/config/gcafs/config.prep new file mode 100644 index 00000000000..a336ff45fdc --- /dev/null +++ b/parm/config/gcafs/config.prep @@ -0,0 +1,75 @@ +#! /usr/bin/env bash + +########## config.prep ########## +# Prep step specific + +echo "BEGIN: config.prep" + +# Get task specific resources +source "${EXPDIR}/config.resources" prep + +export cdate10=${PDY}${cyc} + +# Relocation and syndata QC +export PROCESS_TROPCY=${PROCESS_TROPCY:-NO} +export TROPCYQCRELOSH="${SCRgcafs}/exglobal_atmos_tropcy_qc_reloc.sh" +export COMINsyn=${COMINsyn:-$(compath.py "${envir}/com/gfs/${gfs_ver}")/syndat} + +# Allow users to control the generation or use of either operational or +# their processed prepbufr, prepbufr.acft_profiles, nsstbufr files +export MAKE_PREPBUFR="YES" # Generate prepbufr, etc. files by executing obsproc +if [[ "${MAKE_PREPBUFR^^}" != "YES" ]]; then + export USE_PREPBUFR_FROM_OPS="YES" # Copy operational prepbufr, etc files + if [[ "${USE_PREPBUFR_FROM_OPS^^}" != "YES" ]]; then + # Location of user processed prepbufr, etc files + export PREPBUFR_DIR="${DMPDIR:-}" # Replace DMPDIR with the location of processed files + fi +fi + +# Adjust observation error for GFS v16 parallels +# +# NOTE: Remember to set OBERROR in config.anal as PRVT is set below +# +# Set default prepobs_errtable.global +export PRVT="${FIXgcafs}/gsi/prepobs_errtable.global" + +# Set prepobs.errtable.global for GFS v16 retrospective parallels +if [[ ${RUN_ENVIR} == "emc" ]]; then + if [[ "${PDY}${cyc}" -ge "2019021900" && "${PDY}${cyc}" -lt "2019110706" ]]; then + export PRVT="${FIXgcafs}/gsi/gfsv16_historical/prepobs_errtable.global.2019021900" + fi + +# Place GOES-15 AMVs in monitor, assimilate GOES-17 AMVs, assimilate KOMPSAT-5 gps + if [[ "${PDY}${cyc}" -ge "2019110706" && "${PDY}${cyc}" -lt "2020040718" ]]; then + export PRVT="${FIXgcafs}/gsi/gfsv16_historical/prepobs_errtable.global.2019110706" + fi + +# NOTE: +# As of 2020040718, gfsv16_historical/prepobs_errtable.global.2020040718 is +# identical to ../prepobs_errtable.global. Thus, the logic below is not +# needed at this time + +# Set observation errors for type 135 (T) & 235 (uv) Canadian AMDAR observations +# if [[ "${PDY}${cyc}" -ge "2020040718" && "${PDY}${cyc}" -lt "YYYMMDDHH" ]]; then +# export PRVT=$EXPDIR/prepobs_errtable.global +# fi + +fi + +# NSST bufr was created with a different set of files prior to 2020102200 +# See comments at the end of +# https://github.com/NOAA-EMC/global-workflow/issues/313 +if [[ "${PDY}${cyc}" -ge "2020102200" ]]; then + export DTYPS_nsst='sfcshp tesac bathy trkob' +else + export DTYPS_nsst='sfcshp dbuoyb mbuoyb tesac bathy trkob' +fi + +# Set flags to optionally use external bias correction files +# TODO: remove when JEDI can cycle bias correcdtion files +export SOURCE_BIASCOR="{{ SOURCE_BIASCOR }}" +export COPY_BIASCOR_SOURCE="{{ COPY_BIASCOR_SOURCE }}" +export COPY_BIASCOR_STATIC="{{ COPY_BIASCOR_STATIC }}" +export CONVERT_BIASCOR="{{ CONVERT_BIASCOR }}" + +echo "END: config.prep" \ No newline at end of file diff --git a/parm/config/gcafs/config.prep_emissions b/parm/config/gcafs/config.prep_emissions new file mode 100644 index 00000000000..47bf5e26922 --- /dev/null +++ b/parm/config/gcafs/config.prep_emissions @@ -0,0 +1,13 @@ +#! /usr/bin/env bash + +########## config.prep_emissions ########## +# aerosol emissions preprocessing specific + +echo "BEGIN: config.prep_emissions" + +# Get task specific resources +source "${EXPDIR}/config.resources" prep_emissions + +source "${EXPDIR}/config.aero" + +echo "END: config.prep_emissions" diff --git a/parm/config/gcafs/config.prepareobs b/parm/config/gcafs/config.prepareobs new file mode 100644 index 00000000000..a172e3df9bd --- /dev/null +++ b/parm/config/gcafs/config.prepareobs @@ -0,0 +1,9 @@ +#!/bin/bash -x + +########## config.prepareobs ########## + +echo "BEGIN: config.prepareobs" + +# Get task specific resources +source "${EXPDIR}/config.resources" prepareobs +echo "END: config.prepareobs" diff --git a/parm/config/gcafs/config.resources b/parm/config/gcafs/config.resources new file mode 100644 index 00000000000..64495d54508 --- /dev/null +++ b/parm/config/gcafs/config.resources @@ -0,0 +1,1426 @@ +#! /usr/bin/env bash +# shellcheck disable=SC2034 + +########## config.resources ########## +# Set resource information for job tasks +# e.g. walltime, node, cores per node, memory etc. +# Note: machine-specific resources should be placed into the appropriate config file: +# config.resources.${machine} + +if (( $# != 1 )); then + + echo "Must specify an input task argument to set resource variables!" + echo "argument can be any one of the following:" + echo "stage_ic aerosol_init fetch" + echo "prep" + echo "atmanlinit atmanlvar atmanlfv3inc atmanlfinal" + echo "atmensanlinit atmensanlobs atmensanlsol atmensanlletkf atmensanlfv3inc atmensanlfinal ecen_fv3jedi analcalc_fv3jedi" + echo "snowanl esnowanl" + echo "prep_emissions" + echo "aeroanlinit aeroanlvar aeroanlfinal aeroanlgenb" + echo "prepareobs" + echo "offlineanl" + echo "anal sfcanl analcalc analdiag anlstat fcst echgres" + echo "upp atmos_products" + echo "tracker genesis genesis_fsu" + echo "verfozn verfrad vminmon fit2obs metp arc_vrfy arc_tars cleanup" + echo "eobs ediag eupd ecen esfc efcs epos earc_vrfy earc_tars" + echo "init_chem mom6ic oceanice_products" + echo "waveinit waveprep wavepostsbs wavepostbndpnt wavepostbndpntbll wavepostpnt" + echo "wavegempak waveawipsbulls waveawipsgridded" + echo "postsnd awips gempak npoess" + echo "marineanlinit marinebmat marineanlvar marineanlecen marineanalletkf marineanlchkpt marineanlfinal ocnanalvrfy" + exit 1 + +fi + +step=$1 + +echo "BEGIN: config.resources" + +case ${machine} in + "WCOSS2") + max_tasks_per_node=128 + # WCOSS2 nodes have 512GB of RAM, but only 500GB are reservable + mem_node_max="500GB" + ;; + "HERA") + max_tasks_per_node=40 + mem_node_max="96GB" + ;; + "URSA") + max_tasks_per_node=192 + mem_node_max="360GB" + ;; + "GAEAC5") + max_tasks_per_node=128 + mem_node_max="251GB" + ;; + "GAEAC6") + max_tasks_per_node=192 + mem_node_max="384GB" + ;; + "ORION") + max_tasks_per_node=40 + mem_node_max="180GB" + ;; + "HERCULES") + max_tasks_per_node=80 + mem_node_max="500GB" + ;; + "JET") + case ${PARTITION_BATCH} in + "xjet") + max_tasks_per_node=24 + mem_node_max="61GB" + ;; + "vjet") + max_tasks_per_node=16 + mem_node_max="61GB" + ;; + "sjet") + max_tasks_per_node=16 + mem_node_max="29GB" + ;; + "kjet") + max_tasks_per_node=40 + mem_node_max="88GB" + ;; + *) + echo "FATAL ERROR: Unknown partition ${PARTITION_BATCH} specified for ${machine}" + exit 3 + esac + ;; + "S4") + case ${PARTITION_BATCH} in + "s4") max_tasks_per_node=32 + mem_node_max="168GB" + ;; + "ivy") + max_tasks_per_node=20 + mem_node_max="128GB" + ;; + *) + echo "FATAL ERROR: Unknown partition ${PARTITION_BATCH} specified for ${machine}" + exit 3 + esac + ;; + "AWSPW") + export PARTITION_BATCH="compute" + npe_node_max=48 + max_tasks_per_node=48 + # TODO Supply a max mem/node value for AWS + mem_node_max="" + ;; + "AZUREPW") + export PARTITION_BATCH="compute" + npe_node_max=36 + max_tasks_per_node=36 + # TODO Supply a max mem/node value for AZURE + mem_node_max="" + ;; + "GOOGLEPW") + export PARTITION_BATCH="compute" + npe_node_max=30 + max_tasks_per_node=30 + # TODO Supply a max mem/node value for GOOGLE + mem_node_max="" + ;; + "CONTAINER") + max_tasks_per_node=1 + # TODO Supply a max mem/node value for a container + mem_node_max="" + ;; + *) + echo "FATAL ERROR: Unknown machine encountered by ${BASH_SOURCE[0]}" + exit 2 + ;; +esac + +export max_tasks_per_node + +case ${step} in + "prep") + walltime='00:30:00' + ntasks=14 + tasks_per_node=14 + threads_per_task=1 + memory="${mem_node_max}" + ;; + + "aerosol_init") + walltime="00:05:00" + ntasks=1 + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + NTASKS=${ntasks} + memory="6GB" + ;; + + "waveinit") + walltime="00:10:00" + ntasks=12 + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + NTASKS=${ntasks} + memory="2GB" + ;; + + "waveprep") + walltime="00:10:00" + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + if [[ "${RUN}" = *gdas ]]; then + ntasks=5 + memory="100GB" + elif [[ "${RUN}" = *gfs ]]; then + ntasks=65 + memory="150GB" + fi + NTASKS=${ntasks} + ;; + + "wavepostsbs") + # Walltime is per forecast hour; will be multipled by group size + ntasks=8 + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + NTASKS=${ntasks} + memory="20GB" + walltime="00:15:00" + ;; + + # The wavepost*pnt* jobs are I/O heavy and do not scale well to large nodes. + # Limit the number of tasks/node to 40. + "wavepostbndpnt") + walltime="00:30:00" + ntasks=1 + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export is_exclusive=True + if [[ ${tasks_per_node} -gt 40 ]]; then + tasks_per_node=40 + export is_exclusive=False + fi + NTASKS=${ntasks} + ;; + + "wavepostbndpntbll") + walltime="00:10:00" + ntasks=2 + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export is_exclusive=True + if [[ ${tasks_per_node} -gt 40 ]]; then + tasks_per_node=40 + export is_exclusive=False + fi + NTASKS=${ntasks} + ;; + + "wavepostpnt") + walltime="00:35:00" + ntasks=3 + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export is_exclusive=True + if [[ ${tasks_per_node} -gt 40 ]]; then + tasks_per_node=40 + export is_exclusive=False + fi + NTASKS=${ntasks} + ;; + + "wavegempak") + walltime="00:10:00" + ntasks=1 + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + NTASKS=${ntasks} + memory="1GB" + ;; + + "waveawipsbulls") + walltime="00:20:00" + ntasks=1 + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + NTASKS=${ntasks} + export is_exclusive=True + ;; + + "waveawipsgridded") + walltime="02:00:00" + ntasks=1 + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + NTASKS=${ntasks} + if [[ "${RUN}" = *gfs ]]; then + memory="1GB" + fi + ;; + + "atmanlinit") + export layout_x=${layout_x_atmanl} + export layout_y=${layout_y_atmanl} + + export layout_gsib_x=$(( layout_x * 3 )) + export layout_gsib_y=$(( layout_y * 2 )) + + walltime="00:10:00" + ntasks=1 + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + memory="4GB" + ;; + + "atmanlvar") + export layout_x=${layout_x_atmanl} + export layout_y=${layout_y_atmanl} + + walltime="00:30:00" + memory="96GB" + ntasks=$(( layout_x * layout_y * 6 )) + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export is_exclusive=True + ;; + + "atmanlfv3inc") + export layout_x=${layout_x_atmanl} + export layout_y=${layout_y_atmanl} + + walltime="00:30:00" + memory="96GB" + ntasks=$(( layout_x * layout_y * 6 )) + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export is_exclusive=True + ;; + + "atmanlfinal") + walltime="00:30:00" + ntasks=${max_tasks_per_node} + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export is_exclusive=True + ;; + + "snowanl") + # below lines are for creating JEDI YAML + case "${CASE}" in + "C1152" | "C768") + layout_x=6 + layout_y=6 + ;; + "C384") + layout_x=5 + layout_y=5 + ;; + "C192" | "C96" | "C48") + layout_x=1 + layout_y=1 + ;; + *) + echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}" + exit 4 + esac + + export layout_x + export layout_y + + walltime="00:15:00" + ntasks=$(( layout_x * layout_y * 6 )) + threads_per_task=2 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + ;; + + "esnowanl") + # below lines are for creating JEDI YAML + case "${CASE}" in + "C1152" | "C768") + layout_x=6 + layout_y=6 + ;; + "C384") + layout_x=5 + layout_y=5 + ;; + "C192" | "C96" | "C48") + layout_x=1 + layout_y=1 + ;; + *) + echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}" + exit 4 + esac + + export layout_x + export layout_y + + walltime="00:30:00" + ntasks=$(( layout_x * layout_y * 6 )) + threads_per_task=2 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + ;; + + "prepareobs") + ntasks=80 + walltime="00:20:00" + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + memory="3072M" + ;; + + + "aeroanlinit") + # below lines are for creating JEDI YAML + case "${CASE}" in + "C1152" | "C768") + layout_x=8 + layout_y=8 + walltime="00:30:00" + ;; + "C384") + layout_x=9 + layout_y=8 + walltime="00:20:00" + ;; + "C192" | "C96") + layout_x=4 + layout_y=4 + walltime="00:10:00" + ;; + "C48" ) + # this case is for testing only + layout_x=1 + layout_y=1 + walltime="00:10:00" + ;; + *) + echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}" + exit 4 + esac + + export layout_x + export layout_y + ntasks=1 + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + memory="3072M" + ;; + + "aeroanlvar") + + threads_per_task=1 + + case "${CASE}" in + "C1152" | "C768") + layout_x=8 + layout_y=8 + walltime="00:45:00" + tasks_per_node=24 + ;; + "C384") + layout_x=9 + layout_y=8 + walltime="00:30:00" + tasks_per_node=48 + ;; + "C192" | "C96") + layout_x=4 + layout_y=4 + walltime="00:20:00" + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + ;; + "C48" ) + # this case is for testing only + layout_x=1 + layout_y=1 + walltime="00:20:00" + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + ;; + *) + echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}" + exit 4 + esac + + export layout_x + export layout_y + + ntasks=$(( layout_x * layout_y * 6 )) + export is_exclusive=True + ;; + + "aeroanlgenb") + case "${CASE}" in + "C1152" | "C768") + layout_x=8 + layout_y=8 + ;; + "C384") + layout_x=6 + layout_y=6 + ;; + "C192" | "C96") + layout_x=4 + layout_y=4 + ;; + "C48" ) + # this case is for testing only + layout_x=1 + layout_y=1 + ;; + *) + echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}" + exit 4 + esac + + export layout_x + export layout_y + + walltime="00:30:00" + ntasks=$(( layout_x * layout_y * 6 )) + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export is_exclusive=True + + ;; + + + "aeroanlfinal") + case ${CASE} in + "C1152" | "C768") + layout_x=8 + layout_y=8 + walltime="00:30:00" + ;; + "C384") + layout_x=6 + layout_y=6 + walltime="00:20:00" + ;; + "C192" | "C96") + layout_x=4 + layout_y=4 + walltime="00:10:00" + ;; + "C48" ) + # this case is for testing only + layout_x=1 + layout_y=1 + walltime="00:10:00" + ;; + *) + echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}" + exit 4 + esac + ntasks=1 + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + memory="13072M" + ;; + + "marineanlinit") + walltime="00:10:00" + ntasks=1 + threads_per_task=1 + tasks_per_node=${max_tasks_per_node} + memory="24GB" + ;; + + "marinebmat") + npes=16 + ntasks=16 + case ${OCNRES} in + "025") + ntasks=480 + memory="256GB" + ;; + "050") ntasks=16;; + "100") ntasks=16;; + "500") ntasks=16;; + *) + echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${OCNRES}" + exit 4 + esac + + walltime="00:30:00" + threads_per_task=1 + export is_exclusive=True + tasks_per_node=$(( max_tasks_per_node / 2 )) + ;; + + "marineanlvar") + ntasks=16 + case ${OCNRES} in + "025") + ntasks=480 + memory="256GB" + ;; + "050") + ntasks=16 + memory="96GB" + ;; + "100") + ntasks=16 + memory="96GB" + ;; + "500") + ntasks=16 + memory="24GB" + ;; + *) + echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${OCNRES}" + exit 4 + esac + + walltime="00:30:00" + threads_per_task=1 + export is_exclusive=True + tasks_per_node=$(( max_tasks_per_node / 2 )) + ;; + + "marineanlecen") + ntasks=16 + case ${OCNRES} in + "025") + ntasks=40 + memory="256GB" + ;; + "050") + ntasks=16 + memory="96GB" + ;; + "100") + ntasks=16 + memory="96GB" + ;; + "500") + ntasks=16 + memory="24GB" + ;; + *) + echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${OCNRES}" + exit 4 + esac + + walltime="00:10:00" + export is_exclusive=True + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / 2 )) + ;; + + "marineanlletkf") + ntasks=16 + case ${OCNRES} in + "025") + ntasks=480 + memory="256GB" + ;; + "050") + ntasks=16 + memory="96GB" + ;; + "100") + ntasks=16 + memory="96GB" + ;; + "500") + ntasks=16 + memory="24GB" + ;; + *) + echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${OCNRES}" + exit 4 + esac + + walltime="00:30:00" + export is_exclusive=True + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / 2 )) + ;; + + + "marineanlchkpt") + walltime="00:10:00" + ntasks=1 + threads_per_task=1 + tasks_per_node=${max_tasks_per_node} + case ${OCNRES} in + "025") + memory="128GB" + ntasks=16;; + "050") + memory="32GB" + ntasks=16;; + "100") + memory="32GB" + ntasks=16;; + "500") + memory="32GB" + ntasks=8;; + *) + echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${OCNRES}" + exit 4 + esac + ;; + + "marineanlfinal") + walltime="00:30:00" + ntasks=${max_tasks_per_node} + threads_per_task=1 + tasks_per_node=${max_tasks_per_node} + ;; + + "ocnanalvrfy") + walltime="00:35:00" + ntasks=1 + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + memory="24GB" + ;; + + "offlineanl") + walltime="00:10:00" + ntasks=1 + threads_per_task=${max_tasks_per_node} + tasks_per_node=1 + ;; + + + "anal") + if [[ "${RUN}" = *gdas || "${RUN}" = gcdas ]]; then + walltime="01:20:00" + case ${CASE} in + "C1152" | "C768") + ntasks=780 + threads_per_task=5 + ;; + "C384" | "C192") + ntasks=160 + threads_per_task=10 + ;; + "C96" | "C48") + ntasks=84 + threads_per_task=5 + ;; + *) + echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}" + exit 4 + esac + elif [[ "${RUN}" = *gfs ]]; then + walltime="01:00:00" + case ${CASE} in + "C1152" | "C768") + ntasks=825 + threads_per_task=5 + ;; + "C384" | "C192") + ntasks=160 + threads_per_task=10 + ;; + "C96" | "C48") + ntasks=84 + threads_per_task=5 + ;; + *) + echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}" + exit 4 + esac + fi + + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export threads_per_task_cycle=${threads_per_task} + export tasks_per_node_cycle=$(( max_tasks_per_node / threads_per_task_cycle )) + export is_exclusive=True + ;; + + "ecen_fv3jedi") + export layout_x=${layout_x_ecen_fv3jedi} + export layout_y=${layout_y_ecen_fv3jedi} + + walltime="00:15:00" + ntasks_correction_increment=$(( layout_x * layout_y * 6 )) + ntasks_ensemble_recenter=$(( layout_x * layout_y * 6 * NMEM_ENS )) + ntasks=$(( ntasks_ensemble_recenter )) + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export is_exclusive=True + ;; + + "analcalc_fv3jedi") + export layout_x=${layout_x_analcalc_fv3jedi} + export layout_y=${layout_y_analcalc_fv3jedi} + + walltime="00:15:00" + ntasks=$(( layout_x * layout_y * 6 )) + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export is_exclusive=True + ;; + + "analcalc") + walltime="00:15:00" + ntasks=127 + export ntasks_calcanl="${ntasks}" + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + if [[ "${RUN}" = *gdas ]]; then + export threads_per_task_echgres=4 + elif [[ "${RUN}" = *gfs ]]; then + export threads_per_task_echgres=12 + fi + export is_exclusive=True + memory="48GB" + if [[ "${CASE}" == "C384" || "${CASE}" == "C768" || "${CASE}" == "C1152" ]]; then + memory="${mem_node_max}" + fi + ;; + + "analdiag") + walltime="00:15:00" + ntasks=96 # Should be at least twice ediag's tasks + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + memory="48GB" + ;; + + "anlstat") + walltime="00:30:00" + ntasks=20 + threads_per_task=1 + tasks_per_node=20 + memory="90GB" + ;; + + "sfcanl") + walltime="00:20:00" + ntasks=${ntiles:-6} + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export is_exclusive=True + ;; + + "prep_emissions") + export walltime="00:35:00" + export ntasks=1 + export threads_per_task=${max_tasks_per_node} + export tasks_per_node=1 + export is_exclusive=True + ;; + + "fcst" | "efcs") + export is_exclusive=True + + # Determine if using ESMF-managed threading or traditional threading + # If using traditional threading, set them to 1 + if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + export UFS_THREADS=1 + else # traditional threading + export UFS_THREADS=${nthreads_ufs:-1} + nthreads_fv3=1 + nthreads_mediator=1 + if [[ "${DO_WAVE}" == "YES" ]]; then + nthreads_ww3=1 + fi + if [[ "${DO_OCN}" == "YES" ]]; then + nthreads_mom6=1 + fi + if [[ "${DO_ICE}" == "YES" ]]; then + nthreads_cice6=1 + fi + fi + + if (( ntiles > 6 )); then + export layout_x_nest=${layout_x_nest:-10} + export layout_y_nest=${layout_y_nest:-10} + export npx_nest=${npx_nest:-1441} + export npy_nest=${npy_nest:-961} + fi + + # FV3 + if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + (( FV3THREADS = nthreads_fv3 )) + (( FV3PETS = ntasks_fv3 * nthreads_fv3 )) + else + (( FV3THREADS = UFS_THREADS )) + (( FV3PETS = ntasks_fv3 )) + fi + echo "FV3 using (nthreads, PETS) = (${FV3THREADS}, ${FV3PETS})" + + # Write grid component + QUILTPETS=0; QUILTTHREADS=0 + if [[ "${QUILTING:-}" == ".true." ]]; then + if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + (( QUILTTHREADS = nthreads_fv3 )) + (( QUILTPETS = ntasks_quilt * nthreads_fv3 )) + else + (( QUILTTHREADS = UFS_THREADS )) + (( QUILTPETS = ntasks_quilt )) + fi + (( WRTTASK_PER_GROUP = WRTTASK_PER_GROUP_PER_THREAD )) + export WRTTASK_PER_GROUP + fi + echo "QUILT using (nthreads, PETS) = (${QUILTTHREADS}, ${QUILTPETS})" + + # Total PETS for the atmosphere component + ATMTHREADS=${FV3THREADS} + (( ATMPETS = FV3PETS + QUILTPETS )) + export ATMPETS ATMTHREADS + echo "FV3ATM using (nthreads, PETS) = (${ATMTHREADS}, ${ATMPETS})" + + # Total PETS for the coupled model (starting w/ the atmosphere) + NTASKS_TOT=${ATMPETS} + + # Mediator + # The mediator PETS can overlap with other components, usually it lands on the atmosphere tasks. + # However, it is suggested limiting mediator PETS to 300, as it may cause the slow performance. + # See https://docs.google.com/document/d/1bKpi-52t5jIfv2tuNHmQkYUe3hkKsiG_DG_s6Mnukog/edit + # TODO: Update reference when moved to ufs-weather-model RTD + MEDTHREADS=${nthreads_mediator:-1} + MEDPETS=${MEDPETS:-${FV3PETS}} + (( "${MEDPETS}" > 300 )) && MEDPETS=300 + export MEDPETS MEDTHREADS + echo "MEDIATOR using (threads, PETS) = (${MEDTHREADS}, ${MEDPETS})" + + # GOCART + CHMPETS=0; CHMTHREADS=0 + if [[ "${DO_AERO_FCST}" == "YES" ]]; then + # GOCART shares the same grid and forecast tasks as FV3 (do not add write grid component tasks). + (( CHMTHREADS = ATMTHREADS )) + (( CHMPETS = FV3PETS )) + # Do not add to NTASKS_TOT + echo "GOCART using (threads, PETS) = (${CHMTHREADS}, ${CHMPETS})" + fi + export CHMPETS CHMTHREADS + + # Waves + WAVPETS=0; WAVTHREADS=0 + if [[ "${DO_WAVE}" == "YES" ]]; then + if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + (( WAVTHREADS = nthreads_ww3 )) + (( WAVPETS = ntasks_ww3 * nthreads_ww3 )) + else + (( WAVTHREADS = UFS_THREADS )) + (( WAVPETS = ntasks_ww3 )) + fi + echo "WW3 using (threads, PETS) = (${WAVTHREADS}, ${WAVPETS})" + (( NTASKS_TOT = NTASKS_TOT + WAVPETS )) + fi + export WAVPETS WAVTHREADS + + # Ocean + OCNPETS=0; OCNTHREADS=0 + if [[ "${DO_OCN}" == "YES" ]]; then + if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + (( OCNTHREADS = nthreads_mom6 )) + (( OCNPETS = ntasks_mom6 * nthreads_mom6 )) + else + (( OCNTHREADS = UFS_THREADS )) + (( OCNPETS = ntasks_mom6 )) + fi + echo "MOM6 using (threads, PETS) = (${OCNTHREADS}, ${OCNPETS})" + (( NTASKS_TOT = NTASKS_TOT + OCNPETS )) + fi + export OCNPETS OCNTHREADS + + # Ice + ICEPETS=0; ICETHREADS=0 + if [[ "${DO_ICE}" == "YES" ]]; then + if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + (( ICETHREADS = nthreads_cice6 )) + (( ICEPETS = ntasks_cice6 * nthreads_cice6 )) + else + (( ICETHREADS = UFS_THREADS )) + (( ICEPETS = ntasks_cice6 )) + fi + echo "CICE6 using (threads, PETS) = (${ICETHREADS}, ${ICEPETS})" + (( NTASKS_TOT = NTASKS_TOT + ICEPETS )) + fi + export ICEPETS ICETHREADS + + echo "Total PETS for ${RUN:-gfs} = ${NTASKS_TOT}" + + declare -x "ntasks"="${NTASKS_TOT}" + declare -x "threads_per_task"="${UFS_THREADS}" + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + + # TODO: make walltimes APP dependent + if [[ "${RUN}" == gdas || "${RUN}" == gcdas ]]; then + case "${CASE}" in + "C48" | "C96" | "C192") + declare -x "walltime"="00:20:00" + ;; + "C384") + declare -x "walltime"="00:30:00" + ;; + "C768" | "C1152") + # Not valid resolutions for ensembles + declare -x "walltime"="01:20:00" + ;; + *) + echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE} for ${RUN}" + exit 4 + ;; + esac + elif [[ "${RUN}" == enkfgcafs || "${RUN}" == enkfgdas ]]; then + case "${CASE_ENS}" in + "C48" | "C96" | "C192") + declare -x "walltime"="00:20:00" + ;; + "C384") + declare -x "walltime"="00:30:00" + ;; + *) + echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE_ENS} for ${RUN}" + exit 4 + ;; + esac + elif [[ "${RUN}" == gfs || "${RUN}" == gcafs ]]; then + case "${CASE}" in + "C48" | "C96" | "C192") + declare -x "walltime"="03:00:00" + ;; + "C384" | "C768" | "C1152") + declare -x "walltime"="06:00:00" + ;; + *) + echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE} for ${RUN}" + exit 4 + ;; + esac + fi + + unset NTASKS_TOT + ;; + + "oceanice_products") + # Walltime is per forecast hour; will be multipled by group size + walltime="00:15:00" + ntasks=1 + tasks_per_node=1 + threads_per_task=1 + memory="96GB" + ;; + + "upp") + case "${CASE}" in + "C48" | "C96") + ntasks=${CASE:1} + ;; + "C192" | "C384" | "C768") + ntasks=120 + memory="${mem_node_max}" + ;; + "C1152") + ntasks=200 + memory="${mem_node_max}" + ;; + *) + echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}" + exit 4 + ;; + esac + tasks_per_node=${ntasks} + if [[ "${CASE}" == "C1152" ]]; then + tasks_per_node=40 + fi + + threads_per_task=1 + + walltime="00:15:00" + if (( tasks_per_node > max_tasks_per_node )); then + tasks_per_node=${max_tasks_per_node} + fi + export is_exclusive=True + ;; + + "atmos_products") + # Walltime is per forecast hour; will be multipled by group size + walltime="00:15:00" + ntasks=24 + threads_per_task=1 + tasks_per_node="${ntasks}" + export is_exclusive=True + ;; + + "verfozn") + walltime="00:10:00" + ntasks=1 + threads_per_task=1 + tasks_per_node=1 + memory="1G" + ;; + + "verfrad") + walltime="00:40:00" + ntasks=1 + threads_per_task=1 + tasks_per_node=1 + memory="5G" + ;; + + "vminmon") + walltime="00:05:00" + ntasks=1 + threads_per_task=1 + tasks_per_node=1 + memory="1G" + ;; + + "tracker") + walltime="00:10:00" + ntasks=1 + threads_per_task=1 + tasks_per_node=1 + memory="4G" + ;; + + "genesis") + walltime="00:25:00" + ntasks=1 + threads_per_task=1 + tasks_per_node=1 + memory="10G" + ;; + + "genesis_fsu") + walltime="00:10:00" + ntasks=1 + threads_per_task=1 + tasks_per_node=1 + memory="10G" + ;; + + "fit2obs") + walltime="00:20:00" + ntasks=3 + threads_per_task=1 + tasks_per_node=1 + memory="20G" + if [[ "${CASE}" == "C768" || "${CASE}" == "C1152" ]]; then + memory="80GB" + fi + ;; + + "metp") + threads_per_task=1 + if [[ "${RUN}" = *gdas ]]; then + walltime="03:00:00" + elif [[ "${RUN}" = *gfs || "${RUN}" = *gcafs ]]; then + walltime="06:00:00" + fi + ntasks=1 + tasks_per_node=1 + export memory="80G" + ;; + + "echgres") + walltime="00:10:00" + ntasks=3 + threads_per_task=${max_tasks_per_node} + tasks_per_node=1 + ;; + + "init") + walltime="00:30:00" + ntasks=24 + threads_per_task=1 + tasks_per_node=6 + memory="70GB" + ;; + + "init_chem") + walltime="00:30:00" + ntasks=1 + tasks_per_node=1 + export is_exclusive=True + ;; + + "mom6ic") + walltime="00:30:00" + ntasks=24 + tasks_per_node=24 + export is_exclusive=True + ;; + + "arch_tars" | "getic" | "fetch" | "globus" | "globus_earc" ) + walltime="06:00:00" + ntasks=1 + tasks_per_node=1 + threads_per_task=1 + memory="4096M" + ;; + + "arch_vrfy" | "earc_vrfy") + walltime="00:15:00" + ntasks=1 + tasks_per_node=1 + threads_per_task=1 + memory="4096M" + ;; + + "cleanup") + walltime="00:15:00" + ntasks=1 + tasks_per_node=1 + threads_per_task=1 + memory="4096M" + ;; + + "stage_ic") + walltime="00:15:00" + ntasks=1 + tasks_per_node=1 + threads_per_task=1 + memory="4096M" + ;; + + "atmensanlinit") + export layout_x=${layout_x_atmensanl} + export layout_y=${layout_y_atmensanl} + + walltime="00:10:00" + ntasks=1 + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + memory="3072M" + ;; + + "atmensanlobs") + export layout_x=${layout_x_atmensanl} + export layout_y=${layout_y_atmensanl} + + walltime="00:30:00" + ntasks=$(( layout_x * layout_y * 6 )) + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + memory="96GB" + export is_exclusive=True + ;; + + "atmensanlsol") + export layout_x=${layout_x_atmensanl} + export layout_y=${layout_y_atmensanl} + + walltime="00:30:00" + ntasks=$(( layout_x * layout_y * 6 )) + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + memory="96GB" + export is_exclusive=True + ;; + + "atmensanlletkf") + export layout_x=${layout_x_atmensanl} + export layout_y=${layout_y_atmensanl} + + walltime="00:30:00" + ntasks=$(( layout_x * layout_y * 6 )) + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + memory="96GB" + export is_exclusive=True + ;; + + "atmensanlfv3inc") + export layout_x=${layout_x_atmensanl} + export layout_y=${layout_y_atmensanl} + + walltime="00:30:00" + ntasks=$(( layout_x * layout_y * 6 )) + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + memory="96GB" + export is_exclusive=True + ;; + + "atmensanlfinal") + walltime="00:30:00" + ntasks=${max_tasks_per_node} + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export is_exclusive=True + ;; + + "eobs") + walltime="00:15:00" + + case ${CASE} in + "C1152" | "C768") ntasks=200;; + "C384") ntasks=100;; + "C192" | "C96" | "C48") ntasks=40;; + *) + echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}" + exit 4 + ;; + esac + threads_per_task=2 + # NOTE The number of tasks and cores used must be the same for eobs + # See https://github.com/NOAA-EMC/global-workflow/issues/2092 for details + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export is_exclusive=True + # Unset tasks_per_node if it is not a multiple of max_tasks_per_node + # to prevent dropping data on the floor. This should be set int + # config.resources.{machine} instead. This will result in an error at + # experiment setup time if not set in config.resources.{machine}. + if [[ $(( max_tasks_per_node % tasks_per_node )) != 0 ]]; then + unset max_tasks_per_node + fi + ;; + + "ediag") + walltime="00:15:00" + ntasks=48 + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + memory="30GB" + ;; + + "eupd") + walltime="00:30:00" + case "${CASE}" in + "C1152" | "C768") + ntasks=480 + threads_per_task=6 + ;; + "C384") + ntasks=270 + threads_per_task=8 + ;; + "C192" | "C96" | "C48") + ntasks=42 + threads_per_task=2 + ;; + *) + echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}" + exit 4 + ;; + esac + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export is_exclusive=True + ;; + + "ecen") + walltime="00:10:00" + ntasks=80 + threads_per_task=4 + if [[ "${CASE}" == "C384" || "${CASE}" == "C192" || "${CASE}" == "C96" || "${CASE}" == "C48" ]]; then + threads_per_task=2 + fi + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export threads_per_task_cycle=${threads_per_task} + export tasks_per_node_cycle=${tasks_per_node} + export is_exclusive=True + ;; + + "esfc") + walltime="01:00:00" + ntasks=80 + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + threads_per_task_cycle=${threads_per_task} + tasks_per_node_cycle=$(( max_tasks_per_node / threads_per_task_cycle )) + ;; + + "epos") + walltime="00:15:00" + if [[ "${CASE}" == "C768" || "${CASE}" == "C1152" ]]; then + walltime="00:25:00" + fi + ntasks=80 + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + memory="${mem_node_max}" + export is_exclusive=True + ;; + + "postsnd") + walltime="02:00:00" + export ntasks=141 + export ntasks_postsndcfp=9 + case "${CASE}" in + "C768") + tasks_per_node=21 + threads_per_task=6 + ;; + "C1152") + tasks_per_node=9 + threads_per_task=14 + ;; + *) + tasks_per_node=21 + threads_per_task=6 + ;; + esac + export tasks_per_node_postsndcfp=1 + postsnd_req_cores=$(( tasks_per_node * threads_per_task )) + if (( postsnd_req_cores > max_tasks_per_node )); then + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + fi + export is_exclusive=True + ;; + + "awips") + walltime="03:30:00" + ntasks=1 + tasks_per_node=1 + threads_per_task=1 + memory="3GB" + ;; + + "npoess") + walltime="03:30:00" + ntasks=1 + tasks_per_node=1 + threads_per_task=1 + memory="3GB" + ;; + + "gempak") + # Walltime is per forecast hour; will be multipled by group size + walltime="00:30:00" + threads_per_task=1 + if [[ "${RUN}" = *gdas ]]; then + ntasks=2 + tasks_per_node=2 + memory="4GB" + elif [[ "${RUN}" = *gfs ]]; then + ntasks=28 + tasks_per_node=28 + memory="2GB" + fi + ;; + + "fbwind") + walltime="00:05:00" + ntasks=1 + threads_per_task=1 + memory="4GB" + tasks_per_node=1 + ;; + + *) + echo "FATAL ERROR: Invalid job ${step} passed to ${BASH_SOURCE[0]}" + exit 1 + ;; + +esac + +# Get machine-specific resources, overriding/extending the above assignments +if [[ -f "${EXPDIR}/config.resources.${machine}" ]]; then + source "${EXPDIR}/config.resources.${machine}" +fi + +# Check for RUN-specific variables and export them +for resource_var in threads_per_task ntasks tasks_per_node NTASKS memory walltime; do + run_resource_var="${resource_var}_${RUN}" + if [[ -n "${!run_resource_var+0}" ]]; then + declare -x "${resource_var}"="${!run_resource_var}" + elif [[ -n "${!resource_var+0}" ]]; then + export "${resource_var?}" + fi +done + +echo "END: config.resources" diff --git a/parm/config/gcafs/config.resources.WCOSS2 b/parm/config/gcafs/config.resources.WCOSS2 new file mode 100644 index 00000000000..c9e49c5b326 --- /dev/null +++ b/parm/config/gcafs/config.resources.WCOSS2 @@ -0,0 +1,67 @@ +#! /usr/bin/env bash +# shellcheck disable=SC2034 + +# WCOSS2-specific job resources + +case ${step} in + "prep") + export is_exclusive=True + export tasks_per_node=5 + export memory="480GB" + ;; + + "anal") + if [[ "${CASE}" == "C768" || "${CASE}" == "C1152" ]]; then + export threads_per_task=8 + # Make ntasks a multiple of 16 + export ntasks=1200 + export ntasks_gdas=1200 + export ntasks_gcafs=1200 + export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + fi + ;; + + "atmanlvar" | "atmensanlobs") + export tasks_per_node=48 + export memory="400GB" + ;; + + "fit2obs") + export tasks_per_node=3 + ;; + + "echgres") + export memory="200GB" + ;; + + "eupd") + case "${CASE}" in + "C1152" | "C768" | "C384") + if [[ "${RUN}" = *gdas ]]; then + export ntasks=315 + elif [[ "${RUN}" = *gfs ]]; then + export ntasks=135 + fi + export threads_per_task=14 + ;; + *) + ;; + esac + export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + ;; + + "eobs") + case "${CASE}" in + "C1152" | "C768" | "C384") + export tasks_per_node=50 + ;; + *) + export tasks_per_node=40 + ;; + esac + ;; + + *) + ;; + +esac diff --git a/parm/config/gcafs/config.sfcanl b/parm/config/gcafs/config.sfcanl new file mode 100644 index 00000000000..554847c602f --- /dev/null +++ b/parm/config/gcafs/config.sfcanl @@ -0,0 +1,36 @@ +#! /usr/bin/env bash + +########## config.sfcanl ########## +# GFS surface analysis specific + +echo "BEGIN: config.sfcanl" + +# Get task specific resources +source "${EXPDIR}/config.resources" sfcanl + +# Run global_cycle in coupled mode +if [[ "${DO_OCN}" == "YES" && "${DO_ICE}" == "YES" ]]; then + export COUPLED=".true." +fi + +# Turn off NST in JEDIATMVAR +if [[ "${DO_JEDIATMVAR}" == "YES" ]]; then + export DONST="NO" +fi + +if [[ "${RUN/enkf}" == "gfs" ]]; then + echo "turning off gsi soilda for gfs run" + DO_GSISOILDA="NO" +fi + +if [[ "${DO_GSISOILDA}" == "YES" ]]; then + if [[ ${DO_LAND_IAU} = ".true." ]]; then + export GCYCLE_DO_SOILINCR=".false." + else + export GCYCLE_DO_SOILINCR=".true." + fi + export GCYCLE_INTERP_LANDINCR=".false." + export REGRID_EXEC="${HOMEgcafs}/exec/regridStates.x" +fi + +echo "END: config.sfcanl" diff --git a/parm/config/gcafs/config.stage_ic b/parm/config/gcafs/config.stage_ic new file mode 100644 index 00000000000..cd2316c42ae --- /dev/null +++ b/parm/config/gcafs/config.stage_ic @@ -0,0 +1,48 @@ +#! /usr/bin/env bash + +########## config.stage_ic ########## + +echo "BEGIN: config.stage_ic" + +# Get task specific resources +source "${EXPDIR}/config.resources" stage_ic + +if [[ "${DO_FETCH_HPSS^^}" =~ "Y" || "${DO_FETCH_LOCAL^^}" =~ "Y" ]]; then + export ICSDIR="${DATAROOT}" # fetch untars data into DATAROOT +else + export ICSDIR="/lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR/C96C48/20250327" # User provided ICSDIR; blank if not provided +fi + +export BASE_IC="/lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR" # Platform home for staged ICs + +export STAGE_IC_YAML_TMPL="${PARMgcafs}/stage/master_${NET}.yaml.j2" + +source "${HOMEgcafs}/versions/ic.ver" + +if [[ ${EXP_WARM_START} = ".false." ]] ; then + export DOIAU="NO" # Turn off for staging +fi + +# Set ICSDIR (if not defined) +if [[ -z "${ICSDIR}" ]] ; then + + if (( NMEM_ENS > 0 )) ; then + ensic="${CASE_ENS}" + fi + + if [[ "${DO_OCN:-NO}" == "YES" ]] ; then + ocnic="mx${OCNRES}" + fi + + dir_name="${CASE}${ensic:-}${ocnic:-}" + ic_ver="${ic_versions[${dir_name}]}" + + export ICSDIR="${BASE_IC}/${dir_name}/${ic_ver}" + +fi + +#use of perturbations files for ensembles +export USE_OCN_ENS_PERTURB_FILES="NO" +export USE_ATM_ENS_PERTURB_FILES="NO" + +echo "END: config.stage_ic" \ No newline at end of file diff --git a/parm/config/gcafs/config.ufs b/parm/config/gcafs/config.ufs new file mode 100644 index 00000000000..e74a03dad11 --- /dev/null +++ b/parm/config/gcafs/config.ufs @@ -0,0 +1,696 @@ +#! /usr/bin/env bash + +########## config.ufs ########## +# UFS model resolution specific parameters +# e.g. time-step, processor layout, physics and dynamics parameters +# This config sets default variables for FV3, MOM6, CICE6 for their resolutions +# User can over-ride after sourcing this config file + +echo "BEGIN: config.ufs" + +if (( $# <= 1 )); then + + echo "Must specify an input resolution argument to set variables!" + echo "argument can be any one of the following:" + echo "--fv3 C48|C96|C192|C384|C768|C1152|C3072" + echo "--mom6 500|100|025" + echo "--cice6 500|100|025" + echo "--ww3 gwes_30m|glo_025|glo_100|glo_200|glo_500|mx025|uglo_100km" + echo "--gocart" + + exit 1 + +fi + +# Initialize +skip_mom6=true +skip_cice6=true +skip_ww3=true +skip_gocart=true +skip_mediator=true + +# Loop through named arguments +while (( $# > 0 )); do + key="$1" + case "${key}" in + "--fv3") + fv3_res="$2" + shift + ;; + "--mom6") + mom6_res="$2" + skip_mom6=false + shift + ;; + "--cice6") + cice6_res="$2" + skip_cice6=false + shift + ;; + "--ww3") + ww3_res="$2" + skip_ww3=false + shift + ;; + "--gocart") + skip_gocart=false + ;; + *) # unknown option + echo "FATAL ERROR: Unknown option: ${key}, ABORT!" + exit 1 + ;; + esac + shift +done + +# Mediator is required if any of the non-ATM components are used +if [[ "${skip_mom6}" == "false" ]] || [[ "${skip_cice6}" == "false" ]] || [[ "${skip_ww3}" == "false" ]]; then + skip_mediator=false +fi + +if [[ "${DO_NEST:-NO}" == "YES" ]] ; then + # Describe nest location, interaction with parent, etc. + export grid_type=0 + export stretch_fac=1.0001 + export TARGET_LAT=32.5 + export TARGET_LON=-135.0 + export NEST_LON1=-195.000000 + export NEST_LAT1=-7.500000 + export NEST_LON2=-75.000000 + export NEST_LAT2=72.500000 + export twowaynest=${twowaynest:-.true.} +else + # No nest. + export grid_type=-1 +fi + +# (Standard) Model resolution dependent variables +case "${fv3_res}" in + "C48") + export layout_x=2 + export layout_y=1 + export nthreads_fv3=1 + export nthreads_ufs=1 + export WRITE_GROUP=1 + export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=1 + # TODO: change this back to 1200 when "SKEBINT" can be set + # currently, the value is fixed to 1800 by the WM and must be a multiple of DELTIM + # when running stochastic physics + export DELTIM=900 + export xr_cnvcld=".false." # Do not pass conv. clouds to Xu-Randall cloud fraction + export cdmbgwd="0.071,2.1,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling + export cdmbgwd_gsl="40.0,1.77,1.0,1.0" # settings for GSL drag suite + export psl_gwd_dx_factor=6.0 + export knob_ugwp_tauamp=6.0e-3 # setting for UGWPv1 non-stationary GWD + export k_split=1 + export n_split=4 + export tau=8.0 + export rf_cutoff=300.0 + export fv_sg_adj=3600 + ;; + "C96") + if [[ "${DO_NEST:-NO}" == "YES" ]] ; then + if [[ "${RUN}" = *gdas ]]; then + export WRITE_GROUP=2 + export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=2 + elif [[ "${RUN}" = "gfs" || "${RUN}" = "gcafs" ]]; then + export layout_x=4 + export layout_y=4 + export WRITE_GROUP=2 + export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=2 + fi + + export DELTIM=450 + export layout_x_nest=12 + export layout_y_nest=10 + export nest_refine=4 + export nest_ioffset=4 + export nest_joffset=9 + export npx_nest=361 + export npy_nest=241 + export NEST_DLON=0.25 + export NEST_DLAT=0.25 + export psl_gwd_dx_factor=6.0 + else + export nthreads_fv3=1 + export nthreads_ufs=1 + export layout_x=2 + export layout_y=2 + export WRITE_GROUP=1 + export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=1 + export DELTIM=600 + export xr_cnvcld=.false. # Do not pass conv. clouds to Xu-Randall cloud fraction + export cdmbgwd="0.14,1.8,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling + export cdmbgwd_gsl="20.0,2.5,1.0,1.0" # settings for GSL drag suite + export psl_gwd_dx_factor=6.0 + export knob_ugwp_tauamp=3.0e-3 # setting for UGWPv1 non-stationary GWD + export k_split=1 + export n_split=4 + export tau=8.0 + export rf_cutoff=300.0 + export fv_sg_adj=1800 + fi + ;; + "C192") + if [[ "${DO_NEST:-NO}" == "YES" ]] ; then + export DELTIM=225 + if [[ "${RUN}" = *gdas ]]; then + export WRITE_GROUP=2 + export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=15 + elif [[ "${RUN}" = "gfs" || "${RUN}" = "gcafs" ]]; then + export layout_x=5 + export layout_y=6 + export WRITE_GROUP=2 + export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=15 + fi + export layout_x_nest=15 + export layout_y_nest=25 + export nest_refine=4 + export nest_ioffset=7 + export nest_joffset=19 + export npx_nest=721 + export npy_nest=481 + export NEST_DLON=0.125 + export NEST_DLAT=0.125 + export psl_gwd_dx_factor=6.0 + else + export layout_x=4 + export layout_y=6 + if [[ "${RUN}" = *gdas ]]; then + export nthreads_fv3=1 + export nthreads_ufs=1 + export WRITE_GROUP=1 + export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10 + elif [[ "${RUN}" = "gfs" || "${RUN}" = "gcafs" ]]; then + export nthreads_fv3=2 + export nthreads_ufs=2 + export WRITE_GROUP=2 + export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=5 + fi + export DELTIM=600 + export cdmbgwd="0.23,1.5,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling + export cdmbgwd_gsl="10.0,3.5,1.0,1.0" # settings for GSL drag suite + export psl_gwd_dx_factor=6.0 + export knob_ugwp_tauamp=1.5e-3 # setting for UGWPv1 non-stationary GWD + export k_split=2 + export n_split=4 + export tau=6.0 + export rf_cutoff=300.0 + export fv_sg_adj=1800 + fi + ;; + "C384") + if [[ "${DO_NEST:-NO}" == "YES" ]] ; then + export DELTIM=150 + export layout_x=8 + export layout_y=8 + export WRITE_GROUP=2 + export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=20 + if [[ "${RUN}" = *gdas ]]; then + export nthreads_fv3=1 + export nthreads_ufs=1 + elif [[ "${RUN}" = "gfs" || "${RUN}" = "gcafs" ]]; then + export nthreads_fv3=2 + export nthreads_ufs=2 + fi + export layout_x_nest=34 + export layout_y_nest=24 + export nest_refine=4 + export nest_ioffset=13 + export nest_joffset=37 + export npx_nest=1441 + export npy_nest=961 + export NEST_DLON=0.0625 + export NEST_DLAT=0.0625 + export psl_gwd_dx_factor=2.0 + else + export DELTIM=300 + export layout_x=6 + export layout_y=8 + export nthreads_fv3=2 + export nthreads_ufs=2 + export WRITE_GROUP=2 + export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10 + export cdmbgwd="1.1,0.72,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling + export cdmbgwd_gsl="5.0,5.0,1.0,1.0" # settings for GSL drag suite + export psl_gwd_dx_factor=6.0 + export knob_ugwp_tauamp=0.8e-3 # setting for UGWPv1 non-stationary GWD + export k_split=2 + export n_split=4 + export tau=4.0 + export rf_cutoff=300.0 + export fv_sg_adj=900 + fi + ;; + "C768") + if [[ "${DO_NEST:-NO}" == "YES" ]] ; then + export DELTIM=75 + export layout_x=16 + export layout_y=10 + export nthreads_fv3=2 + export WRITE_GROUP=2 + export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=90 + export layout_x_nest=48 + export layout_y_nest=45 + export nthreads_fv3_nest=2 + export nest_refine=4 + export nest_ioffset=24 + export nest_joffset=72 + export npx_nest=2881 + export npy_nest=1921 + export NEST_DLON=0.0325 + export NEST_DLAT=0.0325 + export psl_gwd_dx_factor=2.0 + else + export nthreads_fv3=4 + export nthreads_ufs=4 + if [[ "${RUN}" = *gdas ]]; then + export layout_x=8 + export layout_y=12 + export WRITE_GROUP=2 + export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=15 #Note this should be 10 for WCOSS2 + elif [[ "${RUN}" = "gfs" || "${RUN}" = "gcafs" ]]; then + export layout_x=12 + export layout_y=16 + export WRITE_GROUP=4 + export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=20 #Note this should be 10 for WCOSS2 + fi + export DELTIM=150 + export cdmbgwd="4.0,0.15,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling + export cdmbgwd_gsl="2.5,7.5,1.0,1.0" # settings for GSL drag suite + export psl_gwd_dx_factor=6.0 + export knob_ugwp_tauamp=0.5e-3 # setting for UGWPv1 non-stationary GWD + export k_split=2 + export n_split=4 + export tau=3.0 + export rf_cutoff=300.0 + export fv_sg_adj=450 + fi + ;; + "C1152") + export DELTIM=150 + export nthreads_fv3=4 + export nthreads_ufs=4 + export WRITE_GROUP=4 + export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=20 # TODO: refine these numbers when a case is available + # For GAEAC6 use 60 + if [[ "${RUN}" = *gdas ]]; then + export layout_x=16 + export layout_y=24 + elif [[ "${RUN}" = gfs ]]; then + export layout_x=32 + export layout_y=32 + export blocksize=36 # Note, default is 32 and MOD((resolution/layout_x)*(resolution/layout_y),blocksize) == 0 + export WRITE_GROUP=8 + export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=50 + elif [[ "${RUN}" = *gcafs ]]; then + export layout_x=16 + export layout_y=24 + fi + export cdmbgwd="4.0,0.10,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling + export cdmbgwd_gsl="1.67,8.8,1.0,1.0" # settings for GSL drag suite + export psl_gwd_dx_factor=2.0 + export knob_ugwp_tauamp=0.35e-3 # setting for UGWPv1 non-stationary GWD + export k_split=2 + export n_split=6 + export tau=2.5 + export rf_cutoff=300.0 + export fv_sg_adj=450 + ;; + "C3072") + export DELTIM=90 + export layout_y=32 + export layout_x=16 + export nthreads_fv3=4 + export nthreads_ufs=4 + export WRITE_GROUP=4 + export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10 + export cdmbgwd="4.0,0.05,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling + export cdmbgwd_gsl="0.625,14.1,1.0,1.0" # settings for GSL drag suite + export psl_gwd_dx_factor=2.0 + export knob_ugwp_tauamp=0.13e-3 # setting for UGWPv1 non-stationary GWD + export k_split=4 + export n_split=5 + export tau=0.5 + export rf_cutoff=300.0 + export fv_sg_adj=300 + ;; + *) + echo "FATAL ERROR: Unsupported FV3 resolution = ${fv3_res}, ABORT!" + exit 1 + ;; +esac + +(( WRTTASK_PER_GROUP_PER_THREAD = WRTTASK_PER_GROUP_PER_THREAD_PER_TILE * 6 )) +export WRTTASK_PER_GROUP_PER_THREAD + +(( ntasks_fv3 = layout_x * layout_y * 6 )) +if [[ "${DO_NEST:-NO}" == "YES" ]] ; then + (( ntasks_fv3 += layout_x_nest * layout_y_nest )) +fi +export ntasks_fv3 + +(( ntasks_quilt = WRITE_GROUP * WRTTASK_PER_GROUP_PER_THREAD )) +export ntasks_quilt + +# Determine whether to use compression in the write grid component +# and whether to use parallel NetCDF based on resolution +case ${fv3_res} in + "C48" | "C96" | "C192") + zstandard_level=0 + ideflate=0 + quantize_nsd=0 + OUTPUT_FILETYPE_ATM="netcdf" + OUTPUT_FILETYPE_SFC="netcdf" + ;; + "C384" | "C768" | "C1152" | "C3072") + zstandard_level=0 + ideflate=1 + quantize_nsd=14 + OUTPUT_FILETYPE_ATM="netcdf_parallel" + if [[ "${fv3_res}" == "C384" ]]; then + OUTPUT_FILETYPE_SFC="netcdf" # For C384, the write grid component is better off with serial netcdf + else + OUTPUT_FILETYPE_SFC="netcdf_parallel" + fi + ;; + *) + echo "FATAL ERROR: Unrecognized FV3 resolution ${fv3_res}" + exit 15 + ;; +esac +export zstandard_level ideflate quantize_nsd +export OUTPUT_FILETYPE_ATM OUTPUT_FILETYPE_SFC + +# cpl defaults +export cpl=".false." +export cplflx=".false." +export cplice=".false." +export cplchm=".false." +export cplwav=".false." +export cplwav2atm=".false." +if [[ "${DO_NEST:-NO}" == "YES" ]] ; then + export CCPP_SUITE="${CCPP_SUITE:-FV3_global_nest_v1}" +else + export CCPP_SUITE="${CCPP_SUITE:-FV3_GFS_v17_p8_ugwpv1}" +fi +model_list="atm" + +# Mediator specific settings +if [[ "${skip_mediator}" == "false" ]]; then + export cpl=".true." + export nthreads_mediator=${nthreads_fv3} # Use same threads as FV3 + export CCPP_SUITE="FV3_GFS_v17_coupled_p8_ugwpv1" # TODO: Does this include FV3_GFS_v17_p8? Can this be used instead of FV3_GFS_v17_p8? +fi + +# MOM6 specific settings +if [[ "${skip_mom6}" == "false" ]]; then + source "${EXPDIR}/config.ocn" + export cplflx=".true." + model_list="${model_list}.ocean" + nthreads_mom6=1 + case "${mom6_res}" in + "500") + ntasks_mom6=8 + OCNTIM=3600 + NX_GLB=72 + NY_GLB=35 + DT_DYNAM_MOM6='3600' + DT_THERM_MOM6='3600' + FRUNOFF="" + CHLCLIM="seawifs_1998-2006_smoothed_2X.nc" + MOM6_RESTART_SETTING='r' + MOM6_RIVER_RUNOFF='False' + case ${RUN} in + gfs|gefs|sfs|gcafs) + MOM6_DIAG_MISVAL="-1e34";; + gdas|enkfgdas|enkfgcafs) + MOM6_DIAG_MISVAL="0.0";; + *) + echo "FATAL ERROR: Unsupported RUN ${RUN} for ${mom6_res}" + exit 10 + ;; + esac + eps_imesh="4.0e-1" + MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_25L.nc" + MOM6_ALLOW_LANDMASK_CHANGES='False' + TOPOEDITS="" + ;; + "100") + ntasks_mom6=20 + OCNTIM=3600 + NX_GLB=360 + NY_GLB=320 + DT_DYNAM_MOM6='1800' + DT_THERM_MOM6='3600' + FRUNOFF="runoff.daitren.clim.1deg.nc" + CHLCLIM="seawifs_1998-2006_smoothed_2X.nc" + MOM6_RESTART_SETTING='r' + MOM6_RIVER_RUNOFF='False' + eps_imesh="2.5e-1" + TOPOEDITS="ufs.topo_edits_011818.nc" + case ${RUN} in + gfs|gefs|sfs|gcafs) + MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_30L.nc" + MOM6_DIAG_MISVAL="-1e34" + ;; + gdas|enkfgdas|enkfgcafs) + MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_75L.nc" + MOM6_DIAG_MISVAL="0.0" + ;; + *) + echo "FATAL ERROR: Unsupported RUN ${RUN} for ${mom6_res}" + exit 10 + ;; + esac + MOM6_ALLOW_LANDMASK_CHANGES='True' + ;; + "050") + ntasks_mom6=60 + OCNTIM=3600 + NX_GLB=720 + NY_GLB=576 + DT_DYNAM_MOM6='1800' + DT_THERM_MOM6='3600' + FRUNOFF="runoff.daitren.clim.${NX_GLB}x${NY_GLB}.v20180328.nc" + CHLCLIM="seawifs-clim-1997-2010.${NX_GLB}x${NY_GLB}.v20180328.nc" + MOM6_RESTART_SETTING='n' + MOM6_RIVER_RUNOFF='True' + eps_imesh="1.0e-1" + case ${RUN} in + gfs|gefs|sfs|gcafs) + MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_30L.nc" + MOM6_DIAG_MISVAL="-1e34" + ;; + gdas|enkfgdas|enkfgcafs) + MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_75L.nc" + MOM6_DIAG_MISVAL="0.0" + ;; + *) + echo "FATAL ERROR: Unsupported RUN ${RUN} for ${mom6_res}" + exit 10 + ;; + esac + MOM6_ALLOW_LANDMASK_CHANGES='False' + TOPOEDITS="" + ;; + "025") + if [[ "${RUN}" = enkfgdas ]]; then + ntasks_mom6=160 + elif [[ "${RUN}" = gdas ]]; then + ntasks_mom6=100 + elif [[ "${RUN}" = gfs ]]; then + ntasks_mom6=240 + else + ntasks_mom6=220 + fi + OCNTIM=1800 + NX_GLB=1440 + NY_GLB=1080 + DT_DYNAM_MOM6='900' + DT_THERM_MOM6='1800' + FRUNOFF="runoff.daitren.clim.${NX_GLB}x${NY_GLB}.v20180328.nc" + CHLCLIM="seawifs-clim-1997-2010.${NX_GLB}x${NY_GLB}.v20180328.nc" + MOM6_RIVER_RUNOFF='True' + MOM6_RESTART_SETTING="r" + eps_imesh="1.0e-1" + case ${RUN} in + gfs|gefs|sfs|gcafs) + MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_30L.nc" + MOM6_DIAG_MISVAL="-1e34" + ;; + gdas|enkfgdas|enkfgcafs) + MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_75L.nc" + MOM6_DIAG_MISVAL="0.0" + ;; + *) + echo "FATAL ERROR: Unsupported RUN ${RUN} for ${mom6_res}" + exit 10 + ;; + esac + MOM6_ALLOW_LANDMASK_CHANGES='False' + TOPOEDITS="" + ;; + *) + echo "FATAL ERROR: Unsupported MOM6 resolution = ${mom6_res}, ABORT!" + exit 1 + ;; + esac + + export nthreads_mom6 ntasks_mom6 + export OCNTIM + export NX_GLB NY_GLB + export DT_DYNAM_MOM6 DT_THERM_MOM6 + export FRUNOFF + export CHLCLIM + export TOPOEDITS + export MOM6_RIVER_RUNOFF + export MOM6_RESTART_SETTING + export eps_imesh + export MOM6_DIAG_COORD_DEF_Z_FILE + export MOM6_DIAG_MISVAL + export MOM6_ALLOW_LANDMASK_CHANGES +fi + +# CICE6 specific settings +if [[ "${skip_cice6}" == "false" ]]; then + source "${EXPDIR}/config.ice" + export cplice=".true." + model_list="${model_list}.ice" + # Ensure we sourced the MOM6 section + if [[ "${skip_mom6}" == "true" ]]; then + echo "FATAL ERROR: CICE6 cannot be configured without MOM6, ABORT!" + exit 1 + fi + + nthreads_cice6=${nthreads_mom6} # CICE6 needs to run on same threads as MOM6 + case "${cice6_res}" in + "500") + ntasks_cice6=4 + cice6_processor_shape="slenderX1" + ;; + "100") + ntasks_cice6=10 + cice6_processor_shape="slenderX2" + ;; + "050") + ntasks_cice6=30 + cice6_processor_shape="slenderX2" + ;; + "025") + if [[ "${RUN}" = gfs ]]; then + ntasks_cice6=240 + elif [[ "${RUN}" = enkfgdas ]]; then + ntasks_cice6=40 + else + ntasks_cice6=40 + fi + cice6_processor_shape="slenderX2" + ;; + *) + echo "FATAL ERROR: Unsupported CICE6 resolution = ${cice6_res}, ABORT!" + exit 1 + ;; + esac + # NX_GLB and NY_GLB are set in the MOM6 section above + # CICE6 runs on the same domain decomposition as MOM6 + export nthreads_cice6 ntasks_cice6 + export cice6_processor_shape +fi + +# WW3 specific settings +if [[ "${skip_ww3}" == "false" ]]; then + source "${EXPDIR}/config.wave" + export cplwav=".true." + export cplwav2atm=".true." + model_list="${model_list}.wave" + nthreads_ww3=1 + case "${ww3_res}" in + "gwes_30m") + ntasks_ww3=100 + ;; + "glo_025") + ntasks_ww3=262 + ;; + "glo_100") + ntasks_ww3=20 + ;; + "glo_200") + ntasks_ww3=30 + ;; + "glo_500") + ntasks_ww3=12 + ;; + "mx025") + ntasks_ww3=80 + ;; + "uglo_15km") + ntasks_ww3=884 + if [[ "${RUN}" = gfs ]]; then + ntasks_ww3=3000 + fi + ;; + "uglo_100km") + ntasks_ww3=40 + ;; + *) + echo "FATAL ERROR: Unsupported WW3 resolution = ${ww3_res}, ABORT!" + exit 1 + ;; + esac + export ntasks_ww3 nthreads_ww3 +fi + +# GOCART specific settings +if [[ "${skip_gocart}" == "false" ]]; then + source "${EXPDIR}/config.aero" + export cplchm=".true." + model_list="${model_list}.aero" +fi + +# Set the name of the UFS (previously nems) configure template to use +# Default ufs.configure templates for supported model configurations +# WW3 restart field variable is different for slow vs fast loop. Add WW3_RSTFLDS="ice" for slow loop variables based on coupling scheme. +case "${model_list}" in + atm) + default_template="${PARMgcafs}/ufs/ufs.configure.atm${tmpl_suffix:-}.IN" + ;; + atm.aero) + default_template="${PARMgcafs}/ufs/ufs.configure.atmaero${tmpl_suffix:-}.IN" + ;; + atm.wave) + default_template="${PARMgcafs}/ufs/ufs.configure.leapfrog_atm_wav${tmpl_suffix:-}.IN" + ;; + atm.ocean.ice) + default_template="${PARMgcafs}/ufs/ufs.configure.s2s${tmpl_suffix:-}.IN" + ;; + atm.ocean.ice.aero) + default_template="${PARMgcafs}/ufs/ufs.configure.s2sa${tmpl_suffix:-}.IN" + ;; + atm.ocean.ice.wave) + default_template="${PARMgcafs}/ufs/ufs.configure.s2sw${tmpl_suffix:-}.IN" + WW3_RSTFLDS="ice" + ;; + atm.ocean.ice.wave.aero) + default_template="${PARMgcafs}/ufs/ufs.configure.s2swa${tmpl_suffix:-}.IN" + WW3_RSTFLDS="ice" + ;; + *) + echo "FATAL ERROR: Unsupported UFSWM configuration for ${model_list}" + exit 16 + ;; +esac + +# Allow user to override the default template +export ufs_configure_template=${ufs_configure_template:-${default_template:-"/dev/null"}} +unset model_list default_template + +# export wave restart variable: +export WW3_RSTFLDS=${WW3_RSTFLDS:-" "} + +if [[ ! -r "${ufs_configure_template}" ]]; then + echo "FATAL ERROR: ${ufs_configure_template} either doesn't exist or is not readable." + exit 17 +fi + +echo "END: config.ufs" diff --git a/parm/config/gcafs/config.upp b/parm/config/gcafs/config.upp new file mode 100644 index 00000000000..ddc1ee087c4 --- /dev/null +++ b/parm/config/gcafs/config.upp @@ -0,0 +1,16 @@ +#! /usr/bin/env bash + +########## config.upp ########## +# UPP specific + +echo "BEGIN: config.upp" + +# Get task specific resources +. "${EXPDIR}/config.resources" upp + +export UPP_CONFIG="${PARMgcafs}/post/upp_gcafs.yaml" + +# No. of forecast hours to process in a single job +export NFHRS_PER_GROUP=3 + +echo "END: config.upp" diff --git a/sorc/build_all.sh b/sorc/build_all.sh index 5091b636a2f..0a8cdfbd146 100755 --- a/sorc/build_all.sh +++ b/sorc/build_all.sh @@ -16,7 +16,7 @@ function _usage() { cat << EOF Builds all of the global-workflow components by calling the individual build scripts in parallel. -Usage: ${BASH_SOURCE[0]} [-a UFS_app][-c build_config][-d][-f][-h][-v] [gfs] [gefs] [sfs] [gcafs] [gsi] [gcdas] [all] +Usage: ${BASH_SOURCE[0]} [-a UFS_app][-c build_config][-d][-f][-h][-v] [gfs] [gefs] [sfs] [gcafs] [gsi] [gdas] [all] -a UFS_app: Build a specific UFS app instead of the default. This will be applied to all UFS (GFS, GEFS, SFS, GCAFS) builds. -d: @@ -32,7 +32,7 @@ Usage: ${BASH_SOURCE[0]} [-a UFS_app][-c build_config][-d][-f][-h][-v] [gfs] [ge -p: Valid only for WCOSS2; enable parallel restart I/O when compiling the UFS - Specified systems (gfs, gefs, sfs, gcafs, gsi, gcdas) are non-exclusive, so they can be built together. + Specified systems (gfs, gefs, sfs, gcafs, gsi, gdas) are non-exclusive, so they can be built together. EOF exit 1 } @@ -78,7 +78,7 @@ else selected_systems="$*" fi -supported_systems=("gfs" "gefs" "sfs" "gcafs" "gsi" "gcdas" "all") +supported_systems=("gfs" "gefs" "sfs" "gcafs" "gsi" "gdas" "all") declare -A system_builds system_builds=( @@ -87,8 +87,8 @@ system_builds=( ["sfs"]="ufs_sfs gfs_utils ufs_utils upp ww3_gefs" ["gcafs"]="ufs_gcafs gfs_utils ufs_utils upp nexus gsi_utils" ["gsi"]="gsi_enkf gsi_monitor gsi_utils" - ["gcdas"]="gcdas gsi_utils" - ["all"]="ufs_gfs gfs_utils ufs_utils upp ww3_gfs ufs_gefs ufs_sfs ufs_gcafs ww3_gefs gcdas gsi_enkf gsi_monitor gsi_utils nexus" + ["gdas"]="gdas gsi_utils" + ["all"]="ufs_gfs gfs_utils ufs_utils upp ww3_gfs ufs_gefs ufs_sfs ufs_gcafs ww3_gefs gdas gsi_enkf gsi_monitor gsi_utils nexus" ) logs_dir="${HOMEgfs}/sorc/logs" @@ -100,7 +100,7 @@ fi # Jobs per build ("min max") declare -A build_jobs build_opts build_scripts build_jobs=( - ["ufs_gfs"]=8 ["ufs_gefs"]=8 ["ufs_sfs"]=8 ["ufs_gcafs"]=8 ["gcdas"]=8 ["gsi_enkf"]=2 ["gfs_utils"]=1 ["ufs_utils"]=1 + ["ufs_gfs"]=8 ["ufs_gefs"]=8 ["ufs_sfs"]=8 ["ufs_gcafs"]=8 ["gdas"]=8 ["gsi_enkf"]=2 ["gfs_utils"]=1 ["ufs_utils"]=1 ["ww3_gfs"]=1 ["ww3_gefs"]=1 ["gsi_utils"]=1 ["gsi_monitor"]=1 ["gfs_utils"]=1 ["upp"]=1 ["nexus"]=1 ) @@ -117,7 +117,7 @@ build_opts=( ["upp"]="${_build_debug}" ["ww3_gfs"]="${_verbose_opt} ${_build_debug}" ["ww3_gefs"]="-w ${_verbose_opt} ${_build_debug}" - ["gcdas"]="${_verbose_opt} ${_build_debug}" + ["gdas"]="${_verbose_opt} ${_build_debug}" ["ufs_utils"]="${_verbose_opt} ${_build_debug}" ["gfs_utils"]="${_verbose_opt} ${_build_debug}" ["gsi_utils"]="${_verbose_opt} ${_build_debug}" @@ -132,7 +132,7 @@ build_scripts=( ["ufs_gefs"]="build_ufs.sh" ["ufs_sfs"]="build_ufs.sh" ["ufs_gcafs"]="build_ufs.sh" - ["gcdas"]="build_gcdas.sh" + ["gdas"]="build_gdas.sh" ["gsi_enkf"]="build_gsi_enkf.sh" ["gfs_utils"]="build_gfs_utils.sh" ["ufs_utils"]="build_ufs_utils.sh" @@ -223,7 +223,7 @@ check_builds() { return 0 } -# Cleanup function to kill the gcdasApp build on ctrl-c or non-clean exit +# Cleanup function to kill the gdasApp build on ctrl-c or non-clean exit # shellcheck disable=SC2329 function cleanup() { echo "Exiting build script. Terminating subprocesses..." diff --git a/sorc/build_gcdas.sh b/sorc/build_gdas.sh similarity index 93% rename from sorc/build_gcdas.sh rename to sorc/build_gdas.sh index b11e73014c4..0e1c5a083a8 100755 --- a/sorc/build_gcdas.sh +++ b/sorc/build_gdas.sh @@ -28,6 +28,6 @@ source "${HOMEgfs_}/ush/detect_machine.sh" BUILD_JOBS="${BUILD_JOBS:-8}" \ WORKFLOW_BUILD="${WORKFLOW_BUILD:-"ON"}" \ WORKFLOW_TESTS="${WORKFLOW_TESTS:-"OFF"}" \ - "${HOMEgfs_}/sorc/gcdas.cd/build.sh" ${_opts} -w ${HOMEgfs_} + "${HOMEgfs_}/sorc/gdas.cd/build.sh" ${_opts} -w ${HOMEgfs_} exit diff --git a/sorc/gcdas.cd b/sorc/gcdas.cd deleted file mode 160000 index b8d4fce37bf..00000000000 --- a/sorc/gcdas.cd +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b8d4fce37bfd663a5856b946de5923d0aa4b2736 diff --git a/sorc/gdas.cd b/sorc/gdas.cd new file mode 160000 index 00000000000..f92b2dc84b6 --- /dev/null +++ b/sorc/gdas.cd @@ -0,0 +1 @@ +Subproject commit f92b2dc84b6f21f17f9f10058cc808b70b6b11b9 diff --git a/sorc/link_workflow_gcafs.sh b/sorc/link_workflow_gcafs.sh index f803792a144..b2fdb8834d1 100755 --- a/sorc/link_workflow_gcafs.sh +++ b/sorc/link_workflow_gcafs.sh @@ -81,14 +81,14 @@ esac # Source fix version file source "${HOMEgfs}/versions/fix.ver" -# Link gcdasapp python packages in ush/python +# Link gdasapp python packages in ush/python packages=("jcb") for package in "${packages[@]}"; do cd "${HOMEgfs}/ush/python" || exit 1 if [[ -s "${package}" ]]; then rm -f "${package}" fi - ${LINK_OR_COPY} "${HOMEgfs}/sorc/gcdas.cd/sorc/${package}/src/${package}" . + ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/sorc/${package}/src/${package}" . done # Link fix directories @@ -166,34 +166,34 @@ if [[ -d "${HOMEgfs}/sorc/ufs_utils.fd" ]]; then fi #------------------------------ -#--add gcdasApp fix directory +#--add gdasApp fix directory #------------------------------ -if [[ -d "${HOMEgfs}/sorc/gcdas.cd" ]]; then +if [[ -d "${HOMEgfs}/sorc/gdas.cd" ]]; then cd "${HOMEgfs}/fix" || exit 1 - mkdir -p gcdas - cd gcdas || exit 1 - for gcdas_sub in fv3jedi obs aero; do - if [[ -d "${gcdas_sub}" ]]; then - rm -rf "${gcdas_sub}" + mkdir -p gdas + cd gdas || exit 1 + for gdas_sub in fv3jedi obs aero; do + if [[ -d "${gdas_sub}" ]]; then + rm -rf "${gdas_sub}" fi - fix_ver="gcdas_${gcdas_sub}_ver" - ${LINK_OR_COPY} "${FIX_DIR}/gcdas/${gcdas_sub}/${!fix_ver}" "${gcdas_sub}" + fix_ver="gdas_${gdas_sub}_ver" + ${LINK_OR_COPY} "${FIX_DIR}/gdas/${gdas_sub}/${!fix_ver}" "${gdas_sub}" done fi #------------------------------ -#--add gcdasApp parm directory +#--add gdasApp parm directory #------------------------------ -if [[ -d "${HOMEgfs}/sorc/gcdas.cd" ]]; then +if [[ -d "${HOMEgfs}/sorc/gdas.cd" ]]; then cd "${HOMEgfs}/parm" || exit 1 - mkdir -p gcdas - cd gcdas || exit 1 - declare -a gcdasapp_comps=("aero" "atm" "io" "ioda" "jcb-gcdas" "jcb-algorithms" "anlstat" "analcalc") - for comp in "${gcdasapp_comps[@]}"; do + mkdir -p gdas + cd gdas || exit 1 + declare -a gdasapp_comps=("aero" "atm" "io" "ioda" "jcb-gdas" "jcb-algorithms" "anlstat" "analcalc") + for comp in "${gdasapp_comps[@]}"; do if [[ -d "${comp}" ]]; then rm -rf "${comp}" fi - ${LINK_OR_COPY} "${HOMEgfs}/sorc/gcdas.cd/parm/${comp}" . + ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/parm/${comp}" . done fi @@ -263,16 +263,16 @@ if [[ -d "${HOMEgfs}/sorc/gsi_utils.fd/install" ]]; then done fi -# gcdasApp executables -if [[ -d "${HOMEgfs}/sorc/gcdas.cd/install" ]]; then - cp -f "${HOMEgfs}/sorc/gcdas.cd/install/bin"/gcdas* ./ +# gdasApp executables +if [[ -d "${HOMEgfs}/sorc/gdas.cd/install" ]]; then + cp -f "${HOMEgfs}/sorc/gdas.cd/install/bin"/gdas* ./ fi -# gcdasApp libraries -if [[ -d "${HOMEgfs}/sorc/gcdas.cd/install" ]]; then +# gdasApp libraries +if [[ -d "${HOMEgfs}/sorc/gdas.cd/install" ]]; then mkdir -p "${HOMEgfs}/lib" || exit 1 cd "${HOMEgfs}/lib" || exit 1 - cp -af "${HOMEgfs}/sorc/gcdas.cd/install/lib/." ./ + cp -af "${HOMEgfs}/sorc/gdas.cd/install/lib/." ./ fi # NEXUS executable diff --git a/ush/jjob_header.sh b/ush/jjob_header.sh index f69524b732a..c85056938b3 100755 --- a/ush/jjob_header.sh +++ b/ush/jjob_header.sh @@ -92,8 +92,10 @@ export pgm=${pgm:-} # Run setpdy and initialize PDY variables ############################################## export cycle="t${cyc}z" +unset_strict setpdy.sh || true source ./PDY || true +set_strict ############################# # Source relevant config files diff --git a/ush/python/pygfs/task/analysis_stats.py b/ush/python/pygfs/task/analysis_stats.py index 7b6415657eb..0bd75a52fd5 100644 --- a/ush/python/pygfs/task/analysis_stats.py +++ b/ush/python/pygfs/task/analysis_stats.py @@ -2,8 +2,6 @@ import os import glob -import gsincdiag_to_ioda.proc_gsi_ncdiag as gsid -import gsincdiag_to_ioda.combine_obsspace as gsios import gzip import tarfile from logging import getLogger @@ -185,102 +183,4 @@ def convert_gsi_diags(self) -> None: ---------- None """ - logger.info("Converting GSI diag files to IODA files for analysis stats") - # copy GSI diag files to DATA path - diag_tars = ['cnvstat', 'radstat', 'oznstat'] - diag_dir_ges_path = os.path.join(self.task_config.DATA, 'atmos_gsi', 'atmos_gsi_ges') - diag_dir_anl_path = os.path.join(self.task_config.DATA, 'atmos_gsi', 'atmos_gsi_anl') - diag_dir_path = os.path.join(self.task_config.DATA, 'atmos_gsi', 'atmos_gsi_diags') - FileHandler({'mkdir': [diag_dir_path, diag_dir_ges_path, diag_dir_anl_path]}).sync() - diag_ioda_dir_ges_path = os.path.join(self.task_config.DATA, 'atmos_gsi', 'atmos_gsi_ioda_ges') - diag_ioda_dir_anl_path = os.path.join(self.task_config.DATA, 'atmos_gsi', 'atmos_gsi_ioda_anl') - output_dir_path = os.path.join(self.task_config.DATA, 'atmos_gsi', 'atmos_gsi_ioda') - FileHandler({'mkdir': [diag_ioda_dir_ges_path, diag_ioda_dir_anl_path, output_dir_path]}).sync() - diag_tar_copy_list = [] - for diag in diag_tars: - input_tar_basename = f"{self.task_config.APREFIX}{diag}.tar" - input_tar = os.path.join(self.task_config.COMIN_ATMOS_ANALYSIS, - input_tar_basename) - dest = os.path.join(diag_dir_path, input_tar_basename) - if os.path.exists(input_tar): - logger.info(f"{input_tar} exists. Preparing to copy it to {dest}") - diag_tar_copy_list.append([input_tar, dest]) - else: - logger.warning(f"{input_tar} does not exist to copy. Skipping ...") - FileHandler({'copy_opt': diag_tar_copy_list}).sync() - - # Untar and gunzip diag files - gsi_diag_tars = glob.glob(os.path.join(diag_dir_path, f"{self.task_config.APREFIX}*stat.tar")) - for diag_tar in gsi_diag_tars: - logger.info(f"Untarring {diag_tar}") - with tarfile.open(diag_tar, "r") as tar: - tar.extractall(path=diag_dir_path) - gsi_diags = glob.glob(os.path.join(diag_dir_path, "diag_*.nc4.gz")) - for diag in gsi_diags: - logger.info(f"Gunzipping {diag}") - output_file = diag.rstrip('.gz') - with gzip.open(diag, 'rb') as f_in: - with open(output_file, 'wb') as f_out: - f_out.write(f_in.read()) - os.remove(diag) - - # Copy diag files to ges or anl directory - anl_diags = glob.glob(os.path.join(diag_dir_path, "diag_*_anl*.nc4")) - ges_diags = glob.glob(os.path.join(diag_dir_path, "diag_*_ges*.nc4")) - copy_anl_diags = [] - for diag in anl_diags: - copy_anl_diags.append([diag, os.path.join(diag_dir_anl_path, os.path.basename(diag))]) - FileHandler({'copy_opt': copy_anl_diags}).sync() - copy_ges_diags = [] - for diag in ges_diags: - copy_ges_diags.append([diag, os.path.join(diag_dir_ges_path, os.path.basename(diag))]) - FileHandler({'copy_opt': copy_ges_diags}).sync() - - # Convert GSI diag files to ioda files using gsincdiag2ioda converter scripts - logger.info("Converting GSI guess diag files to IODA files") - gsid.proc_gsi_ncdiag(ObsDir=diag_ioda_dir_ges_path, DiagDir=diag_dir_ges_path) - logger.info("Converting GSI analysis diag files to IODA files") - gsid.proc_gsi_ncdiag(ObsDir=diag_ioda_dir_anl_path, DiagDir=diag_dir_anl_path) - - # now we need to combine the two sets of ioda files into one file - # by adding certain groups from the anl file to the ges file - ges_ioda_files = glob.glob(os.path.join(diag_ioda_dir_ges_path, '*nc')) - for ges_ioda_file in ges_ioda_files: - anl_ioda_file = ges_ioda_file.replace('_ges_', '_anl_').replace(diag_ioda_dir_ges_path, diag_ioda_dir_anl_path) - if os.path.exists(anl_ioda_file): - logger.info(f"Combining {ges_ioda_file} and {anl_ioda_file}") - out_ioda_file = os.path.join(output_dir_path, os.path.basename(ges_ioda_file).replace('_ges_', '_gsi_')) - gsid.combine_ges_anl_ioda(ges_ioda_file, anl_ioda_file, out_ioda_file) - else: - logger.warning(f"{anl_ioda_file} does not exist to combine with {ges_ioda_file}") - logger.warning("Skipping this file ...") - - # now, for conventional data, we need to combine certain obspaces - logger.info("Combining conventional GSI IODA files by obspace") - conv_obsspaces = ['sondes', 'aircraft', 'sfcship', 'sfc'] - for obspace in conv_obsspaces: - logger.info(f"Combining conventional GSI IODA files for obspace {obspace}") - FileList = glob.glob(os.path.join(output_dir_path, f"{obspace}_*_gsi_*.nc")) - timestamp = self.task_config.current_cycle.strftime('%Y%m%d%H') - combined_outfile = os.path.join(output_dir_path, f"{obspace}_gsi_{timestamp}.nc") - gsios.combine_obsspace(FileList, combined_outfile, False) - - # Tar up the ioda files - iodastatzipfile = os.path.join(self.task_config.DATA, 'atmos_gsi', 'atmos_gsi_ioda', - f"{self.task_config.APREFIX}atmos_gsi_analysis.ioda_hofx.tar.gz") - logger.info(f"Compressing GSI IODA files to {iodastatzipfile}") - # get list of iodastat files to put in tarball - iodastatfiles = glob.glob(os.path.join(output_dir_path, '*nc')) - logger.info(f"Gathering {len(iodastatfiles)} GSI IODA files to {iodastatzipfile}") - with tarfile.open(iodastatzipfile, "w|gz") as archive: - for targetfile in iodastatfiles: - archive.add(targetfile, arcname=os.path.basename(targetfile)) - logger.info(f"Finished compressing GSI IODA files to {iodastatzipfile}") - # copy to COMOUT - outdir = self.task_config.COMOUT_ATMOS_ANALYSIS - if not os.path.exists(outdir): - FileHandler({'mkdir': [outdir]}).sync() - dest = os.path.join(outdir, os.path.basename(iodastatzipfile)) - logger.info(f"Copying {iodastatzipfile} to {dest}") - FileHandler({'copy_opt': [[iodastatzipfile, dest]]}).sync() - logger.info("Finished copying GSI IODA tar file to COMOUT") + logger.info("Not supported for GCAFS workflow.") diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 90362bb7719..75cdb4b07ac 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -162,7 +162,8 @@ def initialize(self) -> None: # GBBEPx NRT files are in a different directory structure # Render the template with the current cycle to get the correct path tmp_dict = {'sdate': self.start_date, - 'FIRE_EMIS_NRT_DIR': self.task_config.FIRE_EMIS_NRT_DIR} + 'FIRE_EMIS_NRT_DIR': self.task_config.FIRE_EMIS_NRT_DIR, + 'nmem_ens': self.task_config.NMEM_ENS} yaml_config = self.render_template(tmp_dict) if self.task_config.AERO_EMIS_FIRE.lower() == 'gbbepx': self.task_config['AERO_EMIS_FIRE_DIR'] = yaml_config.fire_emission.config.NRT_DIRECTORY diff --git a/ush/python/pygfs/task/offline_analysis.py b/ush/python/pygfs/task/offline_analysis.py index b75f1570cd2..81c60b7aba8 100644 --- a/ush/python/pygfs/task/offline_analysis.py +++ b/ush/python/pygfs/task/offline_analysis.py @@ -85,9 +85,9 @@ def initialize(self) -> None: f"{self.task_config.GPREFIX}sfc.f006.nc") files_to_copy.append([sfcfcst_file_in, os.path.join(self.task_config.DATA, "sfcges_mem001")]) # TODO: Re-stage all of the inputs on HPSS to match EE2-compliant filenames - anl_file_in = os.path.join(self.task_config.COMIN_ATMOS_ANALYSIS.replace('analysis', ''), f"{self.task_config.APREFIX_IN}atmanl.nc") + anl_file_in = os.path.join(self.task_config.COMINgfs_ATMOS_ANALYSIS.replace('analysis', ''), f"{self.task_config.APREFIX_IN}atmanl.nc") files_to_copy.append([anl_file_in, os.path.join(self.task_config.DATA, "atmanl.input.nc")]) - sfcanl_file_in = os.path.join(self.task_config.COMIN_ATMOS_ANALYSIS.replace('analysis', ''), f"{self.task_config.APREFIX_IN}sfcanl.nc") + sfcanl_file_in = os.path.join(self.task_config.COMINgfs_ATMOS_ANALYSIS.replace('analysis', ''), f"{self.task_config.APREFIX_IN}sfcanl.nc") files_to_copy.append([sfcanl_file_in, os.path.join(self.task_config.DATA, "sfcanl.input.nc")]) FileHandler({'copy': files_to_copy}).sync() @@ -254,9 +254,9 @@ def finalize(self) -> None: transfer_files = [] transfer_files.append([os.path.join(self.task_config.COMIN_OBSPROC, f"{self.task_config.APREFIX_IN}rtgssthr.grb"), os.path.join(self.task_config.COMOUT_OBS, f"{self.task_config.APREFIX}rtgssthr.grb")]) - transfer_files.append([os.path.join(self.task_config.COMIN_OBSPROC, f"{self.task_config.APREFIX_IN}seaice.5min.blend.grb"), + transfer_files.append([os.path.join(self.task_config.COMINgfs_ATMOS_ANALYSIS, f"{self.task_config.APREFIX_IN}seaice.5min.blend.grb"), os.path.join(self.task_config.COMOUT_OBS, f"{self.task_config.APREFIX}seaice.5min.blend.grb")]) - transfer_files.append([os.path.join(self.task_config.COMIN_OBSPROC, f"{self.task_config.APREFIX_IN}snogrb_t1534.3072.1536"), + transfer_files.append([os.path.join(self.task_config.COMINgfs_ATMOS_ANALYSIS, f"{self.task_config.APREFIX_IN}snogrb_t1534.3072.1536"), os.path.join(self.task_config.COMOUT_OBS, f"{self.task_config.APREFIX}snogrb_t1534.3072.1536")]) # TODO: Re-stage the inputs for the GCDAS offline analysis on HPSS following EE2-compliant filenames, then update this line transfer_files.append([ diff --git a/ush/python/pyobsforge/__init__.py b/ush/python/pyobsforge/__init__.py new file mode 100644 index 00000000000..57f3d08e052 --- /dev/null +++ b/ush/python/pyobsforge/__init__.py @@ -0,0 +1,5 @@ +import os + +__docformat__ = "restructuredtext" +__version__ = "0.1.0" +pyobsforge_directory = os.path.dirname(__file__) diff --git a/ush/python/pyobsforge/obsdb/__init__.py b/ush/python/pyobsforge/obsdb/__init__.py new file mode 100644 index 00000000000..5f2a238e12f --- /dev/null +++ b/ush/python/pyobsforge/obsdb/__init__.py @@ -0,0 +1 @@ +from .obsdb import BaseDatabase # noqa diff --git a/ush/python/pyobsforge/obsdb/jrr_aod_db.py b/ush/python/pyobsforge/obsdb/jrr_aod_db.py new file mode 100644 index 00000000000..3a14b3a32d7 --- /dev/null +++ b/ush/python/pyobsforge/obsdb/jrr_aod_db.py @@ -0,0 +1,93 @@ +import os +import glob +from datetime import datetime, timedelta +from pyobsforge.obsdb import BaseDatabase + + +class JrrAodDatabase(BaseDatabase): + """Class to manage an observation file database for JRR-AOD data.""" + + def __init__(self, db_name="jrr_aod_obs.db", + dcom_dir="/lfs/h1/ops/prod/dcom/", + obs_dir="jrr_aod"): + base_dir = os.path.join(dcom_dir, '*', obs_dir) + super().__init__(db_name, base_dir) + + def create_database(self): + """ + Create the SQLite database and observation files table. + + The table `obs_files` contains the following columns: + - `id`: A unique identifier for each record (auto-incremented primary key). + - `filename`: The full path to the observation file (must be unique). + - `obs_time`: The timestamp of the observation, extracted from the filename. + - `receipt_time`: The timestamp when the file was added to the `dcom` directory. + - `satellite`: The satellite from which the observation was collected (e.g., NPP, NOAA-20). + """ + query = """ + CREATE TABLE IF NOT EXISTS obs_files ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + filename TEXT UNIQUE, + obs_time TIMESTAMP, + receipt_time TIMESTAMP, + satellite TEXT + ) + """ + self.execute_query(query) + + def parse_filename(self, filename): + """Extract metadata from filenames matching the JRR-AOD pattern.""" + # Make sure the filename matches the expected pattern + # Pattern: JRR-AOD_v3r2_n21_sYYYYMMDDHHMMSS_eYYYYMMDDHHMMSS_cYYYYMMDDHHMMSS.nc + basename = os.path.basename(filename) + parts = basename.split('_') + try: + if len(parts) >= 4 and parts[0] == "JRR-AOD": + obs_time = datetime.strptime(parts[3][1:13], "%Y%m%d%H%M") + receipt_time = datetime.fromtimestamp(os.path.getctime(filename)) + satellite = parts[2] + return filename, obs_time, receipt_time, satellite + except ValueError: + return None + + return None + + def ingest_files(self): + """Scan the directory for new JRR-AOD observation files and insert them into the database.""" + obs_files = glob.glob(os.path.join(self.base_dir, "*.nc")) + print(f"Found {len(obs_files)} new files to ingest") + + records_to_insert = [] + for file in obs_files: + parsed_data = self.parse_filename(file) + if parsed_data: + records_to_insert.append(parsed_data) + + if records_to_insert: + query = """ + INSERT INTO obs_files (filename, obs_time, receipt_time, satellite) + VALUES (?, ?, ?, ?) + """ + self.insert_records(query, records_to_insert) + + +if __name__ == "__main__": + db = JrrAodDatabase(dcom_dir="/home/gvernier/Volumes/hera-s1/runs/realtimeobs/lfs/h1/ops/prod/dcom/") + + # Check for new files + db.ingest_files() + + # Query files for a given DA cycle + da_cycle = "20250316120000" + window_begin = datetime.strptime(da_cycle, "%Y%m%d%H%M%S") - timedelta(hours=3) + window_end = datetime.strptime(da_cycle, "%Y%m%d%H%M%S") + timedelta(hours=3) + + valid_files = db.get_valid_files(window_begin=window_begin, + window_end=window_end) + + print(f"Found {len(valid_files)} valid files for DA cycle {da_cycle}") + for valid_file in valid_files: + if os.path.exists(valid_file): + print(f"Valid file: {valid_file}") + else: + print(f"File does not exist: {valid_file}") diff --git a/ush/python/pyobsforge/obsdb/obsdb.py b/ush/python/pyobsforge/obsdb/obsdb.py new file mode 100644 index 00000000000..ba17a947620 --- /dev/null +++ b/ush/python/pyobsforge/obsdb/obsdb.py @@ -0,0 +1,144 @@ +from logging import getLogger +import sqlite3 +from datetime import datetime, timedelta +from wxflow.sqlitedb import SQLiteDB +from wxflow import FileHandler +from os.path import basename, join + +logger = getLogger(__name__.split('.')[-1]) + + +class BaseDatabase(SQLiteDB): + """Base class for managing different types of file-based databases.""" + + def __init__(self, db_name: str, base_dir: str) -> None: + """ + Initialize the database. + + :param db_name: Name of the SQLite database. + :param base_dir: Directory containing observation files. + """ + super().__init__(db_name) + self.base_dir = base_dir + self.create_database() + + def create_database(self): + """Create the SQLite database. Must be implemented by subclasses.""" + raise NotImplementedError("Subclasses must implement create_database method") + + def get_connection(self): + """Return the database connection.""" + return self.connection + + def parse_filename(self): + """Parse a filename and extract relevant metadata. Must be implemented by subclasses.""" + raise NotImplementedError("Subclasses must implement parse_filename method") + + def ingest_files(self): + """Scan the directory for new observation files and insert them into the database.""" + raise NotImplementedError("Subclasses must implement ingest_files method") + + def insert_record(self, query: str, params: tuple) -> None: + """Insert a record into the database.""" + self.connect() + cursor = self.connection.cursor() + try: + cursor.execute(query, params) + self.connection.commit() + except sqlite3.IntegrityError: + pass # Skip duplicates + finally: + self.disconnect() + + def insert_records(self, query: str, params_list: list[tuple]) -> None: + """ + Insert multiple records into the database. + + :param query: SQL query for inserting records. + :param params_list: List of tuples containing the parameters for each record. + """ + self.connect() + cursor = self.connection.cursor() + try: + cursor.executemany(query, params_list) + self.connection.commit() + except sqlite3.IntegrityError: + pass # Skip duplicates + finally: + self.disconnect() + + def execute_query(self, query: str, params: tuple = None) -> list: + """Execute a query and return the results.""" + self.connect() + cursor = self.connection.cursor() + cursor.execute(query, params or []) + results = cursor.fetchall() + self.disconnect() + return results + + def get_valid_files(self, + window_begin: datetime, + window_end: datetime, + dst_dir: str, + instrument: str = None, + satellite: str = None, + obs_type: str = None, + check_receipt: str = "none") -> list: + """ + Retrieve and copy to dst_dir a list of observation files within a specified time window, possibly filtered by instrument, + satellite, and observation type. The check_receipt parameter can be 'gdas', 'gfs', or 'none'. If 'gdas' or + 'gfs' is specified, files are further filtered based on their receipt time to ensure they meet the + required delay criteria. + + :param window_begin: Start of the time window (datetime object). + :param window_end: End of the time window (datetime object). + :param dst_dir: Destination directory where valid files will be copied. + :param instrument: (Optional) Filter by instrument name. + :param satellite: (Optional) Filter by satellite name. + :param obs_type: (Optional) Filter by observation type. + :param check_receipt: (Optional) Specify receipt time check ('gdas', 'gfs', or 'none'). + :return: List of valid observation file paths in the destination directory. + """ + + query = """ + SELECT filename FROM obs_files + WHERE obs_time BETWEEN ? AND ? + """ + minutes_behind_realtime = {'gdas': 160, 'gfs': 20} + params = [window_begin, window_end] + + if instrument: + query += " AND instrument = ?" + params.append(instrument) + if satellite: + query += " AND satellite = ?" + params.append(satellite) + if obs_type: + query += " AND obs_type = ?" + params.append(obs_type) + + results = self.execute_query(query, tuple(params)) + valid_files = [] + for row in results: + filename = row[0] + if check_receipt in ["gdas", "gfs"]: + query = "SELECT receipt_time FROM obs_files WHERE filename = ?" + receipt_time = self.execute_query(query, (filename,))[0][0] + receipt_time = datetime.strptime(receipt_time, "%Y-%m-%d %H:%M:%S.%f") + if receipt_time <= window_end - timedelta(minutes=minutes_behind_realtime[check_receipt]): + continue + + valid_files.append(filename) + + # Copy files to the destination directory + dst_files = [] + if len(valid_files) > 0: + src_dst_obs_list = [] # list of [src_file, dst_file] + for src_file in valid_files: + dst_file = join(dst_dir, f"{basename(src_file)}") + dst_files.append(dst_file) + src_dst_obs_list.append([src_file, dst_file]) + FileHandler({'mkdir': [dst_dir]}).sync() + FileHandler({'copy': src_dst_obs_list}).sync() + + return dst_files diff --git a/ush/python/pyobsforge/task/__init__.py b/ush/python/pyobsforge/task/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ush/python/pyobsforge/task/aero_prepobs.py b/ush/python/pyobsforge/task/aero_prepobs.py new file mode 100644 index 00000000000..69a40a42c1a --- /dev/null +++ b/ush/python/pyobsforge/task/aero_prepobs.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 + +import glob +import os +from logging import getLogger +from typing import Dict, Any + +from wxflow import (AttrDict, Task, add_to_datetime, to_timedelta, + logit, FileHandler, Executable, save_as_yaml) +from pyobsforge.obsdb.jrr_aod_db import JrrAodDatabase +from pyobsforge.task.run_nc2ioda import run_nc2ioda +import pathlib + +logger = getLogger(__name__.split('.')[-1]) + + +class AerosolObsPrep(Task): + """ + Class for preparing and managing aerosol observations + """ + def __init__(self, config: Dict[str, Any]) -> None: + super().__init__(config) + + _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config['assim_freq']}H") / 2) + _window_end = add_to_datetime(self.task_config.current_cycle, +to_timedelta(f"{self.task_config['assim_freq']}H") / 2) + + local_dict = AttrDict( + { + 'window_begin': _window_begin, + 'window_end': _window_end, + 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", + 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z." + } + ) + + # task_config is everything that this task should need + self.task_config = AttrDict(**self.task_config, **local_dict) + + # Initialize the JRR_AOD database + self.jrr_aod_db = JrrAodDatabase(db_name="jrr_aod_obs.db", + dcom_dir=self.task_config.DCOMROOT, + obs_dir="jrr_aod") + + @logit(logger) + def initialize(self) -> None: + """ + """ + # Update the database with new files + self.jrr_aod_db.ingest_files() + + @logit(logger) + def execute(self) -> None: + """ + """ + for platform in self.task_config.platforms: + logger.info(f"========= platform: {platform}") + input_files = self.jrr_aod_db.get_valid_files(window_begin=self.task_config.window_begin, + window_end=self.task_config.window_end, + dst_dir='jrr_aod', + satellite=platform) + logger.info(f"number of valid files: {len(input_files)}") + + if len(input_files) > 0: + logger.info(f"number of valid files: {len(input_files)}") + obs_space = 'jrr_aod' + platform_out = 'n20' if platform == 'j01' else platform + output_file = f"{self.task_config['RUN']}.t{self.task_config['cyc']:02d}z.retrieval_aod_viirs_{platform_out}.nc" + context = {'provider': 'VIIRSAOD', + 'window_begin': self.task_config.window_begin, + 'window_end': self.task_config.window_end, + 'thinning_threshold': self.task_config.thinning_threshold, + 'preqc': self.task_config.preqc, + 'input_files': input_files, + 'output_file': output_file} + for attr in ['binning_stride', 'binning_min_number_of_obs', 'binning_cressman_radius']: + try: + context[attr] = self.task_config[attr] + except KeyError: + pass + result = run_nc2ioda(self.task_config, obs_space, context) + logger.info(f"run_nc2ioda result: {result}") + + @logit(logger) + def finalize(self) -> None: + """ + """ + # Copy the processed ioda files to the destination directory + logger.info("Copying ioda files to destination COMROOT directory") + comout_obs = self.task_config['COMOUT_OBS'] + + # Loop through the observation types + obs_types = ['viirs'] + src_dst_obs_list = [] # list of [src_file, dst_file] + for obs_type in obs_types: + # Glob the ioda files + ioda_files = glob.glob(os.path.join(self.task_config['DATA'], + f"{self.task_config['OPREFIX']}retrieval_aod_{obs_type}_*.nc")) + for ioda_file in ioda_files: + logger.info(f"ioda_file: {ioda_file}") + src_file = ioda_file + dst_file = os.path.join(comout_obs, os.path.basename(ioda_file)) + src_dst_obs_list.append([src_file, dst_file]) + + logger.info("Copying ioda files to destination COMROOT directory") + logger.info(f"src_dst_obs_list: {src_dst_obs_list}") + + FileHandler({'mkdir': [comout_obs]}).sync() + FileHandler({'copy': src_dst_obs_list}).sync() + + # create a file to tell external processes the obs are ready + ready_file = pathlib.Path(os.path.join(comout_obs, f"{self.task_config['OPREFIX']}obsforge_aod_status.log")) + logger.warning("Creating an empty ready file") + ready_file.touch() diff --git a/ush/python/pyobsforge/task/run_nc2ioda.py b/ush/python/pyobsforge/task/run_nc2ioda.py new file mode 100644 index 00000000000..687fff3fa39 --- /dev/null +++ b/ush/python/pyobsforge/task/run_nc2ioda.py @@ -0,0 +1,37 @@ +from logging import getLogger +from wxflow import Executable, WorkflowException, save_as_yaml, parse_j2yaml +from os.path import join + +logger = getLogger(__name__.split('.')[-1]) + + +def run_nc2ioda(task_config: dict, obs_space: str, context: dict) -> int: + """ + Executes the nc2ioda conversion process using a Jinja2 template and a YAML configuration. + + Args: + task_config (dict): Configuration dictionary containing paths and settings for the task. + obs_space (str): Observation space identifier used to generate file paths. + context (dict): Context dictionary with variables to render the Jinja2 template. + + Returns: + int: Returns 0 upon successful execution. Logs errors and warnings for failures. + """ + jinja_template = join(task_config['HOMEgcafs'], "parm", "chem", "nc2ioda.yaml.j2") + yaml_config = parse_j2yaml(jinja_template, context) + nc2ioda_yaml = join(task_config['DATA'], obs_space, f"{obs_space}_nc2ioda.yaml") + save_as_yaml(yaml_config, nc2ioda_yaml) + + # Run the ioda converter + nc2ioda_exe = join(task_config['HOMEgcafs'], 'exec', 'gdas_obsprovider2ioda.x') + exe = Executable(nc2ioda_exe) + exe.add_default_arg([nc2ioda_yaml]) + try: + logger.debug(f"Executing {exe}") + exe(cwd=task_config['DATA']) + except OSError: + logger.exception(f"Failed to execute {exe}") + raise + except Exception as err: + logger.exception(f"An error occurred during execution of {exe}") + raise WorkflowException(f"An error occurred during execution of {exe}") from err