diff --git a/.gitmodules b/.gitmodules index 5c9e5692434..a241d86d295 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,28 +1,28 @@ [submodule "sorc/ufs_model.fd"] path = sorc/ufs_model.fd - url = https://github.com/ufs-community/ufs-weather-model + url = https://github.com/benjamin-cash/ufs-weather-model ignore = dirty [submodule "sorc/wxflow"] path = sorc/wxflow url = https://github.com/NOAA-EMC/wxflow [submodule "sorc/gfs_utils.fd"] path = sorc/gfs_utils.fd - url = https://github.com/NOAA-EMC/gfs-utils + url = https://github.com/benjamin-cash/gfs-utils [submodule "sorc/ufs_utils.fd"] path = sorc/ufs_utils.fd - url = https://github.com/ufs-community/UFS_UTILS.git + url = https://github.com/benjamin-cash/UFS_UTILS.git [submodule "sorc/verif-global.fd"] path = sorc/verif-global.fd url = https://github.com/NOAA-EMC/EMC_verif-global.git [submodule "sorc/gsi_enkf.fd"] path = sorc/gsi_enkf.fd - url = https://github.com/NOAA-EMC/GSI.git + url = https://github.com/benjamin-cash/GSI.git [submodule "sorc/gdas.cd"] path = sorc/gdas.cd url = https://github.com/NOAA-EMC/GDASApp.git [submodule "sorc/gsi_utils.fd"] path = sorc/gsi_utils.fd - url = https://github.com/NOAA-EMC/GSI-Utils.git + url = https://github.com/benjamin-cash/GSI-Utils.git [submodule "sorc/gsi_monitor.fd"] path = sorc/gsi_monitor.fd - url = https://github.com/NOAA-EMC/GSI-Monitor.git + url = https://github.com/benjamin-cash/GSI-Monitor.git diff --git a/modulefiles/module_base.container.lua b/modulefiles/module_base.container.lua new file mode 100644 index 00000000000..72b11a2b4ea --- /dev/null +++ b/modulefiles/module_base.container.lua @@ -0,0 +1,57 @@ +help([[ +Load environment to run GFS on Hera +]]) + +local spack_mod_path=(os.getenv("spack_mod_path") or "None") +prepend_path("MODULEPATH", spack_mod_path) + +setenv("CC", "icc" ) +setenv("CXX","icpc" ) +setenv("F77","ifort" ) +setenv("F90","ifort" ) +setenv("FC", "ifort" ) +load(pathJoin("stack-intel", (os.getenv("stack_intel_ver") or "None"))) +load(pathJoin("stack-intel-oneapi-mpi", (os.getenv("stack_impi_ver") or "None"))) +--load(pathJoin("python", (os.getenv("python_ver") or "None"))) + +--load(pathJoin("hpss", (os.getenv("hpss_ver") or "None"))) +--load(pathJoin("gempak", (os.getenv("gempak_ver") or "None"))) +--load(pathJoin("ncl", (os.getenv("ncl_ver") or "None"))) +load(pathJoin("jasper", (os.getenv("jasper_ver") or "None"))) +load(pathJoin("libpng", (os.getenv("libpng_ver") or "None"))) +load(pathJoin("cdo", (os.getenv("cdo_ver") or "None"))) +--load(pathJoin("R", (os.getenv("R_ver") or "None"))) +--load(pathJoin("perl", (os.getenv("perl_ver") or "None"))) + +load(pathJoin("hdf5", (os.getenv("hdf5_ver") or "None"))) +load(pathJoin("netcdf-c", (os.getenv("netcdf_c_ver") or "None"))) +load(pathJoin("netcdf-fortran", (os.getenv("netcdf_fortran_ver") or "None"))) + +load(pathJoin("nco", (os.getenv("nco_ver") or "None"))) +load(pathJoin("prod_util", (os.getenv("prod_util_ver") or "None"))) +load(pathJoin("grib-util", (os.getenv("grib_util_ver") or "None"))) +load(pathJoin("g2tmpl", (os.getenv("g2tmpl_ver") or "None"))) +load(pathJoin("gsi-ncdiag", (os.getenv("gsi_ncdiag_ver") or "None"))) +load(pathJoin("crtm", (os.getenv("crtm_ver") or "None"))) +load(pathJoin("bufr", (os.getenv("bufr_ver") or "None"))) +load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None"))) +load(pathJoin("py-f90nml", (os.getenv("py_f90nml_ver") or "None"))) +load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None"))) +load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None"))) +load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None"))) +load(pathJoin("py-pandas", (os.getenv("py_pandas_ver") or "None"))) +load(pathJoin("py-python-dateutil", (os.getenv("py_python_dateutil_ver") or "None"))) +load(pathJoin("met", (os.getenv("met_ver") or "None"))) +load(pathJoin("metplus", (os.getenv("metplus_ver") or "None"))) +load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None"))) + +--setenv("WGRIB2","wgrib2") + +-- Stop gap fix for wgrib with spack-stack 1.6.0 +-- TODO Remove this when spack-stack issue #1097 is resolved +--setenv("WGRIB","wgrib") +--setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) + +--prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) + +whatis("Description: GFS run environment") diff --git a/modulefiles/module_base.frontera.lua b/modulefiles/module_base.frontera.lua new file mode 100644 index 00000000000..07447e64dec --- /dev/null +++ b/modulefiles/module_base.frontera.lua @@ -0,0 +1,6 @@ +help([[ +Minimal environment to run GFS on Frontera via containers +]]) +load("intel/23.1.0") +load("impi/21.9.0") +load("tacc-apptainer") diff --git a/modulefiles/module_base.hopper.lua b/modulefiles/module_base.hopper.lua new file mode 100644 index 00000000000..de1f533513c --- /dev/null +++ b/modulefiles/module_base.hopper.lua @@ -0,0 +1,8 @@ +help([[ +Minimal environment to run GFS on Hopper via containers +]]) + +load("hosts/hopper") +load("OneAPI/2022.1.2") +load("singularity") +load("mpi/2021.5.1") diff --git a/run.container.ver b/run.container.ver new file mode 100644 index 00000000000..4f2958f6400 --- /dev/null +++ b/run.container.ver @@ -0,0 +1,36 @@ +export spack_stack_ver=1.6.0 +export python_ver=3.11.6 + +export jasper_ver=2.0.32 +export libpng_ver=1.6.37 +export cdo_ver=2.3.0 +export nco_ver=5.1.6 + +export hdf5_ver=1.14.3 +export netcdf_c_ver=4.9.2 +export netcdf_fortran_ver=4.6.1 + +export bufr_ver=12.1.0 +export gsi_ncdiag_ver=1.1.2 +export g2tmpl_ver=1.13.0 +export crtm_ver=2.4.0.1 +export wgrib2_ver=3.1.1 +export grib_util_ver=1.4.0 +export prod_util_ver=2.1.1 +export py_netcdf4_ver=1.5.8 +export py_pyyaml_ver=6.0 +export py_jinja2_ver=3.1.2 +export py_pandas_ver=1.5.3 +export py_python_dateutil_ver=2.8.2 +export py_f90nml_ver=1.4.3 +export ip_ver=5.0.0 + +export met_ver=11.1.1 +export metplus_ver=5.1.0 +export py_xarray_ver=2023.7.0 + +export obsproc_run_ver=1.1.2 +export prepobs_run_ver=1.0.1 + +export ens_tracker_ver=feature-GFSv17_com_reorg +export fit2obs_ver=1.1.1 diff --git a/sorc/gfs_utils.fd b/sorc/gfs_utils.fd index bd8f13d8677..99ecd2bb926 160000 --- a/sorc/gfs_utils.fd +++ b/sorc/gfs_utils.fd @@ -1 +1 @@ -Subproject commit bd8f13d867721e4ee28de4af437a0de4283609c3 +Subproject commit 99ecd2bb9267e0365020334746a78d1e59edde61 diff --git a/sorc/gsi_enkf.fd b/sorc/gsi_enkf.fd index 529bb796bea..7cfe53aeda9 160000 --- a/sorc/gsi_enkf.fd +++ b/sorc/gsi_enkf.fd @@ -1 +1 @@ -Subproject commit 529bb796bea0e490f186729cd168a91c034bb12d +Subproject commit 7cfe53aeda97858d67726ace1ab77d593049362f diff --git a/sorc/gsi_monitor.fd b/sorc/gsi_monitor.fd index e1f9f21af16..6650ff272f7 160000 --- a/sorc/gsi_monitor.fd +++ b/sorc/gsi_monitor.fd @@ -1 +1 @@ -Subproject commit e1f9f21af16ce912fdc2cd75c5b27094a550a0c5 +Subproject commit 6650ff272f725ad2974a3528eea84bbd32922c60 diff --git a/sorc/gsi_utils.fd b/sorc/gsi_utils.fd index 9382fd01c2a..07f7b5fb828 160000 --- a/sorc/gsi_utils.fd +++ b/sorc/gsi_utils.fd @@ -1 +1 @@ -Subproject commit 9382fd01c2a626c8934c3f553d420a45de2b4dec +Subproject commit 07f7b5fb828db2cdfa5d9d5b192a4ab59e74608b diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 92404afc013..b92ce561980 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -77,6 +77,9 @@ case "${machine}" in "s4") FIX_DIR="/data/prod/glopara/fix" ;; "gaea") FIX_DIR="/gpfs/f5/ufs-ard/world-shared/global/glopara/data/fix" ;; "noaacloud") FIX_DIR="/contrib/global-workflow-shared-data/fix" ;; + "frontera") FIX_DIR="/work2/02441/bcash/frontera/ufs_input/fix" ;; + "container") FIX_DIR="/groups/BCASH/ufs_input/fix" ;; + "hopper") FIX_DIR="/groups/BCASH/ufs_input/fix" ;; *) echo "FATAL: Unknown target machine ${machine}, couldn't set FIX_DIR" exit 1 diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index fcc9f8461db..dde7bf75326 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit fcc9f8461db5eafbfd1f080da61ea79156ca0145 +Subproject commit dde7bf75326a15b72ed6a74ad40249ea70647cb2 diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd index 3ef2e6bd725..9b897bc3551 160000 --- a/sorc/ufs_utils.fd +++ b/sorc/ufs_utils.fd @@ -1 +1 @@ -Subproject commit 3ef2e6bd725d2662fd6ee95897cb7bac222e5144 +Subproject commit 9b897bc3551638125c945831c66d817924e63824 diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index b049a6040ef..2e7c7287a77 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -38,7 +38,8 @@ case $(hostname -f) in [Hh]ercules-login-[1-4].[Hh][Pp][Cc].[Mm]s[Ss]tate.[Ee]du) MACHINE_ID=hercules ;; ### hercules1-4 login[1-4].stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede1-4 - + *frontera.tacc.utexas.edu) MACHINE_ID=frontera ;; + *orc.gmu.edu) MACHINE_ID=hopper ;; login0[1-2].expanse.sdsc.edu) MACHINE_ID=expanse ;; ### expanse1-2 discover3[1-5].prv.cube) MACHINE_ID=discover ;; ### discover31-35 @@ -61,7 +62,10 @@ if [[ "${MACHINE_ID}" != "UNKNOWN" ]]; then fi # Try searching based on paths since hostname may not match on compute nodes -if [[ -d /lfs/h3 ]]; then +if [[ -d /opt/container-scripts ]]; then + # We are in the container + MACHINE_ID=container +elif [[ -d /lfs/h3 ]]; then # We are on NOAA Cactus or Dogwood MACHINE_ID=wcoss2 elif [[ -d /lfs/h1 && ! -d /lfs/h3 ]]; then diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh index ff6f64cece0..eb7bc959f72 100755 --- a/ush/load_fv3gfs_modules.sh +++ b/ush/load_fv3gfs_modules.sh @@ -20,7 +20,7 @@ source "${HOMEgfs}/versions/run.ver" module use "${HOMEgfs}/modulefiles" case "${MACHINE_ID}" in - "wcoss2" | "hera" | "orion" | "hercules" | "gaea" | "jet" | "s4" | "noaacloud") + "wcoss2" | "hera" | "orion" | "hercules" | "gaea" | "jet" | "s4" | "noaacloud" | "container" | "frontera" | "hopper") module load "module_base.${MACHINE_ID}" ;; *) diff --git a/ush/module-setup.sh b/ush/module-setup.sh index 398562652d8..80e18410130 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -3,7 +3,11 @@ set -u source "${HOMEgfs}/ush/detect_machine.sh" -if [[ ${MACHINE_ID} = jet* ]] ; then +if [[ ${MACHINE_ID} = container ]] ; then + source /usr/lmod/lmod/init/bash + module purge + +elif [[ ${MACHINE_ID} = jet* ]] ; then # We are on NOAA Jet if ( ! eval module help > /dev/null 2>&1 ) ; then source /apps/lmod/lmod/init/bash diff --git a/versions/build.container.ver b/versions/build.container.ver new file mode 100644 index 00000000000..7a2b50464b9 --- /dev/null +++ b/versions/build.container.ver @@ -0,0 +1,10 @@ +export stack_intel_ver=2021.10.0 +export stack_impi_ver=2021.12.1 +export spack_env=global-workflow-env +source "${HOMEgfs:-}/versions/run.container.ver" +export spack_mod_path="/opt/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" +export I_MPI_CC=$CC +export I_MPI_CXX=$CXX +export I_MPI_F77=$FC +export I_MPI_FC=$FC +export I_MPI_F90=$FC diff --git a/versions/run.container.ver b/versions/run.container.ver new file mode 100644 index 00000000000..a2a66d50419 --- /dev/null +++ b/versions/run.container.ver @@ -0,0 +1,43 @@ +export spack_stack_ver=1.6.0 +export python_ver=3.11.6 + +export jasper_ver=2.0.32 +export libpng_ver=1.6.37 +export cdo_ver=2.3.0 +export nco_ver=5.1.6 +export esmf_ver=8.6.1 +export fms_ver=2024.02 +export mapl_ver=2.46.3-esmf-8.6.1 +export gftl_shared_ver=1.9.0 + +export hdf5_ver=1.14.3 +export netcdf_c_ver=4.9.2 +export netcdf_fortran_ver=4.6.1 + +export bufr_ver=12.1.0 +export gsi_ncdiag_ver=1.1.2 +export g2tmpl_ver=1.13.0 +export g2_ver=3.5.1 +export cmake_ver=3.27.9 +export crtm_ver=2.4.0.1 +export wgrib2_ver=3.1.1 +export grib_util_ver=1.4.0 +export prod_util_ver=2.1.1 +export py_netcdf4_ver=1.5.8 +export py_pyyaml_ver=6.0 +export py_jinja2_ver=3.1.2 +export parallelio_ver=2.6.2 +export py_pandas_ver=1.5.3 +export py_python_dateutil_ver=2.8.2 +export py_f90nml_ver=1.4.3 +export ip_ver=5.0.0 + +export met_ver=11.1.1 +export metplus_ver=5.1.0 +export py_xarray_ver=2023.7.0 + +export obsproc_run_ver=1.1.2 +export prepobs_run_ver=1.0.1 + +export ens_tracker_ver=feature-GFSv17_com_reorg +export fit2obs_ver=1.1.1 diff --git a/versions/spack.ver b/versions/spack.ver new file mode 100644 index 00000000000..cbe2f9b3aca --- /dev/null +++ b/versions/spack.ver @@ -0,0 +1,45 @@ +export spack_stack_ver=1.8.0 +export python_ver=3.11.6 +export stack_intel_ver=2021.10.0 +export stack_impi_ver=2021.12.1 + +export jasper_ver=2.0.32 +export libpng_ver=1.6.37 +export cdo_ver=2.3.0 +export nco_ver=5.1.6 +export esmf_ver=8.6.1 +export fms_ver=2024.02 +export mapl_ver=2.46.3-esmf-8.6.1 +export gftl_shared_ver=1.9.0 + +export hdf5_ver=1.14.3 +export netcdf_c_ver=4.9.2 +export netcdf_fortran_ver=4.6.1 + +export bufr_ver=12.1.0 +export gsi_ncdiag_ver=1.1.2 +export g2tmpl_ver=1.13.0 +export g2_ver=3.5.1 +export cmake_ver=3.27.9 +export crtm_ver=2.4.0.1 +export wgrib2_ver=3.1.1 +export grib_util_ver=1.4.0 +export prod_util_ver=2.1.1 +export py_netcdf4_ver=1.5.8 +export py_pyyaml_ver=6.0 +export py_jinja2_ver=3.1.2 +export parallelio_ver=2.6.2 +export py_pandas_ver=1.5.3 +export py_python_dateutil_ver=2.8.2 +export py_f90nml_ver=1.4.3 +export ip_ver=5.0.0 + +export met_ver=11.1.1 +export metplus_ver=5.1.0 +export py_xarray_ver=2023.7.0 + +export obsproc_run_ver=1.1.2 +export prepobs_run_ver=1.0.1 + +export ens_tracker_ver=feature-GFSv17_com_reorg +export fit2obs_ver=1.1.1 diff --git a/workflow/hosts.py b/workflow/hosts.py index 34ea067adea..a4c590f2cbc 100644 --- a/workflow/hosts.py +++ b/workflow/hosts.py @@ -17,7 +17,8 @@ class Host: SUPPORTED_HOSTS = ['HERA', 'ORION', 'JET', 'HERCULES', 'WCOSS2', 'S4', 'CONTAINER', 'GAEA', - 'AWSPW', 'AZUREPW', 'GOOGLEPW'] + 'AWSPW', 'AZUREPW', 'GOOGLEPW', + 'HOPPER', 'FRONTERA'] def __init__(self, host=None):