From 0e797852500056acf1ff15fd6b4d22316dac8111 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Thu, 23 Jul 2026 06:15:32 -0400 Subject: [PATCH 1/4] Add support for Debian. Switch Python version from 3.11 to 3.12. --- Dockerfile.rte | 9 +++++---- bin_mounted/ngen_rte/consts.py | 8 ++++---- config.bashrc | 7 +++++-- install_debuggers.sh | 25 ++++++++++++++++++------- ngen_rte_build.sh | 1 + 5 files changed, 33 insertions(+), 17 deletions(-) diff --git a/Dockerfile.rte b/Dockerfile.rte index ef7b9a7..6a7df86 100644 --- a/Dockerfile.rte +++ b/Dockerfile.rte @@ -3,6 +3,8 @@ ARG GH_ORG ARG NGEN_BASE_IMAGE=ghcr.io/${GH_ORG,,}/ngen:latest FROM $NGEN_BASE_IMAGE AS ngen_rte_base ARG GH_ORG +# e.g. "3.12" +ARG PYTHON_VERSION_STR # OCI Metadata Arguments ARG TARGET_IMAGE_DESCRIPTION="Docker image for the National Water Model Runtime Environment (NWM RTE)." @@ -25,9 +27,8 @@ LABEL org.opencontainers.image.description=$TARGET_IMAGE_DESCRIPTION \ ### and log level is controlled by `./bin_mounted/ngen_logging.json`. # ENV FORCING_LOGLEVEL=DEBUG -### Static args +### Static args/vars ARG PYTHON_ENV="/ngen-app/ngen-python/bin" -ARG PYTHON_VERSION_STR="python3.11" ENV PIP_NO_CACHE_DIR=1 ENV OMPI_ALLOW_RUN_AS_ROOT=1 ENV OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 @@ -44,7 +45,7 @@ RUN python -m pip install --upgrade pip ### Install debuggers if specified ARG INSTALL_DEBUGGERS RUN --mount=type=bind,src=nwm-rte/install_debuggers.sh,dst=/src/${GH_ORG}/nwm-rte/install_debuggers.sh \ - "/src/${GH_ORG}/nwm-rte/install_debuggers.sh" "${INSTALL_DEBUGGERS}" + "/src/${GH_ORG}/nwm-rte/install_debuggers.sh" "${INSTALL_DEBUGGERS}" "${PYTHON_VERSION_STR}" ### Install nwm packages @@ -114,7 +115,7 @@ ENV PATH_SAVED=${PATH} ENV PYTHONPATH_SAVED=${PYTHONPATH} # Adjust PATH and PYTHONPATH to give this environment precedent ENV PATH="${EVAL_VERF_PYTHON_BIN_DIR}:${PATH}" -ENV PYTHONPATH="${EVAL_VERF_PYTHON_BIN_DIR}/lib/${PYTHON_VERSION_STR}/site-packages" +ENV PYTHONPATH="${EVAL_VERF_PYTHON_BIN_DIR}/lib/python${PYTHON_VERSION_STR}/site-packages" ARG REPO_TAG_EVAL RUN --mount=type=bind,src=nwm-rte/install_package.sh,dst=/src/${GH_ORG}/nwm-rte/install_package.sh \ diff --git a/bin_mounted/ngen_rte/consts.py b/bin_mounted/ngen_rte/consts.py index 562b90c..d73f1f8 100644 --- a/bin_mounted/ngen_rte/consts.py +++ b/bin_mounted/ngen_rte/consts.py @@ -12,6 +12,8 @@ Objective as CalObjective, ) +PYTHON_VERSION_STRING = "3.12" +"""Python version string, e.g. "3.12". If this is changed here, it should also be changed in the build code in config.bashrc""" CONTAINER_LOGS_DIR = "/ngen-app/logs_rte" RUN_NAME_TIMESTAMP_SUFFIX_FORMAT = r"%Y%m%d-%H%M%S-%f" @@ -21,7 +23,7 @@ ### .config section [Forcing] DEFAULT_FORECAST_RUN_NAME = "fcst_run1" -FORCING_TEMPLATE_DIR = "/ngen-app/ngen-python/lib/python3.11/site-packages/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/" +FORCING_TEMPLATE_DIR = f"/ngen-app/ngen-python/lib/python{PYTHON_VERSION_STRING}/site-packages/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/" """Directory of forcing configuration template yaml files.""" FORCING_ROOT_DIR = "/ngen-app/data" DIR_FORCING_RAW_INPUT = os.path.join(FORCING_ROOT_DIR, "raw_input") @@ -65,9 +67,7 @@ VALID_EVAL_CURTAILMENT_DEFAULT = timedelta(hours=0) # Gets subtracted ### .config section [DataFile] -MODULE_PARAMETER_FILES_DIR = ( - "/ngen-app/ngen-python/lib/python3.11/site-packages/mswm/module_parameter_files" -) +MODULE_PARAMETER_FILES_DIR = f"/ngen-app/ngen-python/lib/python{PYTHON_VERSION_STRING}/site-packages/mswm/module_parameter_files" NGEN_DIR = "/ngen-app/ngen" NWM_RETRO_STREAMFLOW_DIR = f"{DEFAULT_MAIN_DIR}/data/nwm_retrospective" diff --git a/config.bashrc b/config.bashrc index 91eff91..4b9af69 100644 --- a/config.bashrc +++ b/config.bashrc @@ -26,7 +26,7 @@ NO_CACHE=${NO_CACHE:-""} # STAGE=${STAGE:-"ngen_rte_base"} STAGE=${STAGE:-"ngen_rte_eval_verf"} -## \env INSTALL_DEBUGGERS Passed to ./install_debuggers.sh, causes pip package `debugpy` and dnf package `gdb` to be installed. Choose from: `["NO", "YES"]` +## \env INSTALL_DEBUGGERS Passed to ./install_debuggers.sh, causes pip package `debugpy` and OS package `gdb` to be installed. Choose from: `["NO", "YES"]` INSTALL_DEBUGGERS=${INSTALL_DEBUGGERS:-"NO"} # INSTALL_DEBUGGERS=${INSTALL_DEBUGGERS:-"YES"} @@ -102,6 +102,9 @@ TARGET_IMAGE_NAME=${TARGET_IMAGE_NAME:-"ngen_rte_${NGEN_SOURCE_MODE}"} #### Misc +## \env Python version string. If this is changed here, it should also be changed in the Python code in consts.py. +PYTHON_VERSION_STR="3.12" + # OCI Standard labels for Dockerfile.rte image # See https://specs.opencontainers.org/image-spec/annotations/ TARGET_IMAGE_SOURCE=${TARGET_IMAGE_SOURCE:-"https://github.com/${GH_ORG}/nwm-rte"} @@ -142,7 +145,7 @@ MNT__NWM_REGION_MGR__INPUT_DATA="${REPOS_COMMON_ROOT__HOST}/nwm-region-mgr/data/ ##### Installed regionalization results ## \env INSTALLED_REGIONALIZATION_RESULTS Results of regionalization, baked into the MSWM Python package, which can be used for testing the regionalized form of ngen forecasts. -INSTALLED_REGIONALIZATION_RESULTS=/ngen-app/ngen-python/lib/python3.11/site-packages/mswm/example_inputs/regionalization +INSTALLED_REGIONALIZATION_RESULTS=/ngen-app/ngen-python/lib/python${PYTHON_VERSION_STR}/site-packages/mswm/example_inputs/regionalization ### Remote data sources for setup_data.sh and setup_data_one_gage.sh ## \env SOURCE_BUCKET_DEV Name of cloud bucket (no s3:// prefix in the string), used by setup_data.sh and setup_data_one_gage.sh diff --git a/install_debuggers.sh b/install_debuggers.sh index 1639ff3..30552e6 100755 --- a/install_debuggers.sh +++ b/install_debuggers.sh @@ -7,7 +7,7 @@ set -euo pipefail ## Install debugger packages. Used by `Dockerfile.rte`. ## ## \desc -## Install debugger packages: python package `debugpy` and dnf package `gdb`. Used by `Dockerfile.rte`. +## Install debugger packages: python package `debugpy` and OS package `gdb`. Used by `Dockerfile.rte`. ## Optionally skip installation (noop). The choice of noop is to abstract the flow control away into ## a parameter to work around the non-branching nature of Docker build flow control. ## @@ -21,16 +21,27 @@ set -euo pipefail ## install_debuggers=$1 +python_version=${2:-} if [ "$install_debuggers" = "YES" ]; then echo "Installing debugpy via pip" pip install debugpy - echo "Installing gdb via dnf" - dnf install -y gdb - yum install yum-utils -y - yum-config-manager --enable baseos-debug - # TODO parameterize or auto-determine Python version - debuginfo-install python3.11 + echo "Installing gdb" + + if grep -q '^ID=rocky' /etc/os-release; then + echo "Rocky detected. Installing gdb and Python debug symbols" + dnf install -y gdb + yum install yum-utils -y + yum-config-manager --enable baseos-debug + debuginfo-install python${python_version} + elif grep -q '^ID=debian' /etc/os-release; then + echo "Debian detected. Installing gdb. Python debug symbols will not be installed." + apt-get update + apt-get install -y gdb libc6-dbg + else + echo "Error: unexpected OS: `grep ^ID /etc/os-release`" + exit 1 + fi elif [ "$install_debuggers" = "NO" ]; then echo "Not installing debuggers" diff --git a/ngen_rte_build.sh b/ngen_rte_build.sh index 1fe8582..aabadc7 100755 --- a/ngen_rte_build.sh +++ b/ngen_rte_build.sh @@ -155,6 +155,7 @@ SAFE_LOG_NAME=$(echo "${TARGET_IMAGE_NAME}" | tr '/:' '__') sudo docker build -t ${TARGET_IMAGE_NAME} -f Dockerfile.rte ${NO_CACHE} --target ${STAGE} \ --build-arg GH_ORG=${GH_ORG} \ + --build-arg PYTHON_VERSION_STR=${PYTHON_VERSION_STR} \ --build-arg TARGET_IMAGE_SOURCE=${TARGET_IMAGE_SOURCE} \ --build-arg TARGET_IMAGE_VENDOR=${TARGET_IMAGE_VENDOR} \ --build-arg TARGET_IMAGE_VERSION=${TARGET_IMAGE_VERSION} \ From 212d2fade9e2f2ac68caf0367cd3bcedc3813a68 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Thu, 23 Jul 2026 06:31:44 -0400 Subject: [PATCH 2/4] Example calls: comment-out NWM-forcing calibration --- run_suite.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/run_suite.sh b/run_suite.sh index bdcd50c..69edb07 100755 --- a/run_suite.sh +++ b/run_suite.sh @@ -25,11 +25,11 @@ set -x ## \usage ./run_suite.sh ## -# CONUS NWM calibration using a short duration +# CONUS NWM calibration using a shorter duration # sudo rm -rf ~/ngwpc/run_ngen/kge_dds/test_bmi/${TEST_GAGE} -docker_run python -um "ngen_rte.run_calibration" -n 2 -fconfig "nwm" -start "2013-07-25 00:00:00" -dur 2 +# docker_run python -um "ngen_rte.run_calibration" -n 2 -fconfig "nwm" -start "2013-07-25 00:00:00" -dur 2 -# CONUS AORC calibration using a long duration +# CONUS AORC calibration using a longer duration # sudo rm -rf ~/ngwpc/run_ngen/kge_dds/test_bmi/${TEST_GAGE} docker_run python -um "ngen_rte.run_calibration" -n 2 -fconfig "aorc" -start "2013-07-01 00:00:00" -dur 120 From da3eef4e5b6f1fa176a4980a00f1ab7c2290991b Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Thu, 23 Jul 2026 10:20:42 -0400 Subject: [PATCH 3/4] Use Python 3.11 for the 2nd stage venv (its pinned version of teehr is not compatible with Python 3.12) --- Dockerfile.rte | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile.rte b/Dockerfile.rte index 6a7df86..3d20f37 100644 --- a/Dockerfile.rte +++ b/Dockerfile.rte @@ -99,6 +99,9 @@ RUN --mount=type=bind,src=nwm-rte/install_package.sh,dst=/src/${GH_ORG}/nwm-rte/ #### BEGIN SECOND STAGE: Adding a separate Python venv for nwm-eval-mgr (i.e., nwm_eval and nwm_metrics) FROM ngen_rte_base AS ngen_rte_eval_verf +# Install Python 3.11 (required as of 2026-07-23 for the pinned version of teehr which pins duckdb to a version not compatible with Python 3.12) +RUN apt-get update && apt-get install -y --no-install-recommends python3.11 python3.11-venv + # Call modules like `python_eval_verf -m nwm_eval` ARG EVAL_VERF_PYTHON_ALIAS="python_eval_verf" ARG EVAL_VERF_VENV="/ngen-app/venvs/eval_verf" @@ -106,7 +109,7 @@ ARG EVAL_VERF_PYTHON_BIN_DIR="${EVAL_VERF_VENV}/bin" ARG EVAL_VERF_PYTHON_BIN_FILE="${EVAL_VERF_PYTHON_BIN_DIR}/python" # Make the venv -RUN python -m venv "${EVAL_VERF_VENV}" +RUN python3.11 -m venv "${EVAL_VERF_VENV}" # Add an alias for convenience RUN echo "alias ${EVAL_VERF_PYTHON_ALIAS}=\"${EVAL_VERF_PYTHON_BIN_FILE}\"" >> ~/.bashrc From 6f12d8a792aa29134689985266e0c983419a30ef Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Fri, 24 Jul 2026 07:21:46 -0400 Subject: [PATCH 4/4] Add regionalization outputs to .gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 034d14d..4ae1705 100644 --- a/.gitignore +++ b/.gitignore @@ -211,3 +211,7 @@ __marimo__/ /tmp/ /bin_mounted/*_tests_results*.json /.devcontainer/tmp/ + +# Outputs of full regionalization workflows (e.g. testing parreg -> ngen to produce regionalization files) +/outputs/ +/run_time_*/