Update for New Base: Debian Python 3.12

.gitignore

@@ -211,3 +211,7 @@ __marimo__/
 /tmp/
 /bin_mounted/*_tests_results*.json
 /.devcontainer/tmp/
+
+# Outputs of full regionalization workflows (e.g. testing parreg -> ngen to produce regionalization files)
+/outputs/
+/run_time_*/

Dockerfile.rte

@@ -3,6 +3,8 @@ ARG GH_ORG
 ARG NGEN_BASE_IMAGE=ghcr.io/${GH_ORG,,}/ngen:latest
 FROM $NGEN_BASE_IMAGE AS ngen_rte_base
 ARG GH_ORG
+# e.g. "3.12"
+ARG PYTHON_VERSION_STR
 
 # OCI Metadata Arguments
 ARG TARGET_IMAGE_DESCRIPTION="Docker image for the National Water Model Runtime Environment (NWM RTE)."
@@ -25,9 +27,8 @@ LABEL org.opencontainers.image.description=$TARGET_IMAGE_DESCRIPTION \
 ### and log level is controlled by `./bin_mounted/ngen_logging.json`.
 # ENV FORCING_LOGLEVEL=DEBUG
 
-### Static args
+### Static args/vars
 ARG PYTHON_ENV="/ngen-app/ngen-python/bin"
-ARG PYTHON_VERSION_STR="python3.11"
 ENV PIP_NO_CACHE_DIR=1
 ENV OMPI_ALLOW_RUN_AS_ROOT=1
 ENV OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
@@ -44,7 +45,7 @@ RUN python -m pip install --upgrade pip
 ### Install debuggers if specified
 ARG INSTALL_DEBUGGERS
 RUN --mount=type=bind,src=nwm-rte/install_debuggers.sh,dst=/src/${GH_ORG}/nwm-rte/install_debuggers.sh \
-    "/src/${GH_ORG}/nwm-rte/install_debuggers.sh" "${INSTALL_DEBUGGERS}"
+    "/src/${GH_ORG}/nwm-rte/install_debuggers.sh" "${INSTALL_DEBUGGERS}" "${PYTHON_VERSION_STR}"
 
 
 ### Install nwm packages
@@ -98,14 +99,17 @@ RUN --mount=type=bind,src=nwm-rte/install_package.sh,dst=/src/${GH_ORG}/nwm-rte/
 #### BEGIN SECOND STAGE: Adding a separate Python venv for nwm-eval-mgr (i.e., nwm_eval and nwm_metrics)
 FROM ngen_rte_base AS ngen_rte_eval_verf
 
+# Install Python 3.11 (required as of 2026-07-23 for the pinned version of teehr which pins duckdb to a version not compatible with Python 3.12)
+RUN apt-get update && apt-get install -y --no-install-recommends python3.11 python3.11-venv
+
 # Call modules like `python_eval_verf -m nwm_eval`
 ARG EVAL_VERF_PYTHON_ALIAS="python_eval_verf"
 ARG EVAL_VERF_VENV="/ngen-app/venvs/eval_verf"
 ARG EVAL_VERF_PYTHON_BIN_DIR="${EVAL_VERF_VENV}/bin"
 ARG EVAL_VERF_PYTHON_BIN_FILE="${EVAL_VERF_PYTHON_BIN_DIR}/python"
 
 # Make the venv
-RUN python -m venv "${EVAL_VERF_VENV}"
+RUN python3.11 -m venv "${EVAL_VERF_VENV}"
 # Add an alias for convenience
 RUN echo "alias ${EVAL_VERF_PYTHON_ALIAS}=\"${EVAL_VERF_PYTHON_BIN_FILE}\"" >> ~/.bashrc
 
@@ -114,7 +118,7 @@ ENV PATH_SAVED=${PATH}
 ENV PYTHONPATH_SAVED=${PYTHONPATH}
 # Adjust PATH and PYTHONPATH to give this environment precedent
 ENV PATH="${EVAL_VERF_PYTHON_BIN_DIR}:${PATH}"
-ENV PYTHONPATH="${EVAL_VERF_PYTHON_BIN_DIR}/lib/${PYTHON_VERSION_STR}/site-packages"
+ENV PYTHONPATH="${EVAL_VERF_PYTHON_BIN_DIR}/lib/python${PYTHON_VERSION_STR}/site-packages"
 
 ARG REPO_TAG_EVAL
 RUN --mount=type=bind,src=nwm-rte/install_package.sh,dst=/src/${GH_ORG}/nwm-rte/install_package.sh \

bin_mounted/ngen_rte/consts.py

@@ -12,6 +12,8 @@
     Objective as CalObjective,
 )
 
+PYTHON_VERSION_STRING = "3.12"
+"""Python version string, e.g. "3.12". If this is changed here, it should also be changed in the build code in config.bashrc"""
 CONTAINER_LOGS_DIR = "/ngen-app/logs_rte"
 
 RUN_NAME_TIMESTAMP_SUFFIX_FORMAT = r"%Y%m%d-%H%M%S-%f"
@@ -21,7 +23,7 @@
 
 ### .config section [Forcing]
 DEFAULT_FORECAST_RUN_NAME = "fcst_run1"
-FORCING_TEMPLATE_DIR = "/ngen-app/ngen-python/lib/python3.11/site-packages/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/"
+FORCING_TEMPLATE_DIR = f"/ngen-app/ngen-python/lib/python{PYTHON_VERSION_STRING}/site-packages/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/"
 """Directory of forcing configuration template yaml files."""
 FORCING_ROOT_DIR = "/ngen-app/data"
 DIR_FORCING_RAW_INPUT = os.path.join(FORCING_ROOT_DIR, "raw_input")
@@ -65,9 +67,7 @@
 VALID_EVAL_CURTAILMENT_DEFAULT = timedelta(hours=0)  # Gets subtracted
 
 ### .config section [DataFile]
-MODULE_PARAMETER_FILES_DIR = (
-    "/ngen-app/ngen-python/lib/python3.11/site-packages/mswm/module_parameter_files"
-)
+MODULE_PARAMETER_FILES_DIR = f"/ngen-app/ngen-python/lib/python{PYTHON_VERSION_STRING}/site-packages/mswm/module_parameter_files"
 NGEN_DIR = "/ngen-app/ngen"
 NWM_RETRO_STREAMFLOW_DIR = f"{DEFAULT_MAIN_DIR}/data/nwm_retrospective"
 

config.bashrc

@@ -26,7 +26,7 @@ NO_CACHE=${NO_CACHE:-""}
 # STAGE=${STAGE:-"ngen_rte_base"}
 STAGE=${STAGE:-"ngen_rte_eval_verf"}
 
-## \env INSTALL_DEBUGGERS Passed to ./install_debuggers.sh, causes pip package `debugpy` and dnf package `gdb` to be installed. Choose from: `["NO", "YES"]`
+## \env INSTALL_DEBUGGERS Passed to ./install_debuggers.sh, causes pip package `debugpy` and OS package `gdb` to be installed. Choose from: `["NO", "YES"]`
 INSTALL_DEBUGGERS=${INSTALL_DEBUGGERS:-"NO"}
 # INSTALL_DEBUGGERS=${INSTALL_DEBUGGERS:-"YES"}
 
@@ -102,6 +102,9 @@ TARGET_IMAGE_NAME=${TARGET_IMAGE_NAME:-"ngen_rte_${NGEN_SOURCE_MODE}"}
 
 #### Misc
 
+## \env Python version string. If this is changed here, it should also be changed in the Python code in consts.py.
+PYTHON_VERSION_STR="3.12"
+
 # OCI Standard labels for Dockerfile.rte image
 # See https://specs.opencontainers.org/image-spec/annotations/
 TARGET_IMAGE_SOURCE=${TARGET_IMAGE_SOURCE:-"https://github.com/${GH_ORG}/nwm-rte"}
@@ -142,7 +145,7 @@ MNT__NWM_REGION_MGR__INPUT_DATA="${REPOS_COMMON_ROOT__HOST}/nwm-region-mgr/data/
 
 ##### Installed regionalization results
 ## \env INSTALLED_REGIONALIZATION_RESULTS Results of regionalization, baked into the MSWM Python package, which can be used for testing the regionalized form of ngen forecasts.
-INSTALLED_REGIONALIZATION_RESULTS=/ngen-app/ngen-python/lib/python3.11/site-packages/mswm/example_inputs/regionalization
+INSTALLED_REGIONALIZATION_RESULTS=/ngen-app/ngen-python/lib/python${PYTHON_VERSION_STR}/site-packages/mswm/example_inputs/regionalization
 
 ### Remote data sources for setup_data.sh and setup_data_one_gage.sh
 ## \env SOURCE_BUCKET_DEV Name of cloud bucket (no s3:// prefix in the string), used by setup_data.sh and setup_data_one_gage.sh

install_debuggers.sh

@@ -7,7 +7,7 @@ set -euo pipefail
 ## Install debugger packages. Used by `Dockerfile.rte`.
 ## 
 ## \desc
-## Install debugger packages: python package `debugpy` and dnf package `gdb`. Used by `Dockerfile.rte`.
+## Install debugger packages: python package `debugpy` and OS package `gdb`. Used by `Dockerfile.rte`.
 ## Optionally skip installation (noop). The choice of noop is to abstract the flow control away into
 ## a parameter to work around the non-branching nature of Docker build flow control.
 ## 
@@ -21,16 +21,27 @@ set -euo pipefail
 ## 
 
 install_debuggers=$1
+python_version=${2:-}
 
 if [ "$install_debuggers" = "YES" ]; then
     echo "Installing debugpy via pip"
     pip install debugpy
-    echo "Installing gdb via dnf"
-    dnf install -y gdb
-    yum install yum-utils -y
-    yum-config-manager --enable baseos-debug
-    # TODO parameterize or auto-determine Python version
-    debuginfo-install python3.11
+    echo "Installing gdb"
+
+    if grep -q '^ID=rocky' /etc/os-release; then
+        echo "Rocky detected. Installing gdb and Python debug symbols"
+        dnf install -y gdb
+        yum install yum-utils -y
+        yum-config-manager --enable baseos-debug
+        debuginfo-install python${python_version}
+    elif grep -q '^ID=debian' /etc/os-release; then
+        echo "Debian detected. Installing gdb. Python debug symbols will not be installed."
+        apt-get update
+        apt-get install -y gdb libc6-dbg
+    else
+        echo "Error: unexpected OS: `grep ^ID /etc/os-release`"
+        exit 1
+    fi
 
 elif [ "$install_debuggers" = "NO" ]; then
     echo "Not installing debuggers"

ngen_rte_build.sh

@@ -155,6 +155,7 @@ SAFE_LOG_NAME=$(echo "${TARGET_IMAGE_NAME}" | tr '/:' '__')
 
 sudo docker build -t ${TARGET_IMAGE_NAME} -f Dockerfile.rte ${NO_CACHE} --target ${STAGE} \
     --build-arg GH_ORG=${GH_ORG} \
+    --build-arg PYTHON_VERSION_STR=${PYTHON_VERSION_STR} \
     --build-arg TARGET_IMAGE_SOURCE=${TARGET_IMAGE_SOURCE} \
     --build-arg TARGET_IMAGE_VENDOR=${TARGET_IMAGE_VENDOR} \
     --build-arg TARGET_IMAGE_VERSION=${TARGET_IMAGE_VERSION} \

run_suite.sh

@@ -25,11 +25,11 @@ set -x
 ## \usage ./run_suite.sh
 ## 
 
-# CONUS NWM calibration using a short duration
+# CONUS NWM calibration using a shorter duration
 # sudo rm -rf ~/ngwpc/run_ngen/kge_dds/test_bmi/${TEST_GAGE}
-docker_run python -um "ngen_rte.run_calibration" -n 2 -fconfig "nwm" -start "2013-07-25 00:00:00" -dur 2
+# docker_run python -um "ngen_rte.run_calibration" -n 2 -fconfig "nwm" -start "2013-07-25 00:00:00" -dur 2
 
-# CONUS AORC calibration using a long duration
+# CONUS AORC calibration using a longer duration
 # sudo rm -rf ~/ngwpc/run_ngen/kge_dds/test_bmi/${TEST_GAGE}
 docker_run python -um "ngen_rte.run_calibration" -n 2 -fconfig "aorc" -start "2013-07-01 00:00:00" -dur 120