PYTHON-6091 Pin a consistent uv binary version in CI and locally (#3053)

Author: blink1073

.evergreen/scripts/configure-env.sh

@@ -14,30 +14,53 @@ fi
 PROJECT_DIRECTORY="$(pwd)"
 DRIVERS_TOOLS="$(dirname $PROJECT_DIRECTORY)/drivers-tools"
 CARGO_HOME=${CARGO_HOME:-${DRIVERS_TOOLS}/.cargo}
-UV_TOOL_DIR=$PROJECT_DIRECTORY/.local/uv/tools
-UV_CACHE_DIR=$PROJECT_DIRECTORY/.local/uv/cache
 DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS/.bin"
 MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin"
 
-# On Evergreen jobs, "CI" will be set, and we don't want to write to $HOME.
+# On Evergreen jobs, "CI" will be set, and we don't want to write to $HOME or
+# have binaries shared across tasks, so use a TMPDIR. On non-CI hosts
+# (spawn hosts, VMs such as GCP/Azure, and local dev), use the conventional
+# ~/.local/bin which tools on the PATH (or the shell rc) can find.
 if [ "${CI:-}" == "true" ]; then
-  PYMONGO_BIN_DIR=${DRIVERS_TOOLS_BINARIES:-}
-# We want to use a path that's already on PATH on spawn hosts.
+  PYMONGO_BIN_DIR="${TMPDIR:-/tmp}"/pymongo_bin
 else
-  PYMONGO_BIN_DIR=$HOME/cli_bin
+  PYMONGO_BIN_DIR=$HOME/.local/bin
 fi
 
-PATH_EXT="$MONGODB_BINARIES:$DRIVERS_TOOLS_BINARIES:$PYMONGO_BIN_DIR:\$PATH"
+# Cygwin spelling for bash PATH entries; PYMONGO_BIN_DIR itself is converted
+# to native form below for consumers like uv.
+if [ "Windows_NT" = "${OS:-}" ]; then
+  PYMONGO_BIN_DIR_POSIX="$(cygpath -u "$PYMONGO_BIN_DIR")"
+else
+  PYMONGO_BIN_DIR_POSIX="$PYMONGO_BIN_DIR"
+fi
+
+# Add the latest MongoDB toolchain bin dir to PATH if it exists, so that hosts
+# with an old system Python (e.g. RHEL8's 3.6) still get a modern interpreter
+# for tool installs like `uv tool install rust-just`. It goes after
+# PYMONGO_BIN_DIR so the pinned uv (installed there by setup-uv.py) takes
+# precedence over the toolchain's uv.
+if [ "Windows_NT" = "${OS:-}" ]; then
+  _toolchain_bin="/cygdrive/c/Python/Current/Scripts"
+elif [ "$(uname -s)" == "Darwin" ]; then
+  _toolchain_bin="/Library/Frameworks/Python.Framework/Versions/Current/bin"
+else
+  _toolchain_bin="/opt/python/Current/bin"
+fi
+if [ -d "$_toolchain_bin" ]; then
+  PATH_EXT="$MONGODB_BINARIES:$PYMONGO_BIN_DIR_POSIX:$_toolchain_bin:$DRIVERS_TOOLS_BINARIES:\$PATH"
+else
+  PATH_EXT="$MONGODB_BINARIES:$PYMONGO_BIN_DIR_POSIX:$DRIVERS_TOOLS_BINARIES:\$PATH"
+fi
 
 # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory
 if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin
     DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS)
     PROJECT_DIRECTORY=$(cygpath -m $PROJECT_DIRECTORY)
     CARGO_HOME=$(cygpath -m $CARGO_HOME)
-    UV_TOOL_DIR=$(cygpath -m "$UV_TOOL_DIR")
-    UV_CACHE_DIR=$(cygpath -m "$UV_CACHE_DIR")
     DRIVERS_TOOLS_BINARIES=$(cygpath -m "$DRIVERS_TOOLS_BINARIES")
     MONGODB_BINARIES=$(cygpath -m "$MONGODB_BINARIES")
+    # Native form, uniform with the paths above, for consumers like uv.
     PYMONGO_BIN_DIR=$(cygpath -m "$PYMONGO_BIN_DIR")
 fi
 
@@ -62,10 +85,8 @@ export DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS_BINARIES"
 export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
 
 export CARGO_HOME="$CARGO_HOME"
-export UV_TOOL_DIR="$UV_TOOL_DIR"
-export UV_CACHE_DIR="$UV_CACHE_DIR"
-export UV_TOOL_BIN_DIR="$DRIVERS_TOOLS_BINARIES"
 export PYMONGO_BIN_DIR="$PYMONGO_BIN_DIR"
+export PYMONGO_BIN_DIR_POSIX="$PYMONGO_BIN_DIR_POSIX"
 export PATH="$PATH_EXT"
 # shellcheck disable=SC2154
 export PROJECT="${project:-mongo-python-driver}"
@@ -90,25 +111,3 @@ cat <<EOT > expansion.yml
 DRIVERS_TOOLS: "$DRIVERS_TOOLS"
 PROJECT_DIRECTORY: "$PROJECT_DIRECTORY"
 EOT
-
-# If the toolchain is available, symlink binaries to the bin dir.  This has to be done
-# after drivers-tools is cloned, since we might be using its binary dir.
-_bin_path=""
-if [ "Windows_NT" == "${OS:-}" ]; then
-  _bin_path="/cygdrive/c/Python/Current/Scripts"
-elif [ "$(uname -s)" == "Darwin" ]; then
-  _bin_path="/Library/Frameworks/Python.Framework/Versions/Current/bin"
-else
-  _bin_path="/opt/python/Current/bin"
-fi
-if [ -d "${_bin_path}" ]; then
-  _suffix=""
-  if [ "Windows_NT" == "${OS:-}" ]; then
-    _suffix=".exe"
-  fi
-  echo "Symlinking binaries from toolchain"
-  mkdir -p $PYMONGO_BIN_DIR
-  ln -s ${_bin_path}/just${_suffix} $PYMONGO_BIN_DIR/just${_suffix}
-  ln -s ${_bin_path}/uv${_suffix} $PYMONGO_BIN_DIR/uv${_suffix}
-  ln -s ${_bin_path}/uvx${_suffix} $PYMONGO_BIN_DIR/uvx${_suffix}
-fi

.evergreen/scripts/install-dependencies.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 # Install the necessary dependencies.
-set -eu
+set -euo pipefail
 
 HERE=$(dirname ${BASH_SOURCE:-$0})
 HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )"
@@ -11,27 +11,76 @@ if [ -f $HERE/env.sh ]; then
   . $HERE/env.sh
 fi
 
-# Set up the default bin directory.
-if [ -z "${PYMONGO_BIN_DIR:-}" ]; then
-  PYMONGO_BIN_DIR="$HOME/.local/bin"
+# PYMONGO_BIN_DIR is set by setup-system.sh/env.sh (or setup-dev-env.sh); default
+# it for robustness. Native (Windows) form on cygwin, for consumers like uv;
+# PYMONGO_BIN_DIR_POSIX is the cygwin form for bash PATH contexts.
+export PYMONGO_BIN_DIR="${PYMONGO_BIN_DIR:-$HOME/.local/bin}"
+if [ "Windows_NT" = "${OS:-}" ]; then
+  _bin_dir="$(cygpath -m "$PYMONGO_BIN_DIR")"
+  export PYMONGO_BIN_DIR="$_bin_dir"
+  _posix_bin_dir="$(cygpath -u "$_bin_dir")"
+  export PYMONGO_BIN_DIR_POSIX="$_posix_bin_dir"
+else
+  export PYMONGO_BIN_DIR_POSIX="$PYMONGO_BIN_DIR"
 fi
+# UV_TOOL_BIN_DIR is uv's name for the same dir (setup-uv.py reads both).
+# UV_TOOL_DIR is left to ensure_uv.sh.
+export UV_TOOL_BIN_DIR="${UV_TOOL_BIN_DIR:-$PYMONGO_BIN_DIR}"
 
-# Ensure uv is installed.
-if ! command -v uv &>/dev/null; then
-  _BIN_DIR=$PYMONGO_BIN_DIR
-  mkdir -p ${_BIN_DIR}
-  echo "Installing uv..."
-  curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$_BIN_DIR" INSTALLER_NO_MODIFY_PATH=1 sh
-  if [ "Windows_NT" = "${OS:-}" ]; then
-    chmod +x "$(cygpath -u $_BIN_DIR)/uv.exe"
+# Ensure the bin dir is on PATH: hosts without env.sh (e.g. auth-aws-ecs) never
+# export it, so a fresh pinned install there would be invisible to the probe
+# below and to later steps like `uv tool install` and `uv sync`.
+case ":$PATH:" in
+  *":$PYMONGO_BIN_DIR_POSIX:"*) ;;
+  *) export PATH="$PYMONGO_BIN_DIR_POSIX:$PATH" ;;
+esac
+
+# Compute the setup script path once (native Windows path on cygwin).
+_uv_setup_script="$HERE/setup-uv.py"
+if [ "Windows_NT" = "${OS:-}" ]; then
+  _uv_setup_script="$(cygpath -m "$_uv_setup_script")"
+fi
+
+# Skip setup when the pinned uv is already installed: `setup-uv.py --check`
+# compares the uv on PATH with the required-version pin, without side effects.
+# A full `uv sync` here would download Python and install/build dependencies
+# just to make this decision, and would run before setup-uv-python.sh sets
+# UV_PYTHON, creating the environment twice on local dev.
+#
+# On CI we also require UV_CACHE_DIR to be set: ensure_uv.sh scopes uv's cache
+# to a task-local dir, so an unset UV_CACHE_DIR means the uv setup has not run
+# yet in this task and we must do the setup phase.
+_need_setup=1
+if [ "${CI:-}" != "true" ] || [ -n "${UV_CACHE_DIR:-}" ]; then
+  if python3 "$_uv_setup_script" --check >/dev/null 2>&1; then
+    echo "uv is already set up; skipping uv setup."
+    _need_setup=0
+  fi
+fi
+
+# Set up uv if needed.
+if [ "$_need_setup" = "1" ]; then
+  # ensure-uv.sh (drivers-evergreen-tools) finds or installs uv and scopes its env.
+  if [ -n "${DRIVERS_TOOLS:-}" ] && [ -f "$DRIVERS_TOOLS/.evergreen/ensure-uv.sh" ]; then
+    . "$DRIVERS_TOOLS/.evergreen/ensure-uv.sh"
+    ensure_uv || exit 1
+  fi
+
+  # Do the uv setup (bin dir, pinning, env.sh). Uses the toolchain python3
+  # (added to PATH by configure-env.sh) so no project .venv is created here,
+  # and no required-version check is triggered.
+  python3 "$_uv_setup_script"
+
+  # Re-source env.sh so the values setup-uv.py wrote are available.
+  if [ -f $HERE/env.sh ]; then
+    . $HERE/env.sh
   fi
-  export PATH="$PYMONGO_BIN_DIR:$PATH"
-  echo "Installing uv... done."
 fi
 
-# Ensure just is installed.
-if ! command -v just &>/dev/null; then
-  uv tool install rust-just
+# Make just available. It has no version constraint, so if it is already on PATH
+# there is nothing to do; otherwise install it into the bin dir via uv.
+if ! command -v just >/dev/null 2>&1; then
+  uv tool install --no-config rust-just
 fi
 
 popd > /dev/null

.evergreen/scripts/setup-dev-env.sh

@@ -16,22 +16,54 @@ if [ -f $HERE/test-env.sh ]; then
   . $HERE/test-env.sh
 fi
 
-# Handle the value for UV_PYTHON.
-. $HERE/setup-uv-python.sh
+# The bin dir for the pinned uv/just. setup-system.sh sets it on evergreen hosts;
+# default it here so local dev (without setup-system.sh) also has a usable value.
+# Native (Windows) form on cygwin, like install-dependencies.sh.
+export PYMONGO_BIN_DIR="${PYMONGO_BIN_DIR:-$HOME/.local/bin}"
+if [ "Windows_NT" = "${OS:-}" ]; then
+  _bin_dir="$(cygpath -m "$PYMONGO_BIN_DIR")"
+  export PYMONGO_BIN_DIR="$_bin_dir"
+  _posix_bin_dir="$(cygpath -u "$_bin_dir")"
+  export PYMONGO_BIN_DIR_POSIX="$_posix_bin_dir"
+else
+  export PYMONGO_BIN_DIR_POSIX="$PYMONGO_BIN_DIR"
+fi
+
+# install-dependencies.sh runs as a child process, so its PATH changes do not
+# propagate back here: ensure the bin dir is on this process's PATH too, so a
+# fresh install (first run, bin dir not on PATH yet) is visible to the
+# `uv sync` and pre-commit setup below.
+case ":$PATH:" in
+  *":$PYMONGO_BIN_DIR_POSIX:"*) ;;
+  *) export PATH="$PYMONGO_BIN_DIR_POSIX:$PATH" ;;
+esac
+
+# Make sure a login shell can find the bin dir by adding it to the rc file, so
+# local dev (which may never run setup-system.sh) still has it on PATH. env.sh's
+# PATH does not persist past this session. Select the rc file from $SHELL (not
+# by which rc file happens to exist) so the user's actual shell is updated, and
+# create it if it does not exist yet.
+if [ "${CI:-}" != "true" ] && [ "${GITHUB_ACTIONS:-}" != "true" ]; then
+  case "${SHELL:-}" in
+    */zsh) _rc="$HOME/.zshrc" ;;
+    *) _rc="$HOME/.bashrc" ;;
+  esac
+  touch "$_rc"
+  grep -qF 'export PATH="'"$PYMONGO_BIN_DIR_POSIX"':$PATH"' "$_rc" 2>/dev/null || \
+    printf 'export PATH="%s:$PATH"\n' "$PYMONGO_BIN_DIR_POSIX" >> "$_rc"
+fi
 
 # Ensure dependencies are installed.
 bash $HERE/install-dependencies.sh
 
-# Re-source env.sh: install-dependencies.sh may have appended to it, e.g. when it
-# had to install Python on an image that lacks a toolchain.
+# Re-source env.sh in case a dependency install updated it, e.g. on a host
+# without a toolchain where uv was installed into a shared bin dir.
 if [ -f $HERE/env.sh ]; then
   . $HERE/env.sh
 fi
 
-# Add the default install path to the path if needed.
-if [ -z "${PYMONGO_BIN_DIR:-}" ]; then
-  export PATH="$PATH:$HOME/.local/bin"
-fi
+# Handle the value for UV_PYTHON.
+. $HERE/setup-uv-python.sh
 
 # Only run the next part if not running on CI.
 if [ -z "${CI:-}" ]; then