Refactor scripts

.evergreen/combine-coverage.sh

@@ -5,19 +5,12 @@
 
 set -eu
 
-. .evergreen/utils.sh
+# Set up the virtual env.
+. .evergreen/scripts/setup-dev-env.sh
+uv sync --group coverage
+source .venv/bin/activate
 
-if [ -z "${PYTHON_BINARY:-}" ]; then
-    PYTHON_BINARY=$(find_python3)
-fi
-
-createvirtualenv "$PYTHON_BINARY" covenv
-# Keep in sync with run-tests.sh
-# coverage >=5 is needed for relative_files=true.
-pip install -q "coverage[toml]>=5,<=7.5"
-
-pip list
 ls -la coverage/
 
-python -m coverage combine coverage/coverage.*
-python -m coverage html -d htmlcov
+coverage combine coverage/coverage.*
+coverage html -d htmlcov

.evergreen/generated_configs/variants.yml

@@ -72,7 +72,7 @@ buildvariants:
   # Aws auth tests
   - name: auth-aws-ubuntu-20
     tasks:
-      - name: .auth-aws !.auth-aws-ecs
+      - name: .auth-aws
     display_name: Auth AWS Ubuntu-20
     run_on:
       - ubuntu2004-small
@@ -459,8 +459,6 @@ buildvariants:
     display_name: Search Index Helpers RHEL8 Python3.10
     run_on:
       - rhel87-small
-    expansions:
-      PYTHON_BINARY: /opt/python/3.10/bin/python3
 
   # Server version tests
   - name: mongodb-v4.2

.evergreen/run-mongodb-aws-ecs-test.sh

@@ -19,20 +19,14 @@ fi
 # Now we can safely enable xtrace
 set -o xtrace
 
-# Install python with pip.
-PYTHON_VER="python3.10"
+# Install a c compiler.
 apt-get -qq update  < /dev/null > /dev/null
-apt-get -q install -y software-properties-common
-# Use openpgp to avoid gpg key timeout.
-mkdir -p $HOME/.gnupg
-echo "keyserver keys.openpgp.org" >> $HOME/.gnupg/gpg.conf
-add-apt-repository -y 'ppa:deadsnakes/ppa'
-apt-get -qq install $PYTHON_VER $PYTHON_VER-venv build-essential $PYTHON_VER-dev -y  < /dev/null > /dev/null
+apt-get -q install -y build-essential
 
-export PYTHON_BINARY=$PYTHON_VER
 export SET_XTRACE_ON=1
 cd src
 rm -rf .venv
 rm -f .evergreen/scripts/test-env.sh || true
+rm -f .evergreen/scripts/env.sh || true
 bash ./.evergreen/just.sh setup-tests auth_aws ecs-remote
 bash .evergreen/just.sh run-tests

.evergreen/run-tests.sh

@@ -37,6 +37,8 @@ cleanup_tests() {
 trap "cleanup_tests" SIGINT ERR
 
 # Start the test runner.
+echo "Running tests with UV_PYTHON=${UV_PYTHON:-}..."
 uv run ${UV_ARGS} --reinstall-package pymongo .evergreen/scripts/run_tests.py "$@"
+echo "Running tests with UV_PYTHON=${UV_PYTHON:-}... done."
 
 cleanup_tests

.evergreen/scripts/check-import-time.sh

@@ -11,11 +11,10 @@ pushd $HERE/../.. >/dev/null
 BASE_SHA="$1"
 HEAD_SHA="$2"
 
-. .evergreen/utils.sh
-
-if [ -z "${PYTHON_BINARY:-}" ]; then
-    PYTHON_BINARY=$(find_python3)
-fi
+# Set up the virtual env.
+. $HERE/setup-dev-env.sh
+uv venv --seed
+source .venv/bin/activate
 
 # Use the previous commit if this was not a PR run.
 if [ "$BASE_SHA" == "$HEAD_SHA" ]; then
@@ -24,7 +23,6 @@ fi
 
 function get_import_time() {
     local log_file
-    createvirtualenv "$PYTHON_BINARY" import-venv
     python -m pip install -q ".[aws,encryption,gssapi,ocsp,snappy,zstd]"
     # Import once to cache modules
     python -c "import pymongo"

.evergreen/scripts/generate_config.py

@@ -438,8 +438,7 @@ def create_aws_auth_variants():
 
     for host_name in ["ubuntu20", "win64", "macos"]:
         expansions = dict()
-        # PYTHON-5604 - we need to skip ECS tests for now.
-        tasks = [".auth-aws !.auth-aws-ecs"]
+        tasks = [".auth-aws"]
         tags = []
         if host_name == "macos":
             tasks = [".auth-aws !.auth-aws-web-identity !.auth-aws-ecs !.auth-aws-ec2"]

.evergreen/scripts/generate_config_utils.py

@@ -133,43 +133,17 @@ def create_variant(
     *,
     version: str | None = None,
     host: Host | str | None = None,
-    python: str | None = None,
     expansions: dict | None = None,
     **kwargs: Any,
 ) -> BuildVariant:
     expansions = expansions and expansions.copy() or dict()
     if version:
         expansions["VERSION"] = version
-    if python:
-        expansions["PYTHON_BINARY"] = get_python_binary(python, host)
     return create_variant_generic(
         tasks, display_name, version=version, host=host, expansions=expansions, **kwargs
     )
 
 
-def get_python_binary(python: str, host: Host) -> str:
-    """Get the appropriate python binary given a python version and host."""
-    name = host.name
-    if name in ["win64", "win32"]:
-        if name == "win32":
-            base = "C:/python/32"
-        else:
-            base = "C:/python"
-        python_dir = python.replace(".", "").replace("t", "")
-        return f"{base}/Python{python_dir}/python{python}.exe"
-
-    if name in ["rhel8", "ubuntu22", "ubuntu20", "rhel7"]:
-        return f"/opt/python/{python}/bin/python3"
-
-    if name in ["macos", "macos-arm64"]:
-        bin_name = "python3t" if "t" in python else "python3"
-        python_dir = python.replace("t", "")
-        framework_dir = "PythonT" if "t" in python else "Python"
-        return f"/Library/Frameworks/{framework_dir}.Framework/Versions/{python_dir}/bin/{bin_name}"
-
-    raise ValueError(f"no match found for python {python} on {name}")
-
-
 def get_versions_from(min_version: str) -> list[str]:
     """Get all server versions starting from a minimum version."""
     min_version_float = float(min_version)

.evergreen/scripts/install-dependencies.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Install the dependencies needed for an evergreen run.
+# Install the necessary dependencies.
 set -eu
 
 HERE=$(dirname ${BASH_SOURCE:-$0})
@@ -13,65 +13,24 @@ fi
 # Set up the default bin directory.
 if [ -z "${PYMONGO_BIN_DIR:-}" ]; then
   PYMONGO_BIN_DIR="$HOME/.local/bin"
-  export PATH="$PYMONGO_BIN_DIR:$PATH"
-fi
-
-# Helper function to pip install a dependency using a temporary python env.
-function _pip_install() {
-  _HERE=$(dirname ${BASH_SOURCE:-$0})
-  . $_HERE/../utils.sh
-  _VENV_PATH=$(mktemp -d)
-  if [ "Windows_NT" = "${OS:-}" ]; then
-    _VENV_PATH=$(cygpath -m $_VENV_PATH)
-  fi
-  echo "Installing $2 using pip..."
-  createvirtualenv "$(find_python3)" $_VENV_PATH
-  python -m pip install $1
-  _suffix=""
-  if [ "Windows_NT" = "${OS:-}" ]; then
-    _suffix=".exe"
-  fi
-  ln -s "$(which $2)" $PYMONGO_BIN_DIR/${2}${_suffix}
-  # uv also comes with a uvx binary.
-  if [ $2 == "uv" ]; then
-    ln -s "$(which uvx)" $PYMONGO_BIN_DIR/uvx${_suffix}
-  fi
-  echo "Installed to ${PYMONGO_BIN_DIR}"
-  echo "Installing $2 using pip... done."
-}
-
-# Ensure just is installed.
-if ! command -v just &>/dev/null; then
-  # On most systems we can install directly.
-  _TARGET=""
-  if [ "Windows_NT" = "${OS:-}" ]; then
-    _TARGET="--target x86_64-pc-windows-msvc"
-  fi
-  _BIN_DIR=$PYMONGO_BIN_DIR
-  mkdir -p ${_BIN_DIR}
-  echo "Installing just..."
-  mkdir -p "$_BIN_DIR" 2>/dev/null || true
-  curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- $_TARGET --to "$_BIN_DIR" || {
-    # Remove just file if it exists (can be created if there was an install error).
-    rm -f ${_BIN_DIR}/just
-    _pip_install rust-just just
-  }
-  echo "Installing just... done."
 fi
 
 # Ensure uv is installed.
 if ! command -v uv &>/dev/null; then
   _BIN_DIR=$PYMONGO_BIN_DIR
   mkdir -p ${_BIN_DIR}
   echo "Installing uv..."
-  # On most systems we can install directly.
-  curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$_BIN_DIR" INSTALLER_NO_MODIFY_PATH=1 sh || {
-     _pip_install uv uv
-  }
+  curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$_BIN_DIR" INSTALLER_NO_MODIFY_PATH=1 sh
   if [ "Windows_NT" = "${OS:-}" ]; then
     chmod +x "$(cygpath -u $_BIN_DIR)/uv.exe"
   fi
+  export PATH="$PYMONGO_BIN_DIR:$PATH"
   echo "Installing uv... done."
 fi
 
+# Ensure just is installed.
+if ! command -v just &>/dev/null; then
+  uv tool install rust-just
+fi
+
 popd > /dev/null

.evergreen/scripts/kms_tester.py

@@ -33,7 +33,7 @@ def _setup_azure_vm(base_env: dict[str, str]) -> None:
     env["AZUREKMS_CMD"] = "sudo apt-get install -y python3-dev build-essential"
     run_command(f"{azure_dir}/run-command.sh", env=env)
 
-    env["AZUREKMS_CMD"] = "NO_EXT=1 bash .evergreen/just.sh setup-tests kms azure-remote"
+    env["AZUREKMS_CMD"] = "bash .evergreen/just.sh setup-tests kms azure-remote"
     run_command(f"{azure_dir}/run-command.sh", env=env)
     LOGGER.info("Setting up Azure VM... done.")
 
@@ -53,7 +53,7 @@ def _setup_gcp_vm(base_env: dict[str, str]) -> None:
     env["GCPKMS_CMD"] = "sudo apt-get install -y python3-dev build-essential"
     run_command(f"{gcp_dir}/run-command.sh", env=env)
 
-    env["GCPKMS_CMD"] = "NO_EXT=1 bash ./.evergreen/just.sh setup-tests kms gcp-remote"
+    env["GCPKMS_CMD"] = "bash ./.evergreen/just.sh setup-tests kms gcp-remote"
     run_command(f"{gcp_dir}/run-command.sh", env=env)
     LOGGER.info("Setting up GCP VM...")
 

.evergreen/scripts/oidc_tester.py

@@ -89,7 +89,7 @@ def test_oidc_send_to_remote(sub_test_name: str) -> None:
         env[f"{upper_name}OIDC_DRIVERS_TAR_FILE"] = TMP_DRIVER_FILE
         env[
             f"{upper_name}OIDC_TEST_CMD"
-        ] = f"NO_EXT=1 OIDC_ENV={sub_test_name} ./.evergreen/run-mongodb-oidc-test.sh"
+        ] = f"OIDC_ENV={sub_test_name} ./.evergreen/run-mongodb-oidc-test.sh"
     elif sub_test_name in K8S_NAMES:
         env["K8S_DRIVERS_TAR_FILE"] = TMP_DRIVER_FILE
         env["K8S_TEST_CMD"] = "OIDC_ENV=k8s ./.evergreen/run-mongodb-oidc-test.sh"

.evergreen/scripts/setup-dev-env.sh

@@ -1,59 +1,58 @@
 #!/bin/bash
-# Set up a development environment on an evergreen host.
+# Set up development environment.
 set -eu
 
 HERE=$(dirname ${BASH_SOURCE:-$0})
 HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )"
 ROOT=$(dirname "$(dirname $HERE)")
-pushd $ROOT > /dev/null
-
-# Bail early if running on GitHub Actions.
-if [ -n "${GITHUB_ACTION:-}" ]; then
-  exit 0
-fi
 
 # Source the env files to pick up common variables.
 if [ -f $HERE/env.sh ]; then
   . $HERE/env.sh
 fi
-# PYTHON_BINARY or PYTHON_VERSION may be defined in test-env.sh.
+
+# Get variables defined in test-env.sh.
 if [ -f $HERE/test-env.sh ]; then
   . $HERE/test-env.sh
 fi
 
 # Ensure dependencies are installed.
 bash $HERE/install-dependencies.sh
 
-# Get the appropriate UV_PYTHON.
-. $ROOT/.evergreen/utils.sh
-
-if [ -z "${PYTHON_BINARY:-}" ]; then
-    if [ -n "${PYTHON_VERSION:-}" ]; then
-      PYTHON_BINARY=$(get_python_binary $PYTHON_VERSION)
-    else
-      PYTHON_BINARY=$(find_python3)
+# Handle the value for UV_PYTHON.
+. $HERE/setup-uv-python.sh
+
+# Only run the next part if not running on CI.
+if [ -z "${CI:-}" ]; then
+  # Add the default install path to the path if needed.
+  if [ -z "${PYMONGO_BIN_DIR:-}" ]; then
+    export PATH="$PATH:$HOME/.local/bin"
+  fi
+
+  # Set up venv, making sure c extensions build unless disabled.
+  if [ -z "${NO_EXT:-}" ]; then
+    export PYMONGO_C_EXT_MUST_BUILD=1
+  fi
+
+  (
+    cd $ROOT && uv sync
+  )
+
+  # Set up build utilities on Windows spawn hosts.
+  if [ -f $HOME/.visualStudioEnv.sh ]; then
+    set +u
+    SSH_TTY=1 source $HOME/.visualStudioEnv.sh
+    set -u
+  fi
+
+  # Only set up pre-commit if we are in a git checkout.
+  if [ -f $HERE/.git ]; then
+    if ! command -v pre-commit &>/dev/null; then
+      uv tool install pre-commit
     fi
-fi
-export UV_PYTHON=${PYTHON_BINARY}
-echo "Using python $UV_PYTHON"
 
-# Add the default install path to the path if needed.
-if [ -z "${PYMONGO_BIN_DIR:-}" ]; then
-  export PATH="$PATH:$HOME/.local/bin"
-fi
-
-# Set up venv, making sure c extensions build unless disabled.
-if [ -z "${NO_EXT:-}" ]; then
-  export PYMONGO_C_EXT_MUST_BUILD=1
-fi
-# Set up visual studio env on Windows spawn hosts.
-if [ -f $HOME/.visualStudioEnv.sh ]; then
-  set +u
-  SSH_TTY=1 source $HOME/.visualStudioEnv.sh
-  set -u
+    if [ ! -f .git/hooks/pre-commit ]; then
+      uvx pre-commit install
+    fi
+  fi
 fi
-uv sync
-
-echo "Setting up python environment... done."
-
-popd > /dev/null

.evergreen/scripts/setup-system.sh

@@ -8,9 +8,13 @@ echo "Setting up system..."
 bash .evergreen/scripts/configure-env.sh
 source .evergreen/scripts/env.sh
 bash $DRIVERS_TOOLS/.evergreen/setup.sh
-bash .evergreen/scripts/install-dependencies.sh
 popd
 
+# Run spawn host-specific tasks.
+if [ -z "${CI:-}" ]; then
+  bash $HERE/setup-dev-env.sh
+fi
+
 # Enable core dumps if enabled on the machine
 # Copied from https://github.com/mongodb/mongo/blob/master/etc/evergreen.yml
 if [ -f /proc/self/coredump_filter ]; then