MOTOR-1139 Switch to Pytest (#222)

.evergreen/config.yml

@@ -403,9 +403,6 @@ functions:
           set -x
           export LIBMONGOCRYPT_URL="${libmongocrypt_url}"
           export TEST_ENCRYPTION=1
-          if [ -n "${EXTRA_PATH}" ]; then
-            export PATH="${EXTRA_PATH}:$PATH"
-          fi
           PYTHON_BINARY="${PYTHON_BINARY}" \
           TOX_BINARY="${TOX_BINARY}" \
           TOX_ENV="${TOX_ENV}" \
@@ -424,7 +421,6 @@ functions:
         working_dir: "src"
         script: |
           # DO NOT ECHO WITH XTRACE (which PREPARE_SHELL does)
-
           CLIENT_PEM=${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem \
           CA_PEM=${DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem \
           PYTHON_BINARY=${PYTHON_BINARY} \
@@ -799,7 +795,7 @@ tasks:
       commands:
         - func: "run tox"
           vars:
-             TOX_ENV: py3-sphinx-docs
+             TOX_ENV: docs
 
     - name: "doctest"
       commands:
@@ -809,7 +805,7 @@ tasks:
             TOPOLOGY: "server"
         - func: "run tox"
           vars:
-             TOX_ENV: py3-sphinx-doctest
+             TOX_ENV: doctest
 
 # }}}
 
@@ -831,155 +827,111 @@ axes:
   - id: tox-env
     display_name: "Tox Env RHEL8"
     values:
-      - id: "tornado5-pypy37"
+      - id: "test-pypy38"
         variables:
-           TOX_ENV: "tornado5-pypy37"
-           PYTHON_BINARY: "/opt/python/pypy3.7/bin/pypy3"
-      - id: "tornado5-py37"
+           TOX_ENV: "test"
+           PYTHON_BINARY: "/opt/python/pypy3.8/bin/python3"
+      - id: "test-py37"
         variables:
-           TOX_ENV: "tornado5-py37"
+           TOX_ENV: "test"
            PYTHON_BINARY: "/opt/python/3.7/bin/python3"
-      - id: "tornado6-pypy37"
-        variables:
-          TOX_ENV: "tornado6-pypy37"
-          PYTHON_BINARY: "/opt/python/pypy3.7/bin/pypy3"
-      - id: "tornado6-py37"
-        variables:
-           TOX_ENV: "tornado6-py37"
-           PYTHON_BINARY: "/opt/python/3.7/bin/python3"
-      - id: "tornado6-py38"
-        variables:
-           TOX_ENV: "tornado6-py38"
-           PYTHON_BINARY: "/opt/python/3.8/bin/python3"
-      - id: "tornado6-py39"
-        variables:
-           TOX_ENV: "tornado6-py39"
-           PYTHON_BINARY: "/opt/python/3.9/bin/python3"
-      - id: "tornado6-py310"
-        variables:
-           TOX_ENV: "tornado6-py310"
-           PYTHON_BINARY: "/opt/python/3.10/bin/python3"
-      - id: "tornado6-py311"
-        variables:
-           TOX_ENV: "tornado6-py311"
-           PYTHON_BINARY: "/opt/python/3.11/bin/python3"
-      - id: "tornado6-py312"
-        variables:
-           TOX_ENV: "tornado6-py312"
-           PYTHON_BINARY: "/opt/python/3.12/bin/python3"
-      - id: "tornado_git-py38"
-        variables:
-           TOX_ENV: "tornado_git-py38"
-           PYTHON_BINARY: "/opt/python/3.8/bin/python3"
-      - id: "asyncio-pypy37"
-        variables:
-          TOX_ENV: "asyncio-pypy37"
-          PYTHON_BINARY: "/opt/python/pypy3.7/bin/pypy3"
-      - id: "asyncio-py37"
+      - id: "test-py37"
         variables:
-           TOX_ENV: "asyncio-py37"
+           TOX_ENV: "test"
            PYTHON_BINARY: "/opt/python/3.7/bin/python3"
-      - id: "asyncio-py38"
+      - id: "test-py38"
         variables:
-           TOX_ENV: "asyncio-py38"
+           TOX_ENV: "test"
            PYTHON_BINARY: "/opt/python/3.8/bin/python3"
-      - id: "asyncio-py39"
+      - id: "test-py39"
         variables:
-           TOX_ENV: "asyncio-py39"
+           TOX_ENV: "test"
            PYTHON_BINARY: "/opt/python/3.9/bin/python3"
-      - id: "asyncio-py310"
+      - id: "test-py310"
         variables:
-           TOX_ENV: "asyncio-py310"
+           TOX_ENV: "test"
            PYTHON_BINARY: "/opt/python/3.10/bin/python3"
-      - id: "asyncio-py311"
+      - id: "test-py311"
         variables:
-           TOX_ENV: "asyncio-py311"
+           TOX_ENV: "test"
            PYTHON_BINARY: "/opt/python/3.11/bin/python3"
-      - id: "asyncio-py312"
+      - id: "test-py312"
         variables:
-           TOX_ENV: "asyncio-py312"
+           TOX_ENV: "test"
            PYTHON_BINARY: "/opt/python/3.12/bin/python3"
-      - id: "py3-pymongo-latest"
+      - id: "test-pymongo-latest"
         variables:
-           TOX_ENV: "py3-pymongo-latest"
+           TOX_ENV: "test-pymongo-latest"
            PYTHON_BINARY: "/opt/python/3.7/bin/python3"
-      - id: "synchro37"
+      - id: "synchro-py37"
         variables:
-          TOX_ENV: "synchro37"
+          TOX_ENV: "synchro"
           PYTHON_BINARY: "/opt/python/3.7/bin/python3"
-      - id: "synchro312"
+      - id: "synchro-py312"
         variables:
-          TOX_ENV: "synchro312"
+          TOX_ENV: "synchro"
           PYTHON_BINARY: "/opt/python/3.12/bin/python3"
 
   - id: tox-env-rhel7
     display_name: "Tox Env RHEL7"
     values:
-      - id: "tornado6-py39"
+      - id: "test"
         variables:
-           TOX_ENV: "tornado6-py39"
-           PYTHON_BINARY: "/opt/python/3.9/bin/python3"
-      - id: "asyncio-py39"
-        variables:
-           TOX_ENV: "asyncio-py39"
+           TOX_ENV: "test"
            PYTHON_BINARY: "/opt/python/3.9/bin/python3"
 
   # Test Python 3.8 only on Mac.
   - id: tox-env-osx
     display_name: "Tox Env OSX"
     values:
-      - id: "tornado5"
-        variables:
-           TOX_ENV: "tornado5"
-           EXTRA_PATH: "/Library/Frameworks/Python.framework/Versions/3.8/bin/"
-      - id: "tornado_git"
+      - id: "test"
         variables:
-           TOX_ENV: "tornado_git"
-           EXTRA_PATH: "/Library/Frameworks/Python.framework/Versions/3.8/bin/"
+           TOX_ENV: "test"
+           PYTHON_BINARY: "/Library/Frameworks/Python.framework/Versions/3.8/bin/python3"
 
   - id: tox-env-win
     display_name: "Tox Env Windows"
     values:
-      - id: "asyncio"
+      - id: "test-py37"
         variables:
-          TOX_ENV: "asyncio"
-          EXTRA_PATH: "c:\\python\\Python37"
-      - id: "tornado6"
+          TOX_ENV: "test"
+          PYTHON_BINARY: "c:/python/Python37/python.exe"
+      - id: "test-py38"
         variables:
-           TOX_ENV: "tornado6"
-           EXTRA_PATH: "c:\\python\\Python37"
-      - id: "tornado6"
+           TOX_ENV: "test"
+           PYTHON_BINARY: "c:/python/Python39/python.exe"
+      - id: "test-py39"
         variables:
-           TOX_ENV: "tornado6"
-           EXTRA_PATH: "c:\\python\\Python38"
+           TOX_ENV: "test"
+           PYTHON_BINARY: "c:/python/Python39/python.exe"
+      - id: "test-py310"
+        variables:
+           TOX_ENV: "test"
+           PYTHON_BINARY: "c:/python/Python310/python.exe"
+      - id: "test-py311"
+        variables:
+           TOX_ENV: "test"
+           PYTHON_BINARY: "c:/python/Python311/python.exe"
+      - id: "test-py312"
+        variables:
+           TOX_ENV: "test"
+           PYTHON_BINARY: "c:/python/Python312/python.exe"
 
   - id: os
     display_name: "Operating System"
     values:
       - id: "rhel84"
         display_name: "RHEL 8.4"
         run_on: "rhel84-small"
-        variables:
-           INSTALL_TOX: true
-           VIRTUALENV: "/opt/python/3.7/bin/python3 -m virtualenv"
       - id: "rhel76"
         display_name: "RHEL 7.6"
         run_on: "rhel76-small"
-        variables:
-           INSTALL_TOX: true
-           VIRTUALENV: "/opt/python/3.7/bin/python3 -m virtualenv"
       - id: "win"
         display_name: "Windows"
         run_on: "windows-64-vsMulti-small"
-        variables:
-           INSTALL_TOX: true
-           VIRTUALENV: "/cygdrive/c/python/Python37/python.exe -m virtualenv"
       - id: "macos-1100"
         display_name: "macOS 11.00"
         run_on: "macos-1100"
-        variables:
-           INSTALL_TOX: true
-           VIRTUALENV: "/Library/Frameworks/Python.framework/Versions/3.8/bin/python3 -m venv"
 
 buildvariants:
 
@@ -994,7 +946,7 @@ buildvariants:
     # TODO: synchro needs PyMongo's updated SSL test certs,
     # which may require Motor test suite changes.
     - os: "*"
-      tox-env: ["synchro37", "synchro312"]
+      tox-env: ["synchro-py37", "synchro-py312"]
       ssl: "ssl"
   tasks:
      - ".rapid"
@@ -1052,7 +1004,7 @@ buildvariants:
 
 - matrix_name: "enterprise-auth"
   display_name: "Enterprise Auth-${tox-env}"
-  matrix_spec: {"tox-env": ["synchro37", "synchro312"], ssl: "ssl"}
+  matrix_spec: {"tox-env": ["synchro-py37", "synchro-py312"], ssl: "ssl"}
   run_on:
      - "rhel84-small"
   tasks:
@@ -1063,9 +1015,7 @@ buildvariants:
   run_on:
      - "rhel84-small"
   expansions:
-     TOX_ENV: "py3-sphinx-docs"
-     INSTALL_TOX: true
-     VIRTUALENV: "/opt/python/3.7/bin/python3 -m virtualenv"
+     TOX_ENV: "docs"
      PYTHON_BINARY: "/opt/python/3.7/bin/python3"
   tasks:
      - name: "docs"
@@ -1075,9 +1025,7 @@ buildvariants:
   run_on:
      - "rhel84-small"
   expansions:
-     TOX_ENV: "py3-sphinx-doctest"
-     INSTALL_TOX: true
-     VIRTUALENV: "/opt/python/3.7/bin/python3 -m virtualenv"
+     TOX_ENV: "doctest"
      PYTHON_BINARY: "/opt/python/3.7/bin/python3"
   tasks:
      - name: "doctest"

.evergreen/run-enterprise-auth-tests.sh

@@ -26,4 +26,4 @@ export GSSAPI_PRINCIPAL=${PRINCIPAL}
 export TOX_TESTENV_PASSENV="*"
 
 # --sitepackages allows use of pykerberos without a test dep.
-/opt/python/3.7/bin/python3 -m tox -e "$TOX_ENV" --sitepackages -- -x test.test_auth
+/opt/python/3.7/bin/python3 -m tox -m "$TOX_ENV" --sitepackages -- -x test.test_auth

.evergreen/run-tox.sh

@@ -6,14 +6,16 @@ set -o errexit  # Exit the script with error if any of the commands fail
 #       AUTH                    Set to enable authentication. Defaults to "noauth"
 #       SSL                     Set to enable SSL. Defaults to "nossl"
 #       TOX_ENV                 Tox environment name, e.g. "tornado5-py37"
-#       TOX_BINARY              Path to tox executable
-#       INSTALL_TOX             Whether to install tox in a virtualenv
 #       PYTHON_BINARY           Path to python
-#       VIRTUALENV              Path to virtualenv script
 
 AUTH=${AUTH:-noauth}
 SSL=${SSL:-nossl}
 
+if [ -z $PYTHON_BINARY ]; then
+    echo "PYTHON_BINARY is undefined!"
+    exit 1
+fi
+
 if [ "$AUTH" != "noauth" ]; then
     export DB_USER="bob"
     export DB_PASSWORD="pwd123"
@@ -24,24 +26,50 @@ if [ "$SSL" != "nossl" ]; then
     export CA_PEM="$DRIVERS_TOOLS/.evergreen/x509gen/ca.pem"
 fi
 
-if [ "$TOX_ENV" = "synchro37" ]; then
-    SETUP_ARGS="-- --check-exclude-patterns"
-fi
 
-if [ "${INSTALL_TOX}" = "true" ]; then
-    $VIRTUALENV motorenv
-    set +o xtrace
-    if [ -f motorenv/bin/activate ]; then
-        source motorenv/bin/activate
+# Usage:
+# createvirtualenv /path/to/python /output/path/for/venv
+# * param1: Python binary to use for the virtualenv
+# * param2: Path to the virtualenv to create
+createvirtualenv () {
+    PYTHON=$1
+    VENVPATH=$2
+    if $PYTHON -m virtualenv --version; then
+        VIRTUALENV="$PYTHON -m virtualenv"
+    elif $PYTHON -m venv -h > /dev/null; then
+        # System virtualenv might not be compatible with the python3 on our path
+        VIRTUALENV="$PYTHON -m venv"
+    else
+        echo "Cannot test without virtualenv"
+        exit 1
+    fi
+    # Workaround for bug in older versions of virtualenv.
+    $VIRTUALENV $VENVPATH || $PYTHON -m venv $VENVPATH
+    if [ "Windows_NT" = "$OS" ]; then
+        # Workaround https://bugs.python.org/issue32451:
+        # mongovenv/Scripts/activate: line 3: $'\r': command not found
+        dos2unix $VENVPATH/Scripts/activate || true
+        . $VENVPATH/Scripts/activate
     else
-        # Windows.
-        ls -l motorenv
-        source motorenv/Scripts/activate
+        . $VENVPATH/bin/activate
     fi
-    set -o xtrace
-    pip install "tox>=3.18,<4"
-    TOX_BINARY=tox
+
+    python -m pip install --upgrade pip
+    python -m pip install --upgrade setuptools wheel tox
+}
+
+
+if $PYTHON_BINARY -m tox --version; then
+    run_tox() {
+      $PYTHON_BINARY -m tox -m $TOX_ENV "$@"
+    }
+else # No toolchain present, set up virtualenv before installing tox
+    createvirtualenv "$PYTHON_BINARY" toxenv
+    trap "deactivate; rm -rf toxenv" EXIT HUP
+    python -m pip install tox
+    run_tox() {
+      python -m tox -m $TOX_ENV "$@"
+    }
 fi
 
-# Run the tests, and store the results in Evergreen compatible XUnit XML
-${TOX_BINARY} -e ${TOX_ENV} ${SETUP_ARGS} "$@"
+run_tox "${@:1}"