DRIVERS-3335 Add alternate server start scripts using mongodb-runner

.evergreen/config.yml

@@ -568,6 +568,20 @@ functions:
         include_expansions_in_env: [AWS_SECRET_ACCESS_KEY, AWS_ACCESS_KEY_ID, AWS_SESSION_TOKEN]
         args: [src/.evergreen/tests/test-csfle.sh]
 
+  "run server test partial":
+    - command: subprocess.exec
+      type: test
+      params:
+        binary: bash
+        args: [src/.evergreen/tests/test-server.sh, partial]
+
+  "run server test full":
+    - command: subprocess.exec
+      type: test
+      params:
+        binary: bash
+        args: [src/.evergreen/tests/test-server.sh]
+
   "run cli test partial":
     - command: subprocess.exec
       type: test
@@ -1103,6 +1117,11 @@ tasks:
       commands:
         - func: "run cli test full"
 
+    - name: "test-server-full"
+      tags: ["pr"]
+      commands:
+        - func: "run server test full"
+
     - name: "test-ocsp"
       tags: ["pr", "ocsp"]
       commands:
@@ -1118,6 +1137,11 @@ tasks:
       commands:
         - func: "run cli test partial"
 
+    - name: "test-server-partial"
+      tags: ["pr"]
+      commands:
+        - func: "run server test partial"
+
 # }}}
 
 task_groups:
@@ -1556,6 +1580,7 @@ buildvariants:
           - "test-install-binaries"
           - "test-csfle"
           - "test-cli-full"
+          - "test-server-full"
           - "test-happy-eyeballs"
           - "test-ocsp"
           - "test-8.0-standalone-require-api"
@@ -1585,6 +1610,7 @@ buildvariants:
         add_tasks:
           - "test-install-binaries"
           - "test-cli-partial"
+          - "test-server-partial"
 
 - matrix_name: "tests-os-requires-70"
   matrix_spec: {"os-requires-70": "*", auth: "*", ssl: "*" }
@@ -1608,6 +1634,7 @@ buildvariants:
         add_tasks:
           - "test-install-binaries"
           - "test-cli-partial"
+          - "test-server-partial"
 
 # Storage Engine Tests on Ubuntu 20.04
 - matrix_name: "tests-storage-engines"

.evergreen/docker/rhel8/base-entrypoint.sh

@@ -1,8 +1,9 @@
 #!/usr/bin/env bash
 set -eu
 
-# Remove the virtual env.
+# Remove the virtual env and any node install.
 rm -rf $DRIVERS_TOOLS/.evergreen/venv || true
+rm -rf $DRIVERS_TOOLS/.evergreen/node-artifacts || true
 
 # Start the server.
 cd $DRIVERS_TOOLS

.evergreen/docker/ubuntu22.04/base-entrypoint.sh

@@ -1,8 +1,9 @@
 #!/usr/bin/env bash
 set -eu
 
-# Remove the virtual env.
+# Remove the virtual env and any node install.
 rm -rf $DRIVERS_TOOLS/.evergreen/venv || true
+rm -rf $DRIVERS_TOOLS/.evergreen/node-artifacts || true
 
 # Start the server.
 cd $DRIVERS_TOOLS

.evergreen/init-node-and-npm-env.sh

@@ -1,4 +1,5 @@
 #! /usr/bin/env bash
+# shellcheck shell=sh
 ##
 ## This script add the location of `npm` and `node` to the path.
 ## This is necessary because evergreen uses separate bash scripts for
@@ -7,10 +8,12 @@
 ## access to `npm`, `node`, or need to install something globally from
 ## npm.
 
-SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]})
-. $SCRIPT_DIR/handle-paths.sh
-NODE_ARTIFACTS_PATH="$SCRIPT_DIR/node-artifacts"
-if [[ "${OS:-}" == "Windows_NT" ]]; then
+# See https://stackoverflow.com/questions/35006457/choosing-between-0-and-bash-source/35006505#35006505
+# Why we need this syntax when sh is not aliased to bash (this script must be able to be called from sh)
+# shellcheck disable=SC3028
+_SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0})
+NODE_ARTIFACTS_PATH="$_SCRIPT_DIR/node-artifacts"
+if [ "${OS:-}" = "Windows_NT" ]; then
   NODE_ARTIFACTS_PATH=$(cygpath --unix "$NODE_ARTIFACTS_PATH")
 fi
 

.evergreen/install-node.sh

@@ -3,9 +3,13 @@ set -o errexit  # Exit the script with error if any of the commands fail
 
 SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]})
 . $SCRIPT_DIR/handle-paths.sh
-pushd $SCRIPT_DIR
 
-NODE_LTS_VERSION=${NODE_LTS_VERSION:-20}
+DEFAULT_NODE_VERSION=20
+if grep -q "release 7" /etc/redhat-release 2> /dev/null; then
+  DEFAULT_NODE_VERSION=16
+fi
+NODE_LTS_VERSION=${NODE_LTS_VERSION:-$DEFAULT_NODE_VERSION}
+
 # If NODE_LTS_VERSION is numeric and less than 18, default to 9, if less than 20, default to 10.
 # Do not override if it is already set.
 if [[ "$NODE_LTS_VERSION" =~ ^[0-9]+$ && "$NODE_LTS_VERSION" -lt 18 ]]; then
@@ -17,11 +21,32 @@ else
 fi
 export NPM_VERSION=${NPM_VERSION}
 
-source "./init-node-and-npm-env.sh"
+source "$SCRIPT_DIR/init-node-and-npm-env.sh"
 
 if [[ -z "${npm_global_prefix}" ]]; then echo "npm_global_prefix is unset" && exit 1; fi
 if [[ -z "${NODE_ARTIFACTS_PATH}" ]]; then echo "NODE_ARTIFACTS_PATH is unset" && exit 1; fi
 
+function debug_output {
+  echo "node location: $(which node)"
+  echo "node version: $(node -v)"
+  echo "npm location: $(which npm)"
+  echo "npm version: $(npm -v)"
+  echo "Run 'source init-node-and-npm-env.sh' to handle environment setup."
+}
+
+# Bail early if this version of node was already installed.
+if grep -Fxq "$NODE_LTS_VERSION" $NODE_ARTIFACTS_PATH/node-version.txt 2> /dev/null; then
+  echo "Node $NODE_LTS_VERSION already installed!"
+  debug_output
+  exit 0
+fi
+
+# Ensure a clean directory.
+rm -rf $NODE_ARTIFACTS_PATH
+mkdir -p "$NODE_ARTIFACTS_PATH/npm_global"
+
+echo "Installing Node.js $NODE_LTS_VERSION..."
+
 CURL_FLAGS=(
   --fail          # Exit code 1 if request fails
   --compressed    # Request a compressed response should keep fetching fast
@@ -32,19 +57,16 @@ CURL_FLAGS=(
   --max-time 900  # 900 seconds is 15 minutes, evergreen times out at 20
 )
 
-mkdir -p "$NODE_ARTIFACTS_PATH/npm_global"
-
 # Comparisons are all case insensitive
 shopt -s nocasematch
 
 # index.tab is a sorted tab separated values file with the following headers
 # 0       1    2     3   4  5  6    7       8       9   10
 # version date files npm v8 uv zlib openssl modules lts security
-"$SCRIPT_DIR/retry-with-backoff.sh" curl "${CURL_FLAGS[@]}" "https://nodejs.org/dist/index.tab" --output node_index.tab
+"$SCRIPT_DIR/retry-with-backoff.sh" curl "${CURL_FLAGS[@]}" "https://nodejs.org/dist/index.tab" --output node_index.tab > /dev/null 2>&1
 
 while IFS=$'\t' read -r -a row; do
   node_index_version="${row[0]}"
-  echo $node_index_version >> "versions.txt"
   node_index_major_version=$(echo $node_index_version | sed -E 's/^v([0-9]+).*$/\1/')
   node_index_date="${row[1]}"
   [[ "$node_index_version" = "version" ]] && continue # skip tsv header
@@ -93,42 +115,39 @@ node_shasum_url="https://nodejs.org/dist/${node_index_version}/SHASUMS256.txt"
 
 echo "Node.js ${node_index_version} for ${operating_system}-${architecture} released on ${node_index_date}"
 
+"$SCRIPT_DIR/retry-with-backoff.sh" curl "${CURL_FLAGS[@]}" "${node_download_url}" --output "$node_archive_path" > /dev/null 2>&1
+"$SCRIPT_DIR/retry-with-backoff.sh" curl "${CURL_FLAGS[@]}" "${node_shasum_url}" --output "$node_shasum_path" > /dev/null 2>&1
+
+# Remove extra entries from the SHASUMS256.txt file. Not every OS supports the --ignore-missing flag.
+(
+  cd "$NODE_ARTIFACTS_PATH"
+  awk '{ if (system("[ -e \"" $2 "\" ]") == 0) print $0 }' SHASUMS256.txt > SHASUMS256.filtered.txt
+  sha256sum -c SHASUMS256.filtered.txt > /dev/null 2>&1 || sha256sum -c SHASUMS256.filtered.txt
+)
+
 if [[ "$file_extension" = "zip" ]]; then
-  if [[ -d "${NODE_ARTIFACTS_PATH}/nodejs/bin/${node_directory}" ]]; then
-    echo "Node.js already installed!"
-  else
-    "$SCRIPT_DIR/retry-with-backoff.sh" curl "${CURL_FLAGS[@]}" "${node_download_url}" --output "$node_archive_path"
-    "$SCRIPT_DIR/retry-with-backoff.sh" curl "${CURL_FLAGS[@]}" "${node_shasum_url}" --output "$node_shasum_path"
-    (cd "$NODE_ARTIFACTS_PATH" && sha256sum -c SHASUMS256.txt --ignore-missing)
     unzip -q "$node_archive_path" -d "${NODE_ARTIFACTS_PATH}"
     mkdir -p "${NODE_ARTIFACTS_PATH}/nodejs"
     # Windows "bins" are at the top level
     mv "${NODE_ARTIFACTS_PATH}/${node_directory}" "${NODE_ARTIFACTS_PATH}/nodejs/bin"
     # Need to add executable flag ourselves
     chmod +x "${NODE_ARTIFACTS_PATH}/nodejs/bin/node.exe"
     chmod +x "${NODE_ARTIFACTS_PATH}/nodejs/bin/npm"
-  fi
+    chmod +x "${NODE_ARTIFACTS_PATH}/nodejs/bin/npx"
+    chmod +x "${NODE_ARTIFACTS_PATH}/nodejs/bin/npx.CMD"
 else
-  if [[ -d "${NODE_ARTIFACTS_PATH}/nodejs/${node_directory}" ]]; then
-    echo "Node.js already installed!"
-  else
-    "$SCRIPT_DIR/retry-with-backoff.sh" curl "${CURL_FLAGS[@]}" "${node_download_url}" --output "$node_archive_path"
-    "$SCRIPT_DIR/retry-with-backoff.sh" curl "${CURL_FLAGS[@]}" "${node_shasum_url}" --output "$node_shasum_path"
-    pushd $NODE_ARTIFACTS_PATH
-    sha256sum -c SHASUMS256.txt --ignore-missing
-    popd
     tar -xf "$node_archive_path" -C "${NODE_ARTIFACTS_PATH}"
     mv "${NODE_ARTIFACTS_PATH}/${node_directory}" "${NODE_ARTIFACTS_PATH}/nodejs"
-  fi
 fi
+echo "$NODE_LTS_VERSION" > $NODE_ARTIFACTS_PATH/node-version.txt
+echo "Installing Node.js $NODE_LTS_VERSION... done."
 
 if [[ $operating_system != "win" ]]; then
   # Update npm to latest when we can
-  npm install --global npm@$NPM_VERSION
+  echo "Installing npm $NPM_VERSION..."
+  npm install --silent --global npm@$NPM_VERSION
   hash -r
+  echo "Installing npm $NPM_VERSION... done."
 fi
 
-echo "npm location: $(which npm)"
-echo "npm version: $(npm -v)"
-
-popd
+debug_output

.evergreen/orchestration/drivers_orchestration.py

@@ -24,11 +24,14 @@
 
 import psutil
 
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from mongodb_runner import start_mongodb_runner
+
 # Get global values.
 HERE = Path(__file__).absolute().parent
 EVG_PATH = HERE.parent
 DRIVERS_TOOLS = EVG_PATH.parent
-LOGGER = logging.getLogger(__name__)
+LOGGER = logging.getLogger("drivers_orchestration")
 logging.basicConfig(level=logging.INFO, format="%(levelname)-8s %(message)s")
 PLATFORM = sys.platform.lower()
 CRYPT_NAME_MAP = {
@@ -83,6 +86,11 @@ def get_options():
             action="store_true",
             help="Whether to use mongodb-atlas-local to start the server",
         )
+        parser.add_argument(
+            "--mongodb-runner",
+            action="store_true",
+            help="Whether to use mongodb-runner to start the server",
+        )
         parser.add_argument(
             "--orchestration-file", help="The name of the orchestration config file"
         )
@@ -231,6 +239,7 @@ def traverse(root):
         item["ipv6"] = False
         item["bind_ip"] = "0.0.0.0,::1"
         item["dbpath"] = f"/tmp/mongo-{item['port']}"
+        os.makedirs(item["dbpath"], exist_ok=True)
 
     if "routers" in data:
         for router in data["routers"]:
@@ -386,6 +395,7 @@ def run(opts):
     from mongosh_dl import main as mongosh_dl
 
     LOGGER.info("Running orchestration...")
+    stop(opts)
     clean_run(opts)
 
     # NOTE: in general, we need to normalize paths to account for cygwin/Windows.
@@ -414,9 +424,9 @@ def run(opts):
         args = f"{default_args} --version {version}"
         args += " --strip-path-components 2 --component archive"
         if not opts.existing_binaries_dir:
-            LOGGER.info(f"Downloading mongodb {version}...")
+            LOGGER.info(f"Downloading mongodb {version} to {mdb_binaries}...")
             mongodl(shlex.split(args))
-            LOGGER.info(f"Downloading mongodb {version}... done.")
+            LOGGER.info(f"Downloading mongodb {version} to {mdb_binaries}... done.")
         else:
             LOGGER.info(
                 f"Using existing mongod binaries dir: {opts.existing_binaries_dir}"
@@ -470,11 +480,20 @@ def run(opts):
     dl_end = datetime.now()
     mo_start = datetime.now()
 
+    data = get_orchestration_data(opts)
+
+    if opts.mongodb_runner and version in ("3.6", "4.0"):
+        LOGGER.warning(
+            "mongodb-runner does not support MongoDB < 4.2, using mongo-orchestration"
+        )
+        opts.mongodb_runner = False
+
     if opts.local_atlas:
         uri = start_atlas(opts)
+    elif opts.mongodb_runner:
+        uri = start_mongodb_runner(opts, data)
     else:
         mo_home = Path(opts.mongo_orchestration_home)
-        data = get_orchestration_data(opts)
 
         # Write the config file.
         orch_file = Path(mo_home / "config.json")
@@ -645,7 +664,7 @@ def shutdown_proc(proc: psutil.Process) -> None:
     try:
         proc.terminate()
         try:
-            proc.wait(10)  # Wait up to 10 seconds.
+            proc.wait(2)  # Wait up to 2 seconds.
         except psutil.TimeoutExpired:
             proc.kill()
     except Exception as e:
@@ -663,6 +682,7 @@ def shutdown_docker(docker: str, container_id: str) -> None:
 def stop(opts):
     mo_home = Path(opts.mongo_orchestration_home)
     pid_file = mo_home / "server.pid"
+    out_log = mo_home / "out.log"
     container_file = mo_home / "container_id.txt"
     docker = get_docker_cmd()
 
@@ -675,6 +695,26 @@ def stop(opts):
             shutdown_proc(psutil.Process(pid))
             LOGGER.info("Stopping mongo-orchestration using pid file... done.")
 
+    # Next try and use the output.log file as a serialized json file.
+    if out_log.exists():
+        try:
+            data = json.loads(out_log.read_text())
+        except Exception:
+            data = None
+        if data:
+            LOGGER.info("Stopping mongodb-runner cluster...")
+            all_servers = data["serialized"]["servers"]
+            for shard in data["serialized"].get("shards", []):
+                all_servers.extend(shard["servers"])
+            for server in all_servers:
+                pid = server["pid"]
+                if psutil.pid_exists(pid):
+                    shutdown_proc(psutil.Process(pid))
+                if Path(server["dbPath"]).exists():
+                    shutil.rmtree(server["dbPath"])
+            LOGGER.info("Stopping mongodb-runner cluster... done.")
+            out_log.unlink()
+
     # Next try using a docker container file.
     if docker is not None and container_file.exists():
         LOGGER.info("Stopping mongodb_atlas_local using container file...")
@@ -708,7 +748,7 @@ def stop(opts):
             response = subprocess.check_output(
                 shlex.split(cmd), encoding="utf-8"
             ).strip()
-        except subprocess.CalledProcessError as e:
+        except (subprocess.CalledProcessError, FileNotFoundError) as e:
             LOGGER.exception(e)
             response = ""
         for line in response.splitlines():
@@ -724,7 +764,7 @@ def stop(opts):
             name = proc.name()
         except psutil.NoSuchProcess:
             continue
-        if name in ["mongod", "mongos"]:
+        if name in ["mongod", "mongos", "mongod.exe", "mongos.exe"]:
             LOGGER.info(f"Stopping {name} by process name...")
             shutdown_proc(proc)
             LOGGER.info(f"Stopping {name} by process name... done.")