diff --git a/.evergreen/config.yml b/.evergreen/config.yml index b9562fc1..0c236b5d 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -568,6 +568,20 @@ functions: include_expansions_in_env: [AWS_SECRET_ACCESS_KEY, AWS_ACCESS_KEY_ID, AWS_SESSION_TOKEN] args: [src/.evergreen/tests/test-csfle.sh] + "run server test partial": + - command: subprocess.exec + type: test + params: + binary: bash + args: [src/.evergreen/tests/test-server.sh, partial] + + "run server test full": + - command: subprocess.exec + type: test + params: + binary: bash + args: [src/.evergreen/tests/test-server.sh] + "run cli test partial": - command: subprocess.exec type: test @@ -1103,6 +1117,11 @@ tasks: commands: - func: "run cli test full" + - name: "test-server-full" + tags: ["pr"] + commands: + - func: "run server test full" + - name: "test-ocsp" tags: ["pr", "ocsp"] commands: @@ -1118,6 +1137,11 @@ tasks: commands: - func: "run cli test partial" + - name: "test-server-partial" + tags: ["pr"] + commands: + - func: "run server test partial" + # }}} task_groups: @@ -1556,6 +1580,7 @@ buildvariants: - "test-install-binaries" - "test-csfle" - "test-cli-full" + - "test-server-full" - "test-happy-eyeballs" - "test-ocsp" - "test-8.0-standalone-require-api" @@ -1585,6 +1610,7 @@ buildvariants: add_tasks: - "test-install-binaries" - "test-cli-partial" + - "test-server-partial" - matrix_name: "tests-os-requires-70" matrix_spec: {"os-requires-70": "*", auth: "*", ssl: "*" } @@ -1608,6 +1634,7 @@ buildvariants: add_tasks: - "test-install-binaries" - "test-cli-partial" + - "test-server-partial" # Storage Engine Tests on Ubuntu 20.04 - matrix_name: "tests-storage-engines" diff --git a/.evergreen/docker/rhel8/base-entrypoint.sh b/.evergreen/docker/rhel8/base-entrypoint.sh index ae17993a..28241ddd 100755 --- a/.evergreen/docker/rhel8/base-entrypoint.sh +++ b/.evergreen/docker/rhel8/base-entrypoint.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash set -eu -# Remove the virtual env. +# Remove the virtual env and any node install. rm -rf $DRIVERS_TOOLS/.evergreen/venv || true +rm -rf $DRIVERS_TOOLS/.evergreen/node-artifacts || true # Start the server. cd $DRIVERS_TOOLS diff --git a/.evergreen/docker/ubuntu22.04/base-entrypoint.sh b/.evergreen/docker/ubuntu22.04/base-entrypoint.sh index ae17993a..28241ddd 100755 --- a/.evergreen/docker/ubuntu22.04/base-entrypoint.sh +++ b/.evergreen/docker/ubuntu22.04/base-entrypoint.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash set -eu -# Remove the virtual env. +# Remove the virtual env and any node install. rm -rf $DRIVERS_TOOLS/.evergreen/venv || true +rm -rf $DRIVERS_TOOLS/.evergreen/node-artifacts || true # Start the server. cd $DRIVERS_TOOLS diff --git a/.evergreen/init-node-and-npm-env.sh b/.evergreen/init-node-and-npm-env.sh index 68646612..a39e71e6 100755 --- a/.evergreen/init-node-and-npm-env.sh +++ b/.evergreen/init-node-and-npm-env.sh @@ -1,4 +1,5 @@ #! /usr/bin/env bash +# shellcheck shell=sh ## ## This script add the location of `npm` and `node` to the path. ## This is necessary because evergreen uses separate bash scripts for @@ -7,10 +8,12 @@ ## access to `npm`, `node`, or need to install something globally from ## npm. -SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) -. $SCRIPT_DIR/handle-paths.sh -NODE_ARTIFACTS_PATH="$SCRIPT_DIR/node-artifacts" -if [[ "${OS:-}" == "Windows_NT" ]]; then +# See https://stackoverflow.com/questions/35006457/choosing-between-0-and-bash-source/35006505#35006505 +# Why we need this syntax when sh is not aliased to bash (this script must be able to be called from sh) +# shellcheck disable=SC3028 +_SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0}) +NODE_ARTIFACTS_PATH="$_SCRIPT_DIR/node-artifacts" +if [ "${OS:-}" = "Windows_NT" ]; then NODE_ARTIFACTS_PATH=$(cygpath --unix "$NODE_ARTIFACTS_PATH") fi diff --git a/.evergreen/install-node.sh b/.evergreen/install-node.sh index 73cc9fbb..4840aea2 100755 --- a/.evergreen/install-node.sh +++ b/.evergreen/install-node.sh @@ -3,9 +3,13 @@ set -o errexit # Exit the script with error if any of the commands fail SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) . $SCRIPT_DIR/handle-paths.sh -pushd $SCRIPT_DIR -NODE_LTS_VERSION=${NODE_LTS_VERSION:-20} +DEFAULT_NODE_VERSION=20 +if grep -q "release 7" /etc/redhat-release 2> /dev/null; then + DEFAULT_NODE_VERSION=16 +fi +NODE_LTS_VERSION=${NODE_LTS_VERSION:-$DEFAULT_NODE_VERSION} + # If NODE_LTS_VERSION is numeric and less than 18, default to 9, if less than 20, default to 10. # Do not override if it is already set. if [[ "$NODE_LTS_VERSION" =~ ^[0-9]+$ && "$NODE_LTS_VERSION" -lt 18 ]]; then @@ -17,11 +21,32 @@ else fi export NPM_VERSION=${NPM_VERSION} -source "./init-node-and-npm-env.sh" +source "$SCRIPT_DIR/init-node-and-npm-env.sh" if [[ -z "${npm_global_prefix}" ]]; then echo "npm_global_prefix is unset" && exit 1; fi if [[ -z "${NODE_ARTIFACTS_PATH}" ]]; then echo "NODE_ARTIFACTS_PATH is unset" && exit 1; fi +function debug_output { + echo "node location: $(which node)" + echo "node version: $(node -v)" + echo "npm location: $(which npm)" + echo "npm version: $(npm -v)" + echo "Run 'source init-node-and-npm-env.sh' to handle environment setup." +} + +# Bail early if this version of node was already installed. +if grep -Fxq "$NODE_LTS_VERSION" $NODE_ARTIFACTS_PATH/node-version.txt 2> /dev/null; then + echo "Node $NODE_LTS_VERSION already installed!" + debug_output + exit 0 +fi + +# Ensure a clean directory. +rm -rf $NODE_ARTIFACTS_PATH +mkdir -p "$NODE_ARTIFACTS_PATH/npm_global" + +echo "Installing Node.js $NODE_LTS_VERSION..." + CURL_FLAGS=( --fail # Exit code 1 if request fails --compressed # Request a compressed response should keep fetching fast @@ -32,19 +57,16 @@ CURL_FLAGS=( --max-time 900 # 900 seconds is 15 minutes, evergreen times out at 20 ) -mkdir -p "$NODE_ARTIFACTS_PATH/npm_global" - # Comparisons are all case insensitive shopt -s nocasematch # index.tab is a sorted tab separated values file with the following headers # 0 1 2 3 4 5 6 7 8 9 10 # version date files npm v8 uv zlib openssl modules lts security -"$SCRIPT_DIR/retry-with-backoff.sh" curl "${CURL_FLAGS[@]}" "https://nodejs.org/dist/index.tab" --output node_index.tab +"$SCRIPT_DIR/retry-with-backoff.sh" curl "${CURL_FLAGS[@]}" "https://nodejs.org/dist/index.tab" --output node_index.tab > /dev/null 2>&1 while IFS=$'\t' read -r -a row; do node_index_version="${row[0]}" - echo $node_index_version >> "versions.txt" node_index_major_version=$(echo $node_index_version | sed -E 's/^v([0-9]+).*$/\1/') node_index_date="${row[1]}" [[ "$node_index_version" = "version" ]] && continue # skip tsv header @@ -93,13 +115,17 @@ node_shasum_url="https://nodejs.org/dist/${node_index_version}/SHASUMS256.txt" echo "Node.js ${node_index_version} for ${operating_system}-${architecture} released on ${node_index_date}" +"$SCRIPT_DIR/retry-with-backoff.sh" curl "${CURL_FLAGS[@]}" "${node_download_url}" --output "$node_archive_path" > /dev/null 2>&1 +"$SCRIPT_DIR/retry-with-backoff.sh" curl "${CURL_FLAGS[@]}" "${node_shasum_url}" --output "$node_shasum_path" > /dev/null 2>&1 + +# Remove extra entries from the SHASUMS256.txt file. Not every OS supports the --ignore-missing flag. +( + cd "$NODE_ARTIFACTS_PATH" + awk '{ if (system("[ -e \"" $2 "\" ]") == 0) print $0 }' SHASUMS256.txt > SHASUMS256.filtered.txt + sha256sum -c SHASUMS256.filtered.txt > /dev/null 2>&1 || sha256sum -c SHASUMS256.filtered.txt +) + if [[ "$file_extension" = "zip" ]]; then - if [[ -d "${NODE_ARTIFACTS_PATH}/nodejs/bin/${node_directory}" ]]; then - echo "Node.js already installed!" - else - "$SCRIPT_DIR/retry-with-backoff.sh" curl "${CURL_FLAGS[@]}" "${node_download_url}" --output "$node_archive_path" - "$SCRIPT_DIR/retry-with-backoff.sh" curl "${CURL_FLAGS[@]}" "${node_shasum_url}" --output "$node_shasum_path" - (cd "$NODE_ARTIFACTS_PATH" && sha256sum -c SHASUMS256.txt --ignore-missing) unzip -q "$node_archive_path" -d "${NODE_ARTIFACTS_PATH}" mkdir -p "${NODE_ARTIFACTS_PATH}/nodejs" # Windows "bins" are at the top level @@ -107,28 +133,21 @@ if [[ "$file_extension" = "zip" ]]; then # Need to add executable flag ourselves chmod +x "${NODE_ARTIFACTS_PATH}/nodejs/bin/node.exe" chmod +x "${NODE_ARTIFACTS_PATH}/nodejs/bin/npm" - fi + chmod +x "${NODE_ARTIFACTS_PATH}/nodejs/bin/npx" + chmod +x "${NODE_ARTIFACTS_PATH}/nodejs/bin/npx.CMD" else - if [[ -d "${NODE_ARTIFACTS_PATH}/nodejs/${node_directory}" ]]; then - echo "Node.js already installed!" - else - "$SCRIPT_DIR/retry-with-backoff.sh" curl "${CURL_FLAGS[@]}" "${node_download_url}" --output "$node_archive_path" - "$SCRIPT_DIR/retry-with-backoff.sh" curl "${CURL_FLAGS[@]}" "${node_shasum_url}" --output "$node_shasum_path" - pushd $NODE_ARTIFACTS_PATH - sha256sum -c SHASUMS256.txt --ignore-missing - popd tar -xf "$node_archive_path" -C "${NODE_ARTIFACTS_PATH}" mv "${NODE_ARTIFACTS_PATH}/${node_directory}" "${NODE_ARTIFACTS_PATH}/nodejs" - fi fi +echo "$NODE_LTS_VERSION" > $NODE_ARTIFACTS_PATH/node-version.txt +echo "Installing Node.js $NODE_LTS_VERSION... done." if [[ $operating_system != "win" ]]; then # Update npm to latest when we can - npm install --global npm@$NPM_VERSION + echo "Installing npm $NPM_VERSION..." + npm install --silent --global npm@$NPM_VERSION hash -r + echo "Installing npm $NPM_VERSION... done." fi -echo "npm location: $(which npm)" -echo "npm version: $(npm -v)" - -popd +debug_output diff --git a/.evergreen/orchestration/drivers_orchestration.py b/.evergreen/orchestration/drivers_orchestration.py index e706dec3..b1e87791 100644 --- a/.evergreen/orchestration/drivers_orchestration.py +++ b/.evergreen/orchestration/drivers_orchestration.py @@ -24,11 +24,14 @@ import psutil +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from mongodb_runner import start_mongodb_runner + # Get global values. HERE = Path(__file__).absolute().parent EVG_PATH = HERE.parent DRIVERS_TOOLS = EVG_PATH.parent -LOGGER = logging.getLogger(__name__) +LOGGER = logging.getLogger("drivers_orchestration") logging.basicConfig(level=logging.INFO, format="%(levelname)-8s %(message)s") PLATFORM = sys.platform.lower() CRYPT_NAME_MAP = { @@ -83,6 +86,11 @@ def get_options(): action="store_true", help="Whether to use mongodb-atlas-local to start the server", ) + parser.add_argument( + "--mongodb-runner", + action="store_true", + help="Whether to use mongodb-runner to start the server", + ) parser.add_argument( "--orchestration-file", help="The name of the orchestration config file" ) @@ -231,6 +239,7 @@ def traverse(root): item["ipv6"] = False item["bind_ip"] = "0.0.0.0,::1" item["dbpath"] = f"/tmp/mongo-{item['port']}" + os.makedirs(item["dbpath"], exist_ok=True) if "routers" in data: for router in data["routers"]: @@ -386,6 +395,7 @@ def run(opts): from mongosh_dl import main as mongosh_dl LOGGER.info("Running orchestration...") + stop(opts) clean_run(opts) # NOTE: in general, we need to normalize paths to account for cygwin/Windows. @@ -414,9 +424,9 @@ def run(opts): args = f"{default_args} --version {version}" args += " --strip-path-components 2 --component archive" if not opts.existing_binaries_dir: - LOGGER.info(f"Downloading mongodb {version}...") + LOGGER.info(f"Downloading mongodb {version} to {mdb_binaries}...") mongodl(shlex.split(args)) - LOGGER.info(f"Downloading mongodb {version}... done.") + LOGGER.info(f"Downloading mongodb {version} to {mdb_binaries}... done.") else: LOGGER.info( f"Using existing mongod binaries dir: {opts.existing_binaries_dir}" @@ -470,11 +480,20 @@ def run(opts): dl_end = datetime.now() mo_start = datetime.now() + data = get_orchestration_data(opts) + + if opts.mongodb_runner and version in ("3.6", "4.0"): + LOGGER.warning( + "mongodb-runner does not support MongoDB < 4.2, using mongo-orchestration" + ) + opts.mongodb_runner = False + if opts.local_atlas: uri = start_atlas(opts) + elif opts.mongodb_runner: + uri = start_mongodb_runner(opts, data) else: mo_home = Path(opts.mongo_orchestration_home) - data = get_orchestration_data(opts) # Write the config file. orch_file = Path(mo_home / "config.json") @@ -645,7 +664,7 @@ def shutdown_proc(proc: psutil.Process) -> None: try: proc.terminate() try: - proc.wait(10) # Wait up to 10 seconds. + proc.wait(2) # Wait up to 2 seconds. except psutil.TimeoutExpired: proc.kill() except Exception as e: @@ -663,6 +682,7 @@ def shutdown_docker(docker: str, container_id: str) -> None: def stop(opts): mo_home = Path(opts.mongo_orchestration_home) pid_file = mo_home / "server.pid" + out_log = mo_home / "out.log" container_file = mo_home / "container_id.txt" docker = get_docker_cmd() @@ -675,6 +695,26 @@ def stop(opts): shutdown_proc(psutil.Process(pid)) LOGGER.info("Stopping mongo-orchestration using pid file... done.") + # Next try and use the output.log file as a serialized json file. + if out_log.exists(): + try: + data = json.loads(out_log.read_text()) + except Exception: + data = None + if data: + LOGGER.info("Stopping mongodb-runner cluster...") + all_servers = data["serialized"]["servers"] + for shard in data["serialized"].get("shards", []): + all_servers.extend(shard["servers"]) + for server in all_servers: + pid = server["pid"] + if psutil.pid_exists(pid): + shutdown_proc(psutil.Process(pid)) + if Path(server["dbPath"]).exists(): + shutil.rmtree(server["dbPath"]) + LOGGER.info("Stopping mongodb-runner cluster... done.") + out_log.unlink() + # Next try using a docker container file. if docker is not None and container_file.exists(): LOGGER.info("Stopping mongodb_atlas_local using container file...") @@ -708,7 +748,7 @@ def stop(opts): response = subprocess.check_output( shlex.split(cmd), encoding="utf-8" ).strip() - except subprocess.CalledProcessError as e: + except (subprocess.CalledProcessError, FileNotFoundError) as e: LOGGER.exception(e) response = "" for line in response.splitlines(): @@ -724,7 +764,7 @@ def stop(opts): name = proc.name() except psutil.NoSuchProcess: continue - if name in ["mongod", "mongos"]: + if name in ["mongod", "mongos", "mongod.exe", "mongos.exe"]: LOGGER.info(f"Stopping {name} by process name...") shutdown_proc(proc) LOGGER.info(f"Stopping {name} by process name... done.") diff --git a/.evergreen/orchestration/mongodb_runner.py b/.evergreen/orchestration/mongodb_runner.py new file mode 100644 index 00000000..f01cbd5c --- /dev/null +++ b/.evergreen/orchestration/mongodb_runner.py @@ -0,0 +1,283 @@ +import argparse +import json +import logging +import os +import re +import shlex +import shutil +import stat +import subprocess +import sys +import tempfile +import uuid +from pathlib import Path +from typing import Any, Dict, List, Literal, Union +from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse + +TMPDIR = Path(tempfile.gettempdir()) / "drivers_orchestration" +TMPDIR.mkdir(exist_ok=True) +HERE = Path(__file__).absolute().parent +DRIVERS_TOOLS = HERE.parent.parent +S_IRUSR = stat.S_IRUSR # Unix owner read +LOGGER = logging.getLogger("drivers_orchestration") +PLATFORM = sys.platform.lower() + + +def _format_value(value): + value = str(value) + if value in ["True", "False"]: + value = value.lower() + return value + + +def _handle_proc_params(params: dict, args: List[str]): + found_enable_test_commands = False + for key, value in params.items(): + if isinstance(value, dict): + for subkey, subvalue in value.items(): + args.append(f"--{key}") + args.append(f"{subkey}={_format_value(subvalue)}") + if subkey == "enableTestCommands": + found_enable_test_commands = True + elif value is True: + args.append(f"--{key}") + elif value is not False: + args.append(f"--{key}") + args.append(_format_value(value)) + if not found_enable_test_commands: + args.append("--setParameter") + args.append("enableTestCommands=true") + + +def _normalize_path(path: Union[Path, str]) -> str: + if PLATFORM != "win32": + return str(path) + path = Path(path).as_posix() + return re.sub("/cygdrive/(.*?)(/)", r"\1://", path, count=1) + + +def start_mongodb_runner(opts, data): + mo_home = Path(opts.mongo_orchestration_home) + server_log = mo_home / "server.log" + out_log = mo_home / "out.log" + if out_log.exists(): + out_log.unlink() + config = _get_cluster_options(data, opts) + config["runnerDir"] = config["tmpDir"] + # Write the config file. + config_file = mo_home / "config.json" + config_file.write_text(json.dumps(config, indent=2)) + config_file = _normalize_path(config_file) + # Start the runner using node. + # TODO: this will use npx once it is ready. + node = shutil.which("node") + node = _normalize_path(node) + target = HERE / "devtools-shared/packages/mongodb-runner/bin/runner.js" + target = _normalize_path(target) + cmd = f"{node} {target} start --debug --config {config_file}" + LOGGER.info(f"Running mongodb-runner using {node} {target}...") + try: + with server_log.open("w") as fid: + subprocess.check_call( + shlex.split(cmd), stdout=fid, stderr=subprocess.STDOUT + ) + except subprocess.CalledProcessError as e: + LOGGER.error("server.log: %s", server_log.read_text()) + LOGGER.error(str(e)) + raise e + LOGGER.info(f"Running mongodb-runner using {node} {target}... done.") + cluster_file = Path(config["runnerDir"]) / f"m-{config['id']}.json" + server_info = json.loads(cluster_file.read_text()) + cluster_file.unlink() + out_log.write_text(json.dumps(server_info, indent=2)) + + # Get the connection string, keeping only the replicaSet query param. + parsed = urlparse(server_info["connectionString"]) + query_params = dict(parse_qsl(parsed.query)) + new_query = {k: v for k, v in query_params.items() if k == "replicaSet"} + return urlunparse(parsed._replace(query=urlencode(new_query))) + + +def _get_cluster_options(input: dict, opts: Any, static=False) -> Dict[str, Any]: + id_ = uuid.uuid4().hex + rs_members = [] + users = [] + shard_args = [] + mongos_args = [] + tmp_dir = TMPDIR + args: List[str] = [] + roles = [ + {"role": "userAdminAnyDatabase", "db": "admin"}, + {"role": "clusterAdmin", "db": "admin"}, + {"role": "dbAdminAnyDatabase", "db": "admin"}, + {"role": "readWriteAnyDatabase", "db": "admin"}, + {"role": "restore", "db": "admin"}, + {"role": "backup", "db": "admin"}, + ] + + topology: Literal["standalone", "replset", "sharded"] = "standalone" + if opts.topology == "replica_set": + topology = "replset" + elif opts.topology == "sharded_cluster": + topology = "sharded" + + # Top level options + skip_keys = [ + "shards", + "requireApiVersion", + "sslParams", + "routers", + "members", + "login", + "password", + "id", + "name", + "procParams", + ] + for key, value in input.items(): + if key in skip_keys: + continue + if key == "auth_key": + if static: + key_file = "KEY_FILE_PATH" + else: + Path(tmp_dir).mkdir(parents=True, exist_ok=True) + key_file = os.path.join(tmp_dir, f"key-file-{id_}.txt") + with open(key_file, "w") as f: + f.write(input["auth_key"]) + os.chmod(key_file, S_IRUSR) + args.extend(["--keyFile", _normalize_path(key_file)]) + elif value is True: + args.append(f"--{key}") + elif value is not False: + args.append(f"--{key}") + args.append(_format_value(value)) + + if topology == "standalone": + if "procParams" in input: + _handle_proc_params(input["procParams"], args) + + if topology == "replset": + args.append("--replSet") + args.append(str(input["id"])) + for member in input["members"]: + member_rs_options = { + "args": [], + "tags": {}, + "priority": 1, + } + rs_params = member.get("rsParams") + if rs_params: + if "tags" in rs_params: + member_rs_options["tags"] = rs_params["tags"] + if rs_params.get("arbiterOnly"): + member_rs_options["priority"] = 0 + member_rs_options["arbiterOnly"] = True + if "priority" in rs_params: + member_rs_options["priority"] = rs_params["priority"] + if "procParams" in member: + _handle_proc_params(member["procParams"], member_rs_options["args"]) + rs_members.append(member_rs_options) + + # Sharded/topology code + if topology == "sharded": + # Add a blank config srv to start, it must be the first shard. + shard_args = [{"args": [], "rsMembers": [{}]}] + for shard in input["shards"]: + is_config_srv = False + this_shard_options = {"args": [], "rsMembers": []} + for member in shard["shardParams"]["members"]: + member_args = [] + _handle_proc_params(member["procParams"], member_args) + if "--shardsvr" in member_args: + member_args.remove("--shardsvr") + elif "--configsvr" in member_args: + is_config_srv = True + member_args.remove("--configsvr") + this_shard_options["rsMembers"].append({"args": member_args}) + if is_config_srv: + shard_args[0] = this_shard_options + else: + shard_args.append(this_shard_options) + for router in input["routers"]: + this_router_args = [] + _handle_proc_params(router, this_router_args) + mongos_args.append(this_router_args) + + # TLS/SSL options + if "sslParams" in input: + for key, value in input["sslParams"].items(): + if key == "sslPEMKeyFile": + key = "tlsCertificateKeyFile" # noqa: PLW2901 + if key == "sslCAFile": + key = "tlsCAFile" # noqa: PLW2901 + if value is True: + args.append(f"--{key}") + elif value is not False: + args.append(f"--{key}") + args.append(_format_value(value)) + + if input.get("login"): + users.append( + { + "username": input["login"], + "password": input["password"], + "roles": roles, + } + ) + + output = {"topology": topology, "args": args} + if users: + output["users"] = users + if topology == "replset": + output["rsMembers"] = rs_members + elif topology == "sharded": + output["mongosArgs"] = mongos_args + output["shards"] = shard_args + if "requireApiVersion" in input: + output["requireApiVersion"] = input["requireApiVersion"] + + if not static: + output["id"] = uuid.uuid4().hex + output["tmpDir"] = str(tmp_dir) + output["binDir"] = str(opts.mongodb_binaries) + if sys.platform != "win32": + args.extend(["--unixSocketPrefix", "/tmp"]) + + return output + + +def main(): + parser = argparse.ArgumentParser(description="MongoDB Runner Config Migrator") + + parser.add_argument( + "--input-file", type=str, required=True, help="Path to the input file" + ) + parser.add_argument( + "--output-file", type=str, required=True, help="Path to the output file" + ) + parser.add_argument( + "--mongo-orchestration-home", + type=str, + required=False, + help="Path to mongo-orchestration home", + ) + parser.add_argument( + "--topology", + type=str, + required=True, + choices=["standalone", "replica_set", "sharded_cluster"], + help="Server deployment topology (standalone, replica_set, sharded_cluster)", + ) + + opts = parser.parse_args() + with open(opts.input_file) as fid: + data = json.load(fid) + + new_data = _get_cluster_options(data, opts, static=True) + with open(opts.output_file, "w") as fid: + json.dump(new_data, fid, indent=2) + + +if __name__ == "__main__": + main() diff --git a/.evergreen/orchestration/setup.sh b/.evergreen/orchestration/setup.sh index de142734..234f89c8 100755 --- a/.evergreen/orchestration/setup.sh +++ b/.evergreen/orchestration/setup.sh @@ -4,6 +4,7 @@ set -o errexit SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")" +_HERE=${SCRIPT_DIR} . "${SCRIPT_DIR:?}/../handle-paths.sh" export DRIVERS_TOOLS_INSTALL_CLI_OVERRIDES @@ -20,3 +21,19 @@ esac # and $MONGO_ORCHESTRATION_HOME) and the parent directory ($DRIVERS_TOOLS). bash "${SCRIPT_DIR:?}/../install-cli.sh" "${SCRIPT_DIR:?}/.." bash "${SCRIPT_DIR:?}/../install-cli.sh" "${SCRIPT_DIR:?}" + +# TODO: this won't be necessary once #704 is merged. +if [ ! -d "$HERE/../node-artifacts" ]; then + NODE_LTS_VERSION=22 bash $_HERE/../install-node.sh +fi + +# Install the in-progress branch of mongodb-runner. +# TODO: remove once we can use npx. +if [ ! -d $_HERE/devtools-shared ]; then + source $_HERE/../init-node-and-npm-env.sh + git clone -b drivers-tools-followup https://github.com/blink1073/devtools-shared $_HERE/devtools-shared + pushd $_HERE/devtools-shared + npm install --ignore-scripts + npx -y lerna run --scope=mongodb-runner --include-dependencies compile + popd +fi diff --git a/.evergreen/run-orchestration.sh b/.evergreen/run-orchestration.sh index f3565bfa..aa407323 100755 --- a/.evergreen/run-orchestration.sh +++ b/.evergreen/run-orchestration.sh @@ -36,4 +36,6 @@ SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0}) # Ensure the CLIs are up to date. bash $SCRIPT_DIR/orchestration/setup.sh -$SCRIPT_DIR/orchestration/drivers-orchestration run "$@" +export DEBUG=mongodb-runner +. $SCRIPT_DIR/init-node-and-npm-env.sh +$SCRIPT_DIR/orchestration/drivers-orchestration run --mongodb-runner "$@" diff --git a/.evergreen/run-server.sh b/.evergreen/run-server.sh new file mode 100755 index 00000000..d004be84 --- /dev/null +++ b/.evergreen/run-server.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# shellcheck shell=sh +set -eu + +# Supported environment variables: +# AUTH Set to "auth" to enable authentication. Defaults to "noauth" +# ARCH Set to the target architecture of the server binaries (e.g. "x86_64", "arm64", etc.). Defaults to the architecture of the machine running the script. +# SSL Set to "yes" to enable SSL. Defaults to "nossl" +# TOPOLOGY Set to "server", "replica_set", or "sharded_cluster". Defaults to "server" (i.e. standalone). +# MONGODB_VERSION Set the MongoDB version to use. Defaults to "latest". +# MONGODB_DOWNLOAD_URL Set the MongoDB download URL to use for download-mongodb.sh. +# ORCHESTRATION_FILE Set the /.json configuration. +# STORAGE_ENGINE Set to a non-empty string to use the /.json configuration (e.g. STORAGE_ENGINE=inmemory). +# REQUIRE_API_VERSION Set to a non-empty string to set the requireApiVersion parameter. Currently only supported for standalone servers. +# DISABLE_TEST_COMMANDS Set to a non-empty string to use the /disableTestCommands.json configuration (e.g. DISABLE_TEST_COMMANDS=1). +# SKIP_CRYPT_SHARED Set to a non-empty string to skip downloading crypt_shared +# LOAD_BALANCER Set to a non-empty string to enable load balancer. Only supported for sharded clusters. +# AUTH_AWS Set to a non-empty string to enable MONGODB-AWS authentication. +# PYTHON Set the Python binary to use. +# LOCAL_ATLAS Set to use mongodb-atlas-local to start the server. +# INSTALL_LEGACY_SHELL Set to a non-empty string to install the legacy mongo shell. +# TLS_PEM_KEY_FILE Set a .pem file that contains the TLS certificate and key for the server +# TLS_CA_FILE Set a .pem file that contains the root certificate chain for the server + +# The following variables affect the mongo-orchestration server: +# MONGO_ORCHESTRATION_HOME Set the path where the files used by mongo-orchestration will be located. +# MONGODB_BINARIES Set the path to the MONGODB_BINARIES for mongo-orchestration. +# TLS_CERT_KEY_FILE Set a .pem file to be used as the tlsCertificateKeyFile option in mongo-orchestration. + +# See https://stackoverflow.com/questions/35006457/choosing-between-0-and-bash-source/35006505#35006505 +# Why we need this syntax when sh is not aliased to bash (this script must be able to be called from sh) +# shellcheck disable=SC3028 +SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0}) +. $SCRIPT_DIR/handle-paths.sh + +# Ensure the CLIs are up to date. +bash $SCRIPT_DIR/orchestration/setup.sh + +$SCRIPT_DIR/orchestration/drivers-orchestration run --mongodb-runner "$@" diff --git a/.evergreen/stop-server.sh b/.evergreen/stop-server.sh new file mode 100755 index 00000000..c24fe61e --- /dev/null +++ b/.evergreen/stop-server.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# shellcheck shell=sh + +set -o errexit # Exit the script with error if any of the commands fail + +# See https://stackoverflow.com/questions/35006457/choosing-between-0-and-bash-source/35006505#35006505 +# Why we need this syntax when sh is not aliased to bash (this script must be able to be called from sh) +# shellcheck disable=SC3028 +SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0}) +. $SCRIPT_DIR/handle-paths.sh + +# Ensure the CLIs are up to date. +bash $SCRIPT_DIR/orchestration/setup.sh + +$SCRIPT_DIR/orchestration/drivers-orchestration stop "$@" diff --git a/.evergreen/tests/test-ocsp.sh b/.evergreen/tests/test-ocsp.sh index 8293a7e2..5a808cd4 100755 --- a/.evergreen/tests/test-ocsp.sh +++ b/.evergreen/tests/test-ocsp.sh @@ -20,6 +20,8 @@ export ORCHESTRATION_FILE export OCSP_ALGORITHM export SERVER_TYPE +. $SCRIPT_DIR/../init-node-and-npm-env.sh + # # Start a MongoDB server with ocsp enabled. SSL="ssl" make -C ${DRIVERS_TOOLS} run-server diff --git a/.evergreen/tests/test-server.sh b/.evergreen/tests/test-server.sh new file mode 100755 index 00000000..4e47f460 --- /dev/null +++ b/.evergreen/tests/test-server.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash + +# Test usage of start-server.sh +set -eu + +SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) +. $SCRIPT_DIR/../handle-paths.sh +. $SCRIPT_DIR/../find-python3.sh + +pushd $SCRIPT_DIR/.. > /dev/null + +. ./init-node-and-npm-env.sh + +# Connect to the MongoDB server using tls +# shellcheck disable=SC2120 +function connect_mongodb() { + local use_tls=false + + # Parse flags + while [[ $# -gt 0 ]]; do + case "$1" in + --ssl) use_tls=true; shift ;; + *) echo "Unknown option: $1"; return 1 ;; + esac + done + + URI="mongodb://localhost:27017/?directConnection=true&serverSelectionTimeoutMS=10000" + local TLS_OPTS=() + if [[ "$use_tls" == "true" ]]; then + TLS_OPTS+=("--tls" "--tlsCertificateKeyFile" "${DRIVERS_TOOLS}/.evergreen/x509gen/server.pem") + TLS_OPTS+=("--tlsCAFile" "${DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem") + fi + echo "Connecting to server..." + # shellcheck disable=SC2068 + $MONGODB_BINARIES/mongosh "$URI" ${TLS_OPTS[@]:-} --eval "db.runCommand({\"ping\":1})" + echo "Connecting to server... done." +} + +# Test for default, then test cli options. +bash ./run-server.sh +connect_mongodb + +bash ./run-server.sh --topology standalone --auth +connect_mongodb + +bash ./run-server.sh --version 7.0 --topology replica_set --ssl +connect_mongodb --ssl + +bash ./run-server.sh --version latest --topology sharded_cluster --auth --ssl +connect_mongodb --ssl + +# Ensure that we can use a downloaded mongodb directory. +DOWNLOAD_DIR=mongodl_test +rm -rf ${DOWNLOAD_DIR} +bash install-cli.sh "$(pwd)/orchestration" +PYTHON="$(ensure_python3 2>/dev/null)" +$PYTHON mongodl.py --edition enterprise --version 7.0 --component archive --out ${DOWNLOAD_DIR} --strip-path-components 2 --retries 5 +bash ./run-server.sh --existing-binaries-dir=${DOWNLOAD_DIR} +${DOWNLOAD_DIR}/mongod --version | grep v7.0 + +if [ "${1:-}" == "partial" ]; then + popd > /dev/null + make -C ${DRIVERS_TOOLS} test + exit 0 +fi + +for version in rapid 8.0 6.0 5.0 4.4 4.2 +do + bash ./run-server.sh --version "$version" + connect_mongodb +done + +popd > /dev/null +make -C ${DRIVERS_TOOLS} test diff --git a/.gitignore b/.gitignore index 2fb92e3d..722b9a04 100644 --- a/.gitignore +++ b/.gitignore @@ -145,3 +145,7 @@ uv-override-dependencies.txt .python_packages/ appsettings.json local.settings.json + + +# TODO: remove before merging +devtools-shared