Integrate docker tooling with Mongoid

 (#4955)

* Update spec/shared

* Integrate docker tooling with Mongoid

* remove mongo-orchestration

* remove --server flag since jruby claims it is useless

* update distro names

* remove redundant configuration

* set MONGODB_URI

* run old servers on 14.04

* switch more to ubuntu1404

* separate ubuntu1404 and ubuntu1604 configurations

* respect TEST_CMD for docker

* Use ubuntu1604 for app tests to get a newer nodejs runtime

* Install nodejs for Mongoid app tests

* ubuntu1604 is still too old, try ubuntu1804

* use headless jres

* only install Mongoid app testing bits on ubuntu1804

* apt update & install in the same command

* respect INTERACTIVE env var for docker

* fix uri options addition

* construct URI for app tests more intelligently

Co-authored-by: Oleg Pudeyev <[email protected]>

.evergreen/.gitignore

@@ -0,0 +1,2 @@
+/Dockerfile
+

.evergreen/config.yml

@@ -65,11 +65,6 @@ functions:
 
            export DRIVERS_TOOLS="$(pwd)/../drivers-tools"
 
-           # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory
-           if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
-              export DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS)
-           fi
-
            export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration"
            export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin"
            export UPLOAD_BUCKET="${project}"
@@ -110,20 +105,6 @@ functions:
       params:
         file: src/expansion.yml
 
-  "prepare resources":
-    - command: shell.exec
-      params:
-        script: |
-          ${PREPARE_SHELL}
-          rm -rf $DRIVERS_TOOLS
-          if [ "${project}" = "drivers-tools" ]; then
-            # If this was a patch build, doing a fresh clone would not actually test the patch
-            cp -R ${PROJECT_DIRECTORY}/ $DRIVERS_TOOLS
-          else
-            git clone git://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS
-          fi
-          echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" > $MONGO_ORCHESTRATION_HOME/orchestration.config
-
   "exec script" :
     - command: shell.exec
       type: test
@@ -286,17 +267,6 @@ functions:
       params:
         file: ./src/tmp/rspec.xml
 
-  "bootstrap mongo-orchestration":
-    - command: shell.exec
-      params:
-        script: |
-          ${PREPARE_SHELL}
-          sh ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh
-    # run-orchestration generates expansion file with the MONGODB_URI for the cluster
-    - command: expansions.update
-      params:
-        file: mo-expansion.yml
-
   "run tests":
     - command: shell.exec
       type: test
@@ -315,22 +285,6 @@ functions:
             TEST_I18N_FALLBACKS="${TEST_I18N_FALLBACKS}" \
             ./egos .evergreen/run-tests.sh
 
-  "cleanup":
-    - command: shell.exec
-      params:
-        script: |
-          ${PREPARE_SHELL}
-          cd "$MONGO_ORCHESTRATION_HOME"
-          # source the mongo-orchestration virtualenv if it exists
-          if [ -f venv/bin/activate ]; then
-            . venv/bin/activate
-          elif [ -f venv/Scripts/activate ]; then
-            . venv/Scripts/activate
-          fi
-          mongo-orchestration stop
-          cd -
-          rm -rf $DRIVERS_TOOLS || true
-
   "fix absolute paths":
     - command: shell.exec
       params:
@@ -354,7 +308,6 @@ pre:
   - func: "fetch source"
   - func: "fetch egos"
   - func: "create expansions"
-  - func: "prepare resources"
   - func: "fix absolute paths"
   - func: "install dependencies"
 
@@ -364,12 +317,10 @@ post:
   - func: "upload mo artifacts"
   #- func: "upload test results"
   - func: "upload test results to s3"
-  - func: "cleanup"
 
 tasks:
     - name: "test"
       commands:
-        - func: "bootstrap mongo-orchestration"
         - func: "run tests"
 
 axes:
@@ -483,13 +434,9 @@ axes:
   - id: "os"
     display_name: OS
     values:
-      - id: ubuntu1204-test
-        display_name: "Ubuntu 12.04"
-        run_on: ubuntu1204-test
-
       - id: ubuntu-14.04
         display_name: "Ubuntu 14.04"
-        run_on: ubuntu1404-test
+        run_on: ubuntu1404-small
 
       - id: rhel70
         display_name: "RHEL 7.0"
@@ -596,23 +543,34 @@ buildvariants:
   matrix_spec:
     driver: current
     ruby: "ruby-2.6"
-    mongodb-version: '*'
+    mongodb-version: ['3.6', '4.0', '4.2', '4.4']
     topology: ["replica-set", "sharded-cluster"]
   display_name: "${ruby}, ${driver}, ${mongodb-version}, ${topology}"
   run_on:
-    - rhel70-small
+    - ubuntu1604-small
+  tasks:
+     - name: "test"
+- matrix_name: "topologies-old"
+  matrix_spec:
+    driver: current
+    ruby: "ruby-2.6"
+    mongodb-version: ['2.6', '3.0', '3.2', '3.4']
+    topology: ["replica-set", "sharded-cluster"]
+  display_name: "${ruby}, ${driver}, ${mongodb-version}, ${topology}"
+  run_on:
+    - ubuntu1404-small
   tasks:
      - name: "test"
 - matrix_name: "rubies"
   matrix_spec:
     driver: "current"
     # ruby-head temporarily omitted: https://jira.mongodb.org/browse/MONGOID-4833
-    ruby: [ruby-2.6, ruby-2.3]
+    ruby: [ruby-2.3]
     mongodb-version: "4.2"
     topology: "replica-set"
   display_name: "${ruby}, ${driver}, ${mongodb-version}, ${topology}"
   run_on:
-    - ubuntu1604-test
+    - ubuntu1604-small
   tasks:
      - name: "test"
 - matrix_name: "mongodb"
@@ -634,7 +592,7 @@ buildvariants:
     topology: "standalone"
   display_name: "${ruby}, ${driver}, ${mongodb-version}, ${topology}"
   run_on:
-    - rhel70-small
+    - ubuntu1404-small
   tasks:
      - name: "test"
 - matrix_name: "jruby-new"
@@ -689,7 +647,7 @@ buildvariants:
     topology: "standalone"
   display_name: "${ruby}, ${driver}, ${mongodb-version}, ${topology}"
   run_on:
-    - rhel70-small
+    - ubuntu1404-small
   tasks:
      - name: "test"
 - matrix_name: "driver-stable-new"
@@ -711,7 +669,7 @@ buildvariants:
     topology: "standalone"
   display_name: "${ruby}, ${driver}, ${mongodb-version}, ${topology}"
   run_on:
-    - rhel70-small
+    - ubuntu1404-small
   tasks:
      - name: "test"
 - matrix_name: "driver-oldstable"
@@ -824,7 +782,7 @@ buildvariants:
     app-tests: yes
   display_name: "app tests ${driver}, ${ruby}"
   run_on:
-    - rhel70-small
+    - ubuntu1804-small
   tasks:
      - name: "test"
 - matrix_name: app-tests-jruby
@@ -836,6 +794,6 @@ buildvariants:
     app-tests: yes
   display_name: "app tests ${driver}, ${jruby}"
   run_on:
-    - rhel70-small
+    - ubuntu1804-small
   tasks:
      - name: "test"

.evergreen/functions.sh

@@ -21,7 +21,7 @@ set_env_vars() {
   fi
   export CI=evergreen
   # JRUBY_OPTS were initially set for Mongoid
-  export JRUBY_OPTS="--server -J-Xms512m -J-Xmx1G"
+  export JRUBY_OPTS="-J-Xms512m -J-Xmx1G"
 
   if test -n "$SINGLE_MONGOS"; then
     # Tests which perform query count assertions are incompatible with 
@@ -49,22 +49,3 @@ kill_jruby() {
     for pid in $(ps -ef | grep "jruby" | grep -v grep | awk '{print $2}'); do kill -9 $pid; done
   fi
 }
-
-prepare_server() {
-  arch=$1
-  version=$2
-
-  url=http://downloads.10gen.com/linux/mongodb-linux-x86_64-enterprise-$arch-$version.tgz
-  mongodb_dir="$MONGO_ORCHESTRATION_HOME"/mdb
-  mkdir -p "$mongodb_dir"
-  curl $url |tar xz -C "$mongodb_dir" -f -
-  BINDIR="$mongodb_dir"/`basename $url |sed -e s/.tgz//`/bin
-  export PATH="$BINDIR":$PATH
-}
-
-install_mlaunch() {
-  pythonpath="$MONGO_ORCHESTRATION_HOME"/python
-  pip install -t "$pythonpath" 'mtools[mlaunch]'
-  export PATH="$pythonpath/bin":$PATH
-  export PYTHONPATH="$pythonpath"
-}

.evergreen/get-mongodb-download-url

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+
+desired_version, arch = ARGV
+if arch.nil?
+  STDERR.puts "Usage: get-mongodb-download-url desired-version arch"
+  exit 1
+end
+
+$: << File.join(File.dirname(__FILE__), '../spec/shared/lib')
+require 'mrss/server_version_registry'
+
+puts Mrss::ServerVersionRegistry.new(desired_version, arch).download_url

.evergreen/patch-debuggers

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# Patches debuggers to not ask for confirmation on exit.
+# byebug tracking issue: https://github.com/deivid-rodriguez/byebug/issues/404
+# byebug proposed patch: https://github.com/deivid-rodriguez/byebug/pull/605
+
+root="$1"
+if test -z "$root"; then
+  root=$HOME/.rbenv
+fi
+
+find "$root" -name quit.rb -path '*/byebug/*' -exec \
+  sed -e '/quit.confirmations.really/d' -i {} \;
+
+# JRuby ruby-debug
+
+find "$root" -name quit.rb -path '*/ruby-debug/*' -exec \
+  sed -e 's/confirm("Really quit.*")/true/' -i {} \;

.evergreen/run-tests.sh

@@ -10,13 +10,31 @@ set -o errexit  # Exit the script with error if any of the commands fail
 
 . `dirname "$0"`/../spec/shared/shlib/distro.sh
 . `dirname "$0"`/../spec/shared/shlib/set_env.sh
+. `dirname "$0"`/../spec/shared/shlib/server.sh
 . `dirname "$0"`/functions.sh
 . `dirname "$0"`/functions.sh
 
+arch=`host_distro`
+
 set_fcv
 set_env_vars
 set_env_ruby
 
+prepare_server $arch
+
+install_mlaunch_virtualenv
+
+# Launching mongod under $MONGO_ORCHESTRATION_HOME
+# makes its log available through log collecting machinery
+
+export dbdir="$MONGO_ORCHESTRATION_HOME"/db
+mkdir -p "$dbdir"
+
+calculate_server_args
+launch_server "$dbdir"
+
+uri_options="$URI_OPTIONS"
+
 which bundle
 bundle --version
 
@@ -66,7 +84,11 @@ fi
 
 export BUNDLE_GEMFILE
 
-if test -n "$TEST_I18N_FALLBACKS"; then
+export MONGODB_URI="mongodb://localhost:27017/?appName=test-suite&$uri_options"
+
+if test -n "$TEST_CMD"; then
+  eval $TEST_CMD
+elif test -n "$TEST_I18N_FALLBACKS"; then
   bundle exec rspec spec/integration/i18n_fallbacks_spec.rb
 elif test -n "$APP_TESTS"; then
   # Need recent node for rails
@@ -80,3 +102,5 @@ elif test -n "$APP_TESTS"; then
 else
   bundle exec rake ci
 fi
+
+python -m mtools.mlaunch.mlaunch stop --dir "$dbdir"

.evergreen/test-on-docker

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+$: << File.join(File.dirname(__FILE__), '../spec/shared/lib')
+require 'mrss/docker_runner'
+
+Mrss::DockerRunner.new(
+  image_tag: 'mongoid-test',
+  dockerfile_path: '.evergreen/Dockerfile',
+  default_script: 'bash -x .evergreen/run-tests.sh',
+  project_lib_subdir: 'mongoid',
+).run

spec/integration/app_spec.rb

@@ -181,11 +181,44 @@ def clone_application(repo_url, subdir: nil, rails_version: nil)
     end
   end
 
+  def parse_mongodb_uri(uri)
+    pre, query = uri.split('?', 2)
+    if pre =~ %r,\A(mongodb(?:.*?))://([^/]+)(?:/(.*))?\z,
+      protocol = $1
+      hosts = $2
+      database = $3
+      if database == ''
+        database = nil
+      end
+    else
+      raise ArgumentError, "Invalid MongoDB URI: #{uri}"
+    end
+    if query == ''
+      query = nil
+    end
+    {
+      protocol: protocol,
+      hosts: hosts,
+      database: database,
+      query: query,
+    }
+  end
+
+  def build_mongodb_uri(parts)
+    "#{parts.fetch(:protocol)}://#{parts.fetch(:hosts)}/#{parts[:database]}?#{parts[:query]}"
+  end
+
   def write_mongoid_yml
+    # HACK: the driver does not provide a MongoDB URI parser and assembler,
+    # and the Ruby standard library URI module doesn't handle multiple hosts.
+    parts = parse_mongodb_uri(SpecConfig.instance.uri_str)
+    parts[:database] = 'mongoid_test'
+    uri = build_mongodb_uri(parts)
+    p uri
     env_config = {'clients' => {'default' => {
       # TODO massive hack, will fail if uri specifies a database name or
       # any uri options
-      'uri' => "#{SpecConfig.instance.uri_str}/mongoid_test",
+      'uri' => uri,
     }}}
     config = {'development' => env_config, 'production' => env_config}
     File.open('config/mongoid.yml', 'w') do |f|

spec/lite_spec_helper.rb

@@ -53,7 +53,7 @@
     config.add_formatter(RSpec::Core::Formatters::JsonFormatter, File.join(File.dirname(__FILE__), '../tmp/rspec.json'))
   end
 
-  if SpecConfig.instance.ci?
+  if SpecConfig.instance.ci? && !%w(1 true yes).include?(ENV['INTERACTIVE']&.downcase)
     timeout = if SpecConfig.instance.app_tests?
       # Allow 5 minutes per test for the app tests, since they install
       # gems for Rails applications which can take a long time.