Fix RHEL 8 provisioning in tests (#769)

This fixes several things:
- Drop tests for 4.0 and earlier.
- Newer RHEL 8.0 builds are published as just `rhel-8` rather than
`rhel-80`. We now treat these identically.
- Ignore release-candidate releases when iterating the release feed.
Thus, testing `8.0` tests (as of this writing) 8.0.5 rather than
8.0.6-rc1.
- Test no server versions newer than a hard-coded max version, which is
currently 8.0.999. Thus, `latest` tests (as of this writing) 8.0.6-rc1
rather than an 8.1.0 alpha release.
- mongodump testing now makes 6.0 and 7.0 builds accommodate mixed
time-series schemas as needed.
- (Unrelated, but worthy) Checks for
CannotInsertTimeseriesBucketsWithMixedSchema errors now use
mongo.ServerError.HasErrorCode(), which is more comprehensive than
type-asserting to specific concrete error types.

Additionally, this accommodates the following recent `mongo` shell
changes:
- SERVER-91804: Test runs now fail by default if child processes remain
unterminated. We now always set `TestData.ignoreUnterminatedProcesses`
to true.
- SERVER-81339: The `ReplSetTest` global no longer exists by default.
This changeset copies replsettest.js from v8.0 and imports it when the
`mongo` version matches.

Author: FGasper

Diff for: .eslintignore

@@ -1,2 +1,4 @@
 # eslint fails to parse this file for some reason
 test/qa-tests/jstests/ssl/ssl_with_system_ca.js
+
+test/qa-tests/jstests/libs/replsettest-*.js

Diff for: buildscript/build.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -275,9 +276,18 @@ func selectedPkgs(ctx *task.Context) []string {
 	return selectedPkgs
 }
 
-func getPlatform() (platform.Platform, error) {
+func getPlatform() (pf platform.Platform, err error) {
 	if os.Getenv("CI") != "" {
-		return platform.GetFromEnv()
+		pf, err = platform.GetFromEnv()
+		if err == nil {
+			log.Printf("Platform from env: %+v\n", pf)
+		}
+	} else {
+		pf, err = platform.DetectLocal()
+		if err == nil {
+			log.Printf("Platform detected: %+v\n", pf)
+		}
 	}
-	return platform.DetectLocal()
+
+	return
 }

Diff for: common.yml

@@ -771,119 +771,6 @@ tasks:
       - command: expansions.update
       - func: "generate full JSON feed"
 
-  - name: integration-3.6
-    tags: ["3.6"]
-    commands:
-      - func: "fetch source"
-      - command: expansions.update
-      - func: "download mongod and shell"
-        vars:
-          mongo_version: "3.6"
-      - func: "start mongod"
-      - func: "wait for mongod to be ready"
-      - func: "run make target"
-        vars:
-          target: build
-      - func: "run make target"
-        vars:
-          target: test:integration -ssl=true ${testArgs}
-
-  - name: integration-3.6-mmapv1
-    tags: ["3.6", "mmap"]
-    commands:
-      - func: "fetch source"
-      - command: expansions.update
-      - func: "download mongod and shell"
-        vars:
-          mongo_version: "3.6"
-      - func: "start mongod"
-        vars:
-          STORAGE_ENGINE: "mmapv1"
-      - func: "wait for mongod to be ready"
-      - func: "run make target"
-        vars:
-          target: build
-      - func: "run make target"
-        vars:
-          target: test:integration -ssl=true ${testArgs}
-
-  - name: integration-4.0
-    tags: ["4.0"]
-    commands:
-      - func: "fetch source"
-      - command: expansions.update
-      - func: "download mongod and shell"
-        vars:
-          mongo_version: "4.0"
-      - func: "start mongod"
-      - func: "wait for mongod to be ready"
-      - func: "run make target"
-        vars:
-          target: build
-      - func: "run make target"
-        vars:
-          target: test:integration -ssl=true ${testArgs}
-
-  - name: integration-4.0-auth
-    tags: ["4.0", "auth"]
-    commands:
-      - func: "fetch source"
-      # Concat auth args
-      - command: expansions.update
-        params:
-          updates:
-            - key: "mongod_args"
-              concat: " --auth"
-      - func: "download mongod and shell"
-        vars:
-          mongo_version: "4.0"
-      - func: "start mongod"
-      - func: "wait for mongod to be ready"
-      - func: "create mongod users"
-      - func: "run make target"
-        vars:
-          target: build
-      - func: "run make target"
-        vars:
-          target: test:integration -ssl=true -auth=true ${testArgs}
-
-  - name: integration-4.0-cluster
-    tags: ["4.0", "cluster"]
-    commands:
-      - func: "fetch source"
-      - command: expansions.update
-      - func: "download mongod and shell"
-        vars:
-          mongo_version: "4.0"
-      - func: "create repl_set"
-        vars:
-          USE_SSL: "true"
-      - func: "run make target"
-        vars:
-          target: build
-      - func: "run make target"
-        vars:
-          target: test:integration -ssl=true ${testArgs}
-
-  - name: integration-4.0-mmapv1
-    tags: ["4.0", "mmap"]
-    commands:
-      - func: "fetch source"
-      - command: expansions.update
-      - func: "download mongod and shell"
-        vars:
-          mongo_version: "4.0"
-      - func: "start mongod"
-        vars:
-          STORAGE_ENGINE: "mmapv1"
-      - func: "wait for mongod to be ready"
-      - func: "run make target"
-        vars:
-          target: build
-      - func: "run make target"
-        vars:
-          target: test:integration -ssl=true ${testArgs}
-
   - name: integration-4.2
     tags: ["4.2"]
     commands:
@@ -1749,40 +1636,6 @@ tasks:
           resmoke_suite: "core${resmoke_use_tls}"
           excludes: "requires_unstable,${excludes}"
 
-  - name: qa-tests-4.0
-    tags: ["4.0"]
-    commands:
-      - func: "fetch source"
-      - func: "get buildnumber"
-      - func: "setup credentials"
-      - func: "download mongod and shell"
-        vars:
-          mongo_version: "4.0"
-      - func: "run make target"
-        vars:
-          target: build
-      - func: "run qa-tests"
-        vars:
-          resmoke_suite: "core${resmoke_use_ssl}"
-          excludes: "requires_min_mongo_42,requires_unstable,${excludes}"
-
-  - name: qa-tests-3.6
-    tags: ["3.6"]
-    commands:
-      - func: "fetch source"
-      - func: "get buildnumber"
-      - func: "setup credentials"
-      - func: "download mongod and shell"
-        vars:
-          mongo_version: "3.6"
-      - func: "run make target"
-        vars:
-          target: build
-      - func: "run qa-tests"
-        vars:
-          resmoke_suite: "core${resmoke_use_ssl}"
-          excludes: "requires_min_mongo_40,requires_min_mongo_42,requires_unstable,${excludes}"
-
   - name: native-cert-ssl-4.4
     tags: ["4.4"]
     commands:
@@ -2047,16 +1900,12 @@ buildvariants:
         ]
       mongo_os: "rhel80"
       mongo_edition: "enterprise"
-      smoke_use_ssl: --use-ssl
-      resmoke_use_ssl: _ssl
-      smoke_use_tls: --use-tls
       resmoke_use_tls: _tls
       edition: enterprise
       run_kinit: true
       resmoke_args: --jobs 4
     tasks:
       - name: "unit"
-      - name: ".4.0"
       - name: ".4.2"
       - name: ".4.4"
       - name: ".5.0"
@@ -2086,16 +1935,12 @@ buildvariants:
       mongo_os: "rhel80"
       mongo_edition: "enterprise"
       testArgs: "-race=true"
-      smoke_use_ssl: --use-ssl
-      resmoke_use_ssl: _ssl
-      smoke_use_tls: --use-tls
       resmoke_use_tls: _tls
       edition: enterprise
       run_kinit: true
       resmoke_args: --jobs 4
     tasks:
       - name: "unit"
-      - name: ".4.0"
       - name: ".4.2"
       - name: ".4.4"
       - name: ".5.0"
@@ -2233,10 +2078,7 @@ buildvariants:
       mongo_os: "windows-64"
       mongo_edition: "enterprise"
       mongo_target: "windows"
-      smoke_use_ssl: --use-ssl
       resmoke_args: --jobs 4
-      resmoke_use_ssl: _ssl
-      smoke_use_tls: --use-tls
       resmoke_use_tls: _tls
       excludes: requires_large_ram,requires_mongo_24
       edition: enterprise
@@ -2245,7 +2087,6 @@ buildvariants:
       USE_SSL: "true"
     tasks:
       - name: "unit"
-      - name: ".4.0"
       - name: ".4.2"
       - name: ".4.4"
       - name: ".5.0"

Diff for: common/db/db.go

@@ -623,23 +623,10 @@ func CanIgnoreError(err error) bool {
 
 // Returns a boolean based on whether the given error indicates that this timeseries collection needs to be updated to set `timeseriesBucketsMayHaveMixedSchemaData` to `true`.
 func TimeseriesBucketNeedsMixedSchema(err error) bool {
-	if err == nil {
-		return false
-	}
-
-	switch mongoErr := err.(type) {
-	case mongo.WriteError:
-		return mongoErr.Code == ErrCannotInsertTimeseriesBucketsWithMixedSchema
+	var mongoErr mongo.ServerError
 
-	case mongo.BulkWriteException:
-		for _, writeErr := range mongoErr.WriteErrors {
-			if writeErr.Code == ErrCannotInsertTimeseriesBucketsWithMixedSchema {
-				return true
-			}
-		}
-		return false
-	}
-	return false
+	return errors.As(err, &mongoErr) &&
+		mongoErr.HasErrorCode(ErrCannotInsertTimeseriesBucketsWithMixedSchema)
 }
 
 // IsMMAPV1 returns whether the storage engine is MMAPV1. Also returns false

Diff for: mongodump/mongodump_test.go

@@ -379,7 +379,19 @@ func setupTimeseriesWithMixedSchema(dbName string, collName string) error {
 
 	// SERVER-84531 was only backported to 7.3.
 	// TODO: Run collMod command on 6.0 and 7.0 (TOOLS-3597).
-	if cmp, err := testutil.CompareFCV(testutil.GetFCV(client), "7.3"); err != nil || cmp >= 0 {
+	clientFCV := testutil.GetFCV(client)
+
+	shouldAccommodateMixedSchema := clientFCV == "6.0" || clientFCV == "7.0"
+	if !shouldAccommodateMixedSchema {
+		cmp, err := testutil.CompareFCV(clientFCV, "7.3")
+		if err != nil {
+			return errors.Wrapf(err, "failed to compare client FCV (%s)", clientFCV)
+		}
+
+		shouldAccommodateMixedSchema = cmp >= 0
+	}
+
+	if shouldAccommodateMixedSchema {
 		if res := sessionProvider.DB(dbName).RunCommand(context.Background(), bson.D{
 			{"collMod", collName},
 			{"timeseriesBucketsMayHaveMixedSchemaData", true},

Diff for: release/download/download_server.go

@@ -2,10 +2,15 @@ package download
 
 import (
 	"fmt"
+	"strings"
 
+	"github.com/mongodb/mongo-tools/release/platform"
 	"github.com/mongodb/mongo-tools/release/version"
 )
 
+// Increase this as needed for new server versions.
+var maxServerVersion = version.Version{8, 0, 999, ""}
+
 // JSONFeed represents the structure of the JSON
 // document consumed by the MongoDB downloads center.
 type ServerJSONFeed struct {
@@ -37,11 +42,10 @@ var (
 
 func (f *ServerJSONFeed) FindURLHashAndVersion(
 	serverVersion string,
-	target string,
-	arch string,
+	platform platform.Platform,
 	edition string,
 ) (string, string, string, error) {
-	fmt.Printf("Finding %v, %v, %v, %v\n", serverVersion, target, arch, edition)
+	fmt.Printf("Finding %v, %v, %v\n", serverVersion, platform, edition)
 
 	var sv version.Version
 	var err error
@@ -58,19 +62,41 @@ func (f *ServerJSONFeed) FindURLHashAndVersion(
 	// This is useful to find a server release that is not in the feed.
 	versionGuess := ""
 	for _, v := range f.Versions {
+		if serverVersion != "latest" && strings.Contains(v.Version, "-rc") {
+			fmt.Printf("Skipping release candidate: %v\n", v.Version)
+			continue
+		}
+
 		feedVersion, err := version.Parse(v.Version)
 		if err != nil {
 			return "", "", "", fmt.Errorf("Unable to parse feed version: %v", err)
 		}
-		fmt.Printf("feedVersion: %+v\n", feedVersion)
+		fmt.Printf("feedVersion: %+v (%s)\n", feedVersion, v.Version)
 
 		if serverVersion == "latest" ||
 			(feedVersion.Major == sv.Major && feedVersion.Minor == sv.Minor) {
+
+			if feedVersion.GreaterThan(maxServerVersion) {
+				if serverVersion == "latest" {
+					fmt.Printf(
+						"Skipping because it exceeds the max server version (%v)\n",
+						maxServerVersion,
+					)
+					continue
+				} else {
+					return "", "", "", fmt.Errorf("cannot match a server version (%v) that exceeds the max server version (%v)", feedVersion, maxServerVersion)
+				}
+			}
+
 			if versionGuess == "" {
 				versionGuess = feedVersion.String()
 			}
 			for _, dl := range v.Downloads {
-				if dl.Target == target && dl.Arch == arch && dl.Edition == edition {
+				if !platform.TargetMatches(dl.Target) {
+					continue
+				}
+
+				if dl.Arch == platform.Arch.String() && dl.Edition == edition {
 					return dl.Archive.URL, v.GitHash, v.Version, nil
 				}
 			}