[CDRIVER-4400] Rename references of "csfle" to "crypt_shared" (#1022)

* Rename references of "csfle" to "crypt_shared"
* use crypt_shared 6.0.0-rc8
* test with libmongocrypt master
* CDRIVER-4397 update fle2 find payloads
* update download-mongodb.sh to use 6.0.0-rc8
* add TODO comments to pin to 1.5.0-rc0
* fix check for crypt_shared availability
* remove unused compile definition

Co-authored-by: Kevin Albertson <[email protected]>
Co-authored-by: Ezra Chung <[email protected]>

Author: 3 people

.evergreen/compile-unix.sh

@@ -203,9 +203,9 @@ fi
 
 CONFIGURE_FLAGS="$CONFIGURE_FLAGS -DMONGO_SANITIZE=$SANITIZE"
 
-if ! python3 build/mongodl.py --test -C csfle -V 6.0.0-rc4 -o . > /dev/null; then
-   echo "No csfle detected for this platform. Disabling MONGOC_TEST_USE_CSFLE."
-   CONFIGURE_FLAGS="$CONFIGURE_FLAGS -DMONGOC_TEST_USE_CSFLE=OFF"
+if ! python3 build/mongodl.py --test -C crypt_shared -V 6.0.0-rc8 -o . > /dev/null; then
+   echo "No crypt_shared detected for this platform. Disabling MONGOC_TEST_USE_CRYPT_SHARED."
+   CONFIGURE_FLAGS="$CONFIGURE_FLAGS -DMONGOC_TEST_USE_CRYPT_SHARED=OFF"
 fi
 
 CONFIGURE_FLAGS="$CONFIGURE_FLAGS $EXTRA_CONFIGURE_FLAGS"
@@ -227,7 +227,8 @@ pkg-config --modversion libssl || true
 
 if [ "$COMPILE_LIBMONGOCRYPT" = "ON" ]; then
    # Build libmongocrypt, using the previously fetched installed source.
-   git clone https://github.com/mongodb/libmongocrypt --branch 1.5.0-alpha2
+   # TODO (CDRIVER-4397): add "--branch 1.5.0-rc0" in git clone once libmongocrypt 1.5.0-rc0 is released.
+   git clone https://github.com/mongodb/libmongocrypt
    mkdir libmongocrypt/cmake-build
    cd libmongocrypt/cmake-build
    $CMAKE -DENABLE_SHARED_BSON=ON -DCMAKE_BUILD_TYPE="Debug" -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DCMAKE_PREFIX_PATH="$CMAKE_PREFIX_PATH" ../

.evergreen/compile-windows.sh

@@ -115,7 +115,8 @@ fi
 
 if [ "$COMPILE_LIBMONGOCRYPT" = "ON" ]; then
    # Build libmongocrypt, using the previously fetched installed source.
-   git clone https://github.com/mongodb/libmongocrypt --branch 1.5.0-alpha2
+   # TODO (CDRIVER-4397): add "--branch 1.5.0-rc0" in git clone once libmongocrypt 1.5.0-rc0 is released.
+   git clone https://github.com/mongodb/libmongocrypt
    mkdir libmongocrypt/cmake-build
    cd libmongocrypt/cmake-build
    "$CMAKE" -G "$CC" "-DCMAKE_PREFIX_PATH=${INSTALL_DIR}/lib/cmake" -DENABLE_SHARED_BSON=ON -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" ../

.evergreen/download-mongodb.sh

@@ -54,7 +54,7 @@ get_mongodb_download_url_for ()
    # Set VERSION_RAPID to the latest rapid release each quarter.
    VERSION_RAPID="5.3.1"
    VERSION_60_LATEST="v6.0-latest"
-   VERSION_60="6.0.0-rc5"
+   VERSION_60="6.0.0-rc8"
    VERSION_50="5.0.8"
    VERSION_44="4.4.13"
    VERSION_42="4.2.19"

.evergreen/run-tests.sh

@@ -111,9 +111,9 @@ if [ "$CLIENT_SIDE_ENCRYPTION" = "on" ]; then
    wait_for_kms_server 5698
    echo "Waiting for mock KMS servers to start... done."
    if ! test -d /cygdrive/c; then
-      # We have trouble with this test on Windows. only set csflePath on other platforms
-      export MONGOC_TEST_CSFLE_PATH="$(find . -wholename '*src/libmongoc/mongo_csfle_v1.*' -and -regex '.*\(.dll\|.dylib\|.so\)' | head -n1)"
-      echo "Setting env csflePath: [$MONGOC_TEST_CSFLE_PATH]"
+      # We have trouble with this test on Windows. only set cryptSharedLibPath on other platforms
+      export MONGOC_TEST_CRYPT_SHARED_LIB_PATH="$(find . -wholename '*src/libmongoc/mongo_crypt_v1.*' -and -regex '.*\(.dll\|.dylib\|.so\)' | head -n1)"
+      echo "Setting env cryptSharedLibPath: [$MONGOC_TEST_CRYPT_SHARED_LIB_PATH]"
    fi
 fi
 

.evergreen/skip-tests.txt

@@ -86,4 +86,4 @@
 /sessions/unified/snapshot-sessions/"countDocuments operation with snapshot" # (CDRIVER-4355) error: checking expectResult:  { "$numberInt" : "2" }
 /load_balancers/non-lb-connection-establishment/"operations against non-load balanced clusters fail if URI contains loadBalanced=true" # (CDRIVER-4356) error: expected error to contain "Driver attempted to initialize in load balancing mode, but the server does not support this mode", but got: "BSON field 'hello.loadBalanced' is an unknown field."
 
-/client_side_encryption/bypass_spawning_mongocryptd/mongocryptdBypassSpawn  # Fails if csfle is visible
+/client_side_encryption/bypass_spawning_mongocryptd/mongocryptdBypassSpawn  # Fails if crypt_shared is visible

src/libmongoc/CMakeLists.txt

@@ -857,36 +857,36 @@ if (ENABLE_SHM_COUNTERS STREQUAL "ON")
 endif ()
 
 cmake_dependent_option (
-   MONGOC_TEST_USE_CSFLE "If TRUE, download and use a csfle dynamic library for client-side-encryption testing. Otherwise will rely on mongocryptd"
+   MONGOC_TEST_USE_CRYPT_SHARED "If TRUE, download and use a crypt_shared library for client-side-encryption testing. Otherwise will rely on mongocryptd"
    TRUE ENABLE_TESTS FALSE
 )
 
-if (MONGOC_TEST_USE_CSFLE)
+if (MONGOC_TEST_USE_CRYPT_SHARED)
    find_program (_PYTHON3_EXE NAMES python3 py python)
    mark_as_advanced (_PYTHON3_EXE)
    if (NOT _PYTHON3_EXE)
       message (SEND_ERROR
-         "MONGOC_TEST_USE_CSFLE is enabled, but we were unable to find a Python 3 interpreter to use mongodl.py. "
-         "Install a Python 3 or set MONGOC_TEST_USE_CSFLE to FALSE"
+         "MONGOC_TEST_USE_CRYPT_SHARED is enabled, but we were unable to find a Python 3 interpreter to use mongodl.py. "
+         "Install a Python 3 or set MONGOC_TEST_USE_CRYPT_SHARED to FALSE"
       )
    endif ()
 
-   get_filename_component (csfle_dest "${CMAKE_CURRENT_BINARY_DIR}/mongo_csfle_v1${CMAKE_SHARED_LIBRARY_SUFFIX}" ABSOLUTE)
-   # If updating the "--version" for csfle, update the version in compile-unix.sh to match.
+   get_filename_component (crypt_shared_dest "${CMAKE_CURRENT_BINARY_DIR}/mongo_crypt_v1${CMAKE_SHARED_LIBRARY_SUFFIX}" ABSOLUTE)
+   # If updating the "--version" for crypt_shared, update the version in compile-unix.sh to match.
    add_custom_command (
-      OUTPUT "${csfle_dest}"
+      OUTPUT "${crypt_shared_dest}"
       COMMAND
          "${_PYTHON3_EXE}" -u "${mongo-c-driver_SOURCE_DIR}/build/mongodl.py"
-            --component csfle
-            --version 6.0.0-rc5
+            --component crypt_shared
+            --version 6.0.0-rc8
             --edition enterprise
             --out "${CMAKE_CURRENT_BINARY_DIR}"
-            --only "**/mongo_csfle_v1.*"
+            --only "**/mongo_crypt_v1.*"
             --strip-path-components 1
-      COMMENT "Downloading csfle"
+      COMMENT "Downloading crypt_shared"
       VERBATIM
    )
-   add_custom_target (get-csfle ALL DEPENDS "${csfle_dest}")
+   add_custom_target (get-crypt_shared ALL DEPENDS "${crypt_shared_dest}")
 endif ()
 
 function (mongoc_add_test test use_shared)
@@ -1091,10 +1091,6 @@ mongoc_add_test (test-libmongoc FALSE ${test-libmongoc-sources})
 mongoc_add_test (test-mongoc-gssapi FALSE ${PROJECT_SOURCE_DIR}/tests/test-mongoc-gssapi.c)
 mongoc_add_test (test-mongoc-cache FALSE ${PROJECT_SOURCE_DIR}/tests/test-mongoc-cache.c)
 
-if (TARGET get-csfle AND TARGET test-libmongoc)
-   target_compile_definitions (test-libmongoc PRIVATE "TESTING_CSFLE_OVERRIDE_PATH=\"./mongo_csfle_v1${CMAKE_SHARED_LIBRARY_SUFFIX}\"")
-endif ()
-
 if (ENABLE_TESTS)
    # "make test" doesn't compile tests, so we create "make check" which compiles
    # and runs tests: https://gitlab.kitware.com/cmake/cmake/issues/8774

src/libmongoc/doc/mongoc_auto_encryption_opts_set_bypass_query_analysis.rst

@@ -21,8 +21,9 @@ Parameters
 
 
 ``bypass_query_analysis`` disables automatic analysis of outgoing commands.
-``bypass_query_analysis`` is useful for encrypting indexed fields without the ``csfle`` shared library or ``mongocryptd`` process.
-Set ``bypass_query_analysis`` to true to use explicit encryption on indexed fields.
+``bypass_query_analysis`` is useful for encrypting indexed fields without the
+``crypt_shared`` library or ``mongocryptd`` process. Set
+``bypass_query_analysis`` to true to use explicit encryption on indexed fields.
 
 .. seealso::
 

src/libmongoc/doc/mongoc_auto_encryption_opts_set_extra.rst

@@ -25,13 +25,14 @@ Parameters
 * ``mongocryptdBypassSpawn`` set to true to prevent the driver from spawning the mongocryptd process (default behavior is to spawn).
 * ``mongocryptdSpawnPath`` set to a path (with trailing slash) to search for mongocryptd (defaults to empty string and uses default system paths).
 * ``mongocryptdSpawnArgs`` set to an array of string arguments to pass to ``mongocryptd`` when spawning (defaults to ``[ "--idleShutdownTimeoutSecs=60" ]``).
-* ``csflePath`` - Set a filepath string referring to a ``csfle`` dynamic library
-  file. Unset by default.
-
-  * If not set (the default), ``libmongocrypt`` will attempt to load ``csfle``
-    using the host system's default dynamic-library-search system.
-  * If set, the given path should identify the ``csfle`` dynamic library file
-    itself, not the directory that contains it.
+* ``cryptSharedLibPath`` - Set a filepath string referring to a ``crypt_shared``
+  library file. Unset by default.
+
+  * If not set (the default), ``libmongocrypt`` will attempt to load
+    ``crypt_shared`` using the host system's default dynamic-library-search
+    system.
+  * If set, the given path should identify the ``crypt_shared`` dynamic library
+    file itself, not the directory that contains it.
   * If the given path is a relative path and the first path component is
     ``$ORIGIN``, the ``$ORIGIN`` component will be replaced with the absolute
     path to the directory containing the ``libmongocrypt`` library in use by the
@@ -41,17 +42,17 @@ Parameters
 
   * If the given path is a relative path, the path will be resolved relative to
     the working directory of the operating system process.
-  * If this option is set and ``libmongocrypt`` fails to load ``csfle`` from the
+  * If this option is set and ``libmongocrypt`` fails to load ``crypt_shared`` from the
     given filepath, ``libmongocrypt`` will fail to initialize and will not
-    attempt to search for ``csfle`` in any other locations.
+    attempt to search for ``crypt_shared`` in any other locations.
 
-* ``csfleRequired`` - If set to ``true``, and ``libmongocrypt`` fails to load a
-  ``csfle`` dynamic library, initialization of auto-encryption will fail
-  immediately and will not attempt to spawn ``mongocryptd``.
+* ``cryptSharedLibRequired`` - If set to ``true``, and ``libmongocrypt`` fails
+  to load a ``crypt_shared`` library, initialization of auto-encryption will
+  fail immediately and will not attempt to spawn ``mongocryptd``.
 
-  If set to ``false`` (the default), ``csflePath`` is not set, *and*
-  ``libmongocrypt`` fails to load ``csfle``, then ``libmongocrypt`` will proceed
-  without ``csfle`` and fall back to using ``mongocryptd``.
+  If set to ``false`` (the default), ``cryptSharedLibPath`` is not set, *and*
+  ``libmongocrypt`` fails to load ``crypt_shared``, then ``libmongocrypt`` will
+  proceed without ``crypt_shared`` and fall back to using ``mongocryptd``.
 
 For more information, see the `Client-Side Encryption specification <https://github.com/mongodb/specifications/blob/master/source/client-side-encryption/client-side-encryption.rst#extraoptions>`_.
 

src/libmongoc/src/mongoc/mongoc-client-side-encryption.c

@@ -1423,29 +1423,32 @@ _parse_extra (const bson_t *extra,
          }
       }
 
-      if (bson_iter_init_find (&iter, extra, "csflePath")) {
+      if (bson_iter_init_find (&iter, extra, "cryptSharedLibPath")) {
          if (!BSON_ITER_HOLDS_UTF8 (&iter)) {
             bson_set_error (error,
                             MONGOC_ERROR_CLIENT,
                             MONGOC_ERROR_CLIENT_INVALID_ENCRYPTION_ARG,
-                            "Expected a string for 'csflePath'");
+                            "Expected a string for 'cryptSharedLibPath'");
             GOTO (fail);
          }
          size_t len;
          const char *ptr = bson_iter_utf8_unsafe (&iter, &len);
-         bson_free (topology->csfle_override_path);
-         topology->csfle_override_path = bson_strdup (ptr);
+         bson_free (topology->clientSideEncryption.autoOptions.extraOptions
+                       .cryptSharedLibPath);
+         topology->clientSideEncryption.autoOptions.extraOptions
+            .cryptSharedLibPath = bson_strdup (ptr);
       }
 
-      if (bson_iter_init_find (&iter, extra, "csfleRequired")) {
+      if (bson_iter_init_find (&iter, extra, "cryptSharedLibRequired")) {
          if (!BSON_ITER_HOLDS_BOOL (&iter)) {
             bson_set_error (error,
                             MONGOC_ERROR_CLIENT,
                             MONGOC_ERROR_CLIENT_INVALID_ENCRYPTION_ARG,
-                            "Expected a bool for 'csfleRequired'");
+                            "Expected a bool for 'cryptSharedLibRequired'");
             GOTO (fail);
          }
-         topology->csfle_required = bson_iter_bool_unsafe (&iter);
+         topology->clientSideEncryption.autoOptions.extraOptions
+            .cryptSharedLibRequired = bson_iter_bool_unsafe (&iter);
       }
    }
 
@@ -1552,8 +1555,10 @@ _mongoc_cse_client_enable_auto_encryption (mongoc_client_t *client,
                          opts->schema_map,
                          opts->encrypted_fields_map,
                          opts->tls_opts,
-                         client->topology->csfle_override_path,
-                         client->topology->csfle_required,
+                         client->topology->clientSideEncryption.autoOptions
+                            .extraOptions.cryptSharedLibPath,
+                         client->topology->clientSideEncryption.autoOptions
+                            .extraOptions.cryptSharedLibRequired,
                          opts->bypass_auto_encryption,
                          opts->bypass_query_analysis,
                          error);
@@ -1706,15 +1711,18 @@ _mongoc_cse_client_pool_enable_auto_encryption (
       GOTO (fail);
    }
 
-   topology->crypt = _mongoc_crypt_new (opts->kms_providers,
-                                        opts->schema_map,
-                                        opts->encrypted_fields_map,
-                                        opts->tls_opts,
-                                        topology->csfle_override_path,
-                                        topology->csfle_required,
-                                        opts->bypass_auto_encryption,
-                                        opts->bypass_query_analysis,
-                                        error);
+   topology->crypt =
+      _mongoc_crypt_new (opts->kms_providers,
+                         opts->schema_map,
+                         opts->encrypted_fields_map,
+                         opts->tls_opts,
+                         topology->clientSideEncryption.autoOptions.extraOptions
+                            .cryptSharedLibPath,
+                         topology->clientSideEncryption.autoOptions.extraOptions
+                            .cryptSharedLibRequired,
+                         opts->bypass_auto_encryption,
+                         opts->bypass_query_analysis,
+                         error);
    if (!topology->crypt) {
       GOTO (fail);
    }
@@ -1816,8 +1824,8 @@ mongoc_client_encryption_new (mongoc_client_encryption_opts_t *opts,
                          NULL /* schema_map */,
                          NULL /* encrypted_fields_map */,
                          opts->tls_opts,
-                         NULL /* No csfle path */,
-                         false /* csfle not requried */,
+                         NULL /* No crypt_shared path */,
+                         false /* crypt_shared not requried */,
                          true, /* bypassAutoEncryption (We are explicit) */
                          false /* bypass_query_analysis. Not applicable. */,
                          error);

src/libmongoc/src/mongoc/mongoc-crypt-private.h

@@ -38,8 +38,8 @@ _mongoc_crypt_new (const bson_t *kms_providers,
                    const bson_t *schema_map,
                    const bson_t *encrypted_fields_map,
                    const bson_t *tls_opts,
-                   const char *csfle_override_path,
-                   bool csfle_required,
+                   const char *crypt_shared_lib_path,
+                   bool crypt_shared_lib_required,
                    bool bypass_auto_encryption,
                    bool bypass_query_analysis,
                    bson_error_t *error);