[release/8.0]: Light up support for OpenSSL ENGINEs only if they are available.

Some Linux distributions are phasing out support for OpenSSL 1.x ENGINEs. They are doing this by moving the header, `engine.h`, to a separate package or removing the header entirely. The actual OpenSSL shared libraries still contain the engine APIs. This makes the change an API, not ABI, break.

We react to this by disabling OpenSSL engine support on non-portable builds that are missing the engine header. Portable builds will continue to probe the loaded OpenSSL library for support, and non-portable builds will only support ENGINEs if the header is present.

Author: vcsjones

src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.EvpPkey.cs

@@ -217,7 +217,8 @@ internal static ArraySegment<byte> RentEncodeSubjectPublicKeyInfo(SafeEvpPKeyHan
         [LibraryImport(Libraries.CryptoNative, StringMarshalling = StringMarshalling.Utf8)]
         private static partial SafeEvpPKeyHandle CryptoNative_LoadPrivateKeyFromEngine(
             string engineName,
-            string keyName);
+            string keyName,
+            [MarshalAs(UnmanagedType.Bool)] out bool haveEngine);
 
         internal static SafeEvpPKeyHandle LoadPrivateKeyFromEngine(
             string engineName,
@@ -226,7 +227,13 @@ internal static SafeEvpPKeyHandle LoadPrivateKeyFromEngine(
             Debug.Assert(engineName is not null);
             Debug.Assert(keyName is not null);
 
-            SafeEvpPKeyHandle pkey = CryptoNative_LoadPrivateKeyFromEngine(engineName, keyName);
+            SafeEvpPKeyHandle pkey = CryptoNative_LoadPrivateKeyFromEngine(engineName, keyName, out bool haveEngine);
+
+            if (!haveEngine)
+            {
+                pkey.Dispose();
+                throw new CryptographicException(SR.Cryptography_EnginesNotSupported);
+            }
 
             if (pkey.IsInvalid)
             {
@@ -240,7 +247,8 @@ internal static SafeEvpPKeyHandle LoadPrivateKeyFromEngine(
         [LibraryImport(Libraries.CryptoNative, StringMarshalling = StringMarshalling.Utf8)]
         private static partial SafeEvpPKeyHandle CryptoNative_LoadPublicKeyFromEngine(
             string engineName,
-            string keyName);
+            string keyName,
+            [MarshalAs(UnmanagedType.Bool)] out bool haveEngine);
 
         internal static SafeEvpPKeyHandle LoadPublicKeyFromEngine(
             string engineName,
@@ -249,7 +257,13 @@ internal static SafeEvpPKeyHandle LoadPublicKeyFromEngine(
             Debug.Assert(engineName is not null);
             Debug.Assert(keyName is not null);
 
-            SafeEvpPKeyHandle pkey = CryptoNative_LoadPublicKeyFromEngine(engineName, keyName);
+            SafeEvpPKeyHandle pkey = CryptoNative_LoadPublicKeyFromEngine(engineName, keyName, out bool haveEngine);
+
+            if (!haveEngine)
+            {
+                pkey.Dispose();
+                throw new CryptographicException(SR.Cryptography_EnginesNotSupported);
+            }
 
             if (pkey.IsInvalid)
             {

src/libraries/System.Security.Cryptography/src/Resources/Strings.resx

@@ -327,6 +327,9 @@
   <data name="Cryptography_EncryptedIncorrectLength" xml:space="preserve">
     <value>{0} unexpectedly produced a ciphertext with the incorrect length.</value>
   </data>
+  <data name="Cryptography_EnginesNotSupported" xml:space="preserve">
+    <value>OpenSSL ENGINE is not available on this platform.</value>
+  </data>
   <data name="Cryptography_ExceedKdfExtractLimit" xml:space="preserve">
     <value>The total number of bytes extracted cannot exceed UInt32.MaxValue * hash length.</value>
   </data>

src/native/libs/System.Security.Cryptography.Native/configure.cmake

@@ -1,8 +1,10 @@
 include(CheckLibraryExists)
 include(CheckFunctionExists)
+include(CheckSourceCompiles)
 
 set(CMAKE_REQUIRED_INCLUDES ${OPENSSL_INCLUDE_DIR})
 set(CMAKE_REQUIRED_LIBRARIES ${OPENSSL_CRYPTO_LIBRARY} ${OPENSSL_SSL_LIBRARY})
+set(CMAKE_REQUIRED_DEFINITIONS -DOPENSSL_API_COMPAT=0x10100000L)
 
 check_function_exists(
     EC_GF2m_simple_method
@@ -22,6 +24,11 @@ check_function_exists(
     HAVE_OPENSSL_SHA3
 )
 
+check_source_compiles(C "
+#include <openssl/engine.h>
+int main(void) { ENGINE_init(NULL); return 1; }"
+HAVE_OPENSSL_ENGINE)
+
 configure_file(
     ${CMAKE_CURRENT_SOURCE_DIR}/pal_crypto_config.h.in
     ${CMAKE_CURRENT_BINARY_DIR}/pal_crypto_config.h)

src/native/libs/System.Security.Cryptography.Native/opensslshim.h

@@ -17,7 +17,6 @@
 #include <openssl/dsa.h>
 #include <openssl/ec.h>
 #include <openssl/ecdsa.h>
-#include <openssl/engine.h>
 #include <openssl/err.h>
 #include <openssl/evp.h>
 #include <openssl/hmac.h>
@@ -46,6 +45,11 @@
 #include <openssl/provider.h>
 #endif
 
+#if HAVE_OPENSSL_ENGINE
+// Some Linux distributions build without engine support.
+#include <openssl/engine.h>
+#endif
+
 #if OPENSSL_VERSION_NUMBER >= OPENSSL_VERSION_1_1_1_RTM
 #define HAVE_OPENSSL_SET_CIPHERSUITES 1
 #else
@@ -168,6 +172,24 @@ const EVP_MD *EVP_shake256(void);
 int EVP_DigestFinalXOF(EVP_MD_CTX *ctx, unsigned char *md, size_t len);
 #endif
 
+#if !HAVE_OPENSSL_ENGINE
+#undef HAVE_OPENSSL_ENGINE
+#define HAVE_OPENSSL_ENGINE 1
+
+ENGINE *ENGINE_by_id(const char *id);
+int ENGINE_init(ENGINE *e);
+int ENGINE_finish(ENGINE *e);
+ENGINE *ENGINE_new(void);
+int ENGINE_free(ENGINE *e);
+typedef EVP_PKEY *(*ENGINE_LOAD_KEY_PTR)(ENGINE *, const char *,
+                                         UI_METHOD *ui_method,
+                                         void *callback_data);
+EVP_PKEY *ENGINE_load_private_key(ENGINE *e, const char *key_id,
+                                  UI_METHOD *ui_method, void *callback_data);
+EVP_PKEY *ENGINE_load_public_key(ENGINE *e, const char *key_id,
+                                 UI_METHOD *ui_method, void *callback_data);
+#endif
+
 #define API_EXISTS(fn) (fn != NULL)
 
 // List of all functions from the libssl that are used in the System.Security.Cryptography.Native.
@@ -298,12 +320,12 @@ int EVP_DigestFinalXOF(EVP_MD_CTX *ctx, unsigned char *md, size_t len);
     REQUIRED_FUNCTION(EC_POINT_mul) \
     REQUIRED_FUNCTION(EC_POINT_new) \
     REQUIRED_FUNCTION(EC_POINT_set_affine_coordinates_GFp) \
-    REQUIRED_FUNCTION(ENGINE_by_id) \
-    REQUIRED_FUNCTION(ENGINE_finish) \
-    REQUIRED_FUNCTION(ENGINE_free) \
-    REQUIRED_FUNCTION(ENGINE_init) \
-    REQUIRED_FUNCTION(ENGINE_load_public_key) \
-    REQUIRED_FUNCTION(ENGINE_load_private_key) \
+    LIGHTUP_FUNCTION(ENGINE_by_id) \
+    LIGHTUP_FUNCTION(ENGINE_finish) \
+    LIGHTUP_FUNCTION(ENGINE_free) \
+    LIGHTUP_FUNCTION(ENGINE_init) \
+    LIGHTUP_FUNCTION(ENGINE_load_public_key) \
+    LIGHTUP_FUNCTION(ENGINE_load_private_key) \
     REQUIRED_FUNCTION(ERR_clear_error) \
     REQUIRED_FUNCTION(ERR_error_string_n) \
     REQUIRED_FUNCTION(ERR_get_error) \

src/native/libs/System.Security.Cryptography.Native/pal_crypto_config.h.in

@@ -4,3 +4,4 @@
 #cmakedefine01 HAVE_OPENSSL_ALPN
 #cmakedefine01 HAVE_OPENSSL_CHACHA20POLY1305
 #cmakedefine01 HAVE_OPENSSL_SHA3
+#cmakedefine01 HAVE_OPENSSL_ENGINE

src/native/libs/System.Security.Cryptography.Native/pal_evp_pkey.c

@@ -511,41 +511,72 @@ int32_t CryptoNative_EncodeSubjectPublicKeyInfo(EVP_PKEY* pkey, uint8_t* buf)
     return i2d_PUBKEY(pkey, &buf);
 }
 
+#if HAVE_OPENSSL_ENGINE
 static EVP_PKEY* LoadKeyFromEngine(
     const char* engineName,
     const char* keyName,
-    ENGINE_LOAD_KEY_PTR load_func)
+    ENGINE_LOAD_KEY_PTR load_func,
+    int32_t* haveEngine)
 {
+    assert(haveEngine);
     ERR_clear_error();
 
-    EVP_PKEY* ret = NULL;
-    ENGINE* engine = NULL;
+    if (API_EXISTS(ENGINE_by_id) && API_EXISTS(ENGINE_init) && API_EXISTS(ENGINE_finish) && API_EXISTS(ENGINE_free))
+    {
+        *haveEngine = 1;
+        EVP_PKEY* ret = NULL;
+        ENGINE* engine = NULL;
 
-    // Per https://github.com/openssl/openssl/discussions/21427
-    // using EVP_PKEY after freeing ENGINE is correct.
-    engine = ENGINE_by_id(engineName);
+        // Per https://github.com/openssl/openssl/discussions/21427
+        // using EVP_PKEY after freeing ENGINE is correct.
+        engine = ENGINE_by_id(engineName);
 
-    if (engine != NULL)
-    {
-        if (ENGINE_init(engine))
+        if (engine != NULL)
         {
-            ret = load_func(engine, keyName, NULL, NULL);
+            if (ENGINE_init(engine))
+            {
+                ret = load_func(engine, keyName, NULL, NULL);
+
+                ENGINE_finish(engine);
+            }
 
-            ENGINE_finish(engine);
+            ENGINE_free(engine);
         }
 
-        ENGINE_free(engine);
+        return ret;
     }
 
-    return ret;
+    *haveEngine = 0;
+    return NULL;
 }
+#endif
 
-EVP_PKEY* CryptoNative_LoadPrivateKeyFromEngine(const char* engineName, const char* keyName)
+EVP_PKEY* CryptoNative_LoadPrivateKeyFromEngine(const char* engineName, const char* keyName, int32_t* haveEngine)
 {
-    return LoadKeyFromEngine(engineName, keyName, ENGINE_load_private_key);
+#if HAVE_OPENSSL_ENGINE
+    if (API_EXISTS(ENGINE_load_private_key))
+    {
+        return LoadKeyFromEngine(engineName, keyName, ENGINE_load_private_key, haveEngine);
+    }
+#endif
+    (void)engineName;
+    (void)keyName;
+    (void)haveEngine;
+    *haveEngine = 0;
+    return NULL;
 }
 
-EVP_PKEY* CryptoNative_LoadPublicKeyFromEngine(const char* engineName, const char* keyName)
+EVP_PKEY* CryptoNative_LoadPublicKeyFromEngine(const char* engineName, const char* keyName, int32_t* haveEngine)
 {
-    return LoadKeyFromEngine(engineName, keyName, ENGINE_load_public_key);
+#if HAVE_OPENSSL_ENGINE
+    if (API_EXISTS(ENGINE_load_private_key))
+    {
+        return LoadKeyFromEngine(engineName, keyName, ENGINE_load_public_key, haveEngine);
+    }
+#endif
+    (void)engineName;
+    (void)keyName;
+    (void)haveEngine;
+    *haveEngine = 0;
+    return NULL;
 }

src/native/libs/System.Security.Cryptography.Native/pal_evp_pkey.h

@@ -93,12 +93,14 @@ PALEXPORT int32_t CryptoNative_EncodeSubjectPublicKeyInfo(EVP_PKEY* pkey, uint8_
 Load a named key, via ENGINE_load_private_key, from the named engine.
 
 Returns a valid EVP_PKEY* on success, NULL on failure.
+haveEngine is 1 if OpenSSL ENGINE's are supported, otherwise 0.
 */
-PALEXPORT EVP_PKEY* CryptoNative_LoadPrivateKeyFromEngine(const char* engineName, const char* keyName);
+PALEXPORT EVP_PKEY* CryptoNative_LoadPrivateKeyFromEngine(const char* engineName, const char* keyName, int32_t* haveEngine);
 
 /*
 Load a named key, via ENGINE_load_public_key, from the named engine.
 
 Returns a valid EVP_PKEY* on success, NULL on failure.
+haveEngine is 1 if OpenSSL ENGINE's are supported, otherwise 0.
 */
-PALEXPORT EVP_PKEY* CryptoNative_LoadPublicKeyFromEngine(const char* engineName, const char* keyName);
+PALEXPORT EVP_PKEY* CryptoNative_LoadPublicKeyFromEngine(const char* engineName, const char* keyName, int32_t* haveEngine);