External file decryptor bindings

python/pyarrow/_parquet.pxd

@@ -681,6 +681,10 @@ cdef extern from "parquet/encryption/encryption.h" namespace "parquet" nogil:
             " parquet::FileDecryptionProperties":
         pass
 
+    cdef cppclass CExternalFileDecryptionProperties\
+            " parquet::ExternalFileDecryptionProperties":
+        pass
+
     cdef cppclass CFileEncryptionProperties\
             " parquet::FileEncryptionProperties":
         pass
@@ -704,3 +708,19 @@ cdef class FileDecryptionProperties:
 
     cdef inline shared_ptr[CFileDecryptionProperties] unwrap(self):
         return self.properties
+
+cdef class ExternalFileDecryptionProperties(FileDecryptionProperties):
+    """File-level decryption properties for the low-level API"""
+    cdef:
+        shared_ptr[CExternalFileDecryptionProperties] properties
+
+    @staticmethod
+    cdef inline ExternalFileDecryptionProperties wrap_external(
+            shared_ptr[CExternalFileDecryptionProperties] properties):
+
+        result = ExternalFileDecryptionProperties()
+        result.properties = properties
+        return result
+
+    cdef inline shared_ptr[CExternalFileDecryptionProperties] unwrap_external(self):
+        return <shared_ptr[CExternalFileDecryptionProperties]> self.properties

python/pyarrow/_parquet.pyx

@@ -1507,8 +1507,12 @@ cdef class ParquetReader(_Weakrefable):
                 thrift_container_size_limit)
 
         if decryption_properties is not None:
-            properties.file_decryption_properties(
-                decryption_properties.unwrap())
+            if isinstance(decryption_properties, ExternalFileDecryptionProperties):
+                properties.file_decryption_properties(
+                    static_pointer_cast[CFileDecryptionProperties, CExternalFileDecryptionProperties] (
+                        (<ExternalFileDecryptionProperties>decryption_properties).unwrap_external()))
+            else:
+                properties.file_decryption_properties((<FileDecryptionProperties>decryption_properties).unwrap())
 
         arrow_props.set_pre_buffer(pre_buffer)
 

python/pyarrow/_parquet_encryption.pxd

@@ -24,9 +24,11 @@ from pyarrow._parquet cimport (ParquetCipher,
                                CFileEncryptionProperties,
                                CExternalFileEncryptionProperties,
                                CFileDecryptionProperties,
+                               CExternalFileDecryptionProperties,
                                FileEncryptionProperties,
                                ExternalFileEncryptionProperties,
                                FileDecryptionProperties,
+                               ExternalFileDecryptionProperties,
                                ParquetCipher_AES_GCM_V1,
                                ParquetCipher_AES_GCM_CTR_V1,
                                ParquetCipher_EXTERNAL_DBPA_V1)
@@ -52,20 +54,16 @@ cdef class KmsConnectionConfig(_Weakrefable):
     @staticmethod
     cdef wrap(const CKmsConnectionConfig& config)
 
-
-cdef shared_ptr[CCryptoFactory] pyarrow_unwrap_cryptofactory(object crypto_factory) except *
-cdef shared_ptr[CKmsConnectionConfig] pyarrow_unwrap_kmsconnectionconfig(object kmsconnectionconfig) except *
-cdef shared_ptr[CEncryptionConfiguration] pyarrow_unwrap_encryptionconfig(object encryptionconfig) except *
-cdef shared_ptr[CDecryptionConfiguration] pyarrow_unwrap_decryptionconfig(object decryptionconfig) except *
-cdef shared_ptr[CExternalEncryptionConfiguration] pyarrow_unwrap_external_encryptionconfig(object encryptionconfig) except *
-
-
 cdef class ExternalEncryptionConfiguration(EncryptionConfiguration):
     cdef shared_ptr[CExternalEncryptionConfiguration] external_configuration
     cdef inline shared_ptr[CExternalEncryptionConfiguration] unwrap_external(self) nogil
-cdef shared_ptr[CExternalEncryptionConfiguration] pyarrow_unwrap_external_encryptionconfig(object externalencryptionconfig) except *
-
 cdef class ExternalDecryptionConfiguration(DecryptionConfiguration):
     cdef shared_ptr[CExternalDecryptionConfiguration] external_configuration
     cdef inline shared_ptr[CExternalDecryptionConfiguration] unwrap_external(self) nogil
+
+cdef shared_ptr[CCryptoFactory] pyarrow_unwrap_cryptofactory(object crypto_factory) except *
+cdef shared_ptr[CKmsConnectionConfig] pyarrow_unwrap_kmsconnectionconfig(object kmsconnectionconfig) except *
+cdef shared_ptr[CEncryptionConfiguration] pyarrow_unwrap_encryptionconfig(object encryptionconfig) except *
+cdef shared_ptr[CDecryptionConfiguration] pyarrow_unwrap_decryptionconfig(object decryptionconfig) except *
+cdef shared_ptr[CExternalEncryptionConfiguration] pyarrow_unwrap_external_encryptionconfig(object externalencryptionconfig) except *
 cdef shared_ptr[CExternalDecryptionConfiguration] pyarrow_unwrap_external_decryptionconfig(object externaldecryptionconfig) except *

python/pyarrow/_parquet_encryption.pyx

@@ -419,13 +419,29 @@ cdef class ExternalDecryptionConfiguration(DecryptionConfiguration):
     __slots__ = ()
 
     def __init__(self, *, cache_lifetime=None, app_context=None, connection_config=None):
-        super().__init__(cache_lifetime=cache_lifetime)
+        # Initialize the pointer first so the get/set forwards work.
+        # Super init will run the setters/getters below so we need the pointer to exist.
         self.external_configuration.reset(new CExternalDecryptionConfiguration())
+        super().__init__(cache_lifetime=cache_lifetime)
+
+        self.external_configuration.get().cache_lifetime_seconds = \
+            self.configuration.get().cache_lifetime_seconds
 
         if app_context is not None:
             self.app_context = app_context
         if connection_config is not None:
             self.connection_config = connection_config
+
+    """ Forward all attributes get/set methods to the superclass """
+    """ The superclass already converts to/from bytes and does additional processing needed """
+    @property
+    def cache_lifetime(self):
+        return DecryptionConfiguration.cache_lifetime.__get__(self)
+
+    @cache_lifetime.setter
+    def cache_lifetime(self, value):
+        DecryptionConfiguration.cache_lifetime.__set__(self, value)
+        self.external_configuration.get().cache_lifetime_seconds = value.total_seconds()
 
     @property
     def app_context(self):
@@ -759,6 +775,39 @@ cdef class CryptoFactory(_Weakrefable):
             c_file_decryption_properties)
         return FileDecryptionProperties.wrap(file_decryption_properties)
 
+    def external_file_decryption_properties(
+            self,
+            KmsConnectionConfig kms_connection_config,
+            ExternalDecryptionConfiguration decryption_config):
+        """Create file decryption properties.
+        Parameters
+        ----------
+        kms_connection_config : KmsConnectionConfig
+            Configuration of connection to KMS
+        decryption_config : ExternalDecryptionConfiguration
+            Configuration of the decryption, such as cache timeout and the information on how to connect the external decryption service.
+        Returns
+        -------
+        file_decryption_properties : ExternalFileDecryptionProperties
+            File decryption properties.
+        """
+        cdef:
+            CExternalDecryptionConfiguration c_decryption_config
+            CResult[shared_ptr[CExternalFileDecryptionProperties]] \
+                c_file_decryption_properties
+        if decryption_config is None:
+            c_decryption_config = CExternalDecryptionConfiguration()
+        else:
+            c_decryption_config = deref(decryption_config.unwrap_external().get())
+        with nogil:
+            c_file_decryption_properties = \
+                self.factory.get().SafeGetExternalFileDecryptionProperties(
+                    deref(kms_connection_config.unwrap().get()),
+                    c_decryption_config)
+        file_decryption_properties = GetResultValue(
+            c_file_decryption_properties)
+        return ExternalFileDecryptionProperties.wrap_external(file_decryption_properties)
+
     def remove_cache_entries_for_token(self, access_token):
         self.factory.get().RemoveCacheEntriesForToken(tobytes(access_token))
 
@@ -800,4 +849,4 @@ cdef shared_ptr[CDecryptionConfiguration] pyarrow_unwrap_decryptionconfig(object
 cdef shared_ptr[CExternalDecryptionConfiguration] pyarrow_unwrap_external_decryptionconfig(object decryptionconfig) except *:
     if isinstance(decryptionconfig, ExternalDecryptionConfiguration):
         return (<ExternalDecryptionConfiguration> decryptionconfig).unwrap_external()
-    raise TypeError("Expected DecryptionConfiguration, got %s" % type(decryptionconfig))
+    raise TypeError("Expected ExternalDecryptionConfiguration, got %s" % type(decryptionconfig))

python/pyarrow/includes/libparquet_encryption.pxd

@@ -22,6 +22,7 @@ from pyarrow._parquet cimport (ParquetCipher,
                                CFileEncryptionProperties,
                                CExternalFileEncryptionProperties,
                                CFileDecryptionProperties,
+                               CExternalFileDecryptionProperties,
                                ParquetCipher_AES_GCM_V1,
                                ParquetCipher_AES_GCM_CTR_V1,
                                ParquetCipher_EXTERNAL_DBPA_V1)
@@ -126,6 +127,9 @@ cdef extern from "parquet/encryption/crypto_factory.h" \
         shared_ptr[CFileDecryptionProperties] GetFileDecryptionProperties(
             const CKmsConnectionConfig& kms_connection_config,
             const CDecryptionConfiguration& decryption_config) except +*
+        shared_ptr[CExternalFileDecryptionProperties] GetExternalFileDecryptionProperties(
+            const CKmsConnectionConfig& kms_connection_config,
+            const CExternalDecryptionConfiguration& decryption_config) except +*
         void RemoveCacheEntriesForToken(const c_string& access_token) except +
         void RemoveCacheEntriesForAllTokens() except +
 
@@ -164,4 +168,8 @@ cdef extern from "arrow/python/parquet_encryption.h" \
         CResult[shared_ptr[CFileDecryptionProperties]] \
             SafeGetFileDecryptionProperties(
             const CKmsConnectionConfig& kms_connection_config,
-            const CDecryptionConfiguration& decryption_config)
+            const CDecryptionConfiguration& decryption_config)
+        CResult[shared_ptr[CExternalFileDecryptionProperties]] \
+            SafeGetExternalFileDecryptionProperties(
+            const CKmsConnectionConfig& kms_connection_config,
+            const CExternalDecryptionConfiguration& decryption_config)