From b986ba0d5f9969732e7363eaea91db98a8149c3c Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Tue, 19 Aug 2025 17:18:30 -0600 Subject: [PATCH 01/13] Solved conflicts --- python/pyarrow/_parquet.pxd | 21 +++++++++- python/pyarrow/_parquet_encryption.pxd | 3 ++ python/pyarrow/_parquet_encryption.pyx | 41 ++++++++++++++++++- .../includes/libparquet_encryption.pxd | 10 ++++- 4 files changed, 71 insertions(+), 4 deletions(-) diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd index 69226bf0c344..f134b9d7fc0b 100644 --- a/python/pyarrow/_parquet.pxd +++ b/python/pyarrow/_parquet.pxd @@ -680,7 +680,10 @@ cdef extern from "parquet/encryption/encryption.h" namespace "parquet" nogil: cdef cppclass CFileDecryptionProperties\ " parquet::FileDecryptionProperties": pass - + + cdef cppclass CExternalFileDecryptionProperties\ + " parquet::ExternalFileDecryptionProperties": + pass cdef cppclass CFileEncryptionProperties\ " parquet::FileEncryptionProperties": pass @@ -704,3 +707,19 @@ cdef class FileDecryptionProperties: cdef inline shared_ptr[CFileDecryptionProperties] unwrap(self): return self.properties + +cdef class ExternalFileDecryptionProperties: + """File-level decryption properties for the low-level API""" + cdef: + shared_ptr[CExternalFileDecryptionProperties] properties + + @staticmethod + cdef inline ExternalFileDecryptionProperties wrap( + shared_ptr[CExternalFileDecryptionProperties] properties): + + result = ExternalFileDecryptionProperties() + result.properties = properties + return result + + cdef inline shared_ptr[CExternalFileDecryptionProperties] unwrap_external(self): + return self.properties \ No newline at end of file diff --git a/python/pyarrow/_parquet_encryption.pxd b/python/pyarrow/_parquet_encryption.pxd index 97897eb243fd..09f24a61ab98 100644 --- a/python/pyarrow/_parquet_encryption.pxd +++ b/python/pyarrow/_parquet_encryption.pxd @@ -24,9 +24,11 @@ from pyarrow._parquet cimport (ParquetCipher, CFileEncryptionProperties, CExternalFileEncryptionProperties, CFileDecryptionProperties, + CExternalFileDecryptionProperties, FileEncryptionProperties, ExternalFileEncryptionProperties, FileDecryptionProperties, + ExternalFileDecryptionProperties, ParquetCipher_AES_GCM_V1, ParquetCipher_AES_GCM_CTR_V1, ParquetCipher_EXTERNAL_DBPA_V1) @@ -57,6 +59,7 @@ cdef shared_ptr[CCryptoFactory] pyarrow_unwrap_cryptofactory(object crypto_facto cdef shared_ptr[CKmsConnectionConfig] pyarrow_unwrap_kmsconnectionconfig(object kmsconnectionconfig) except * cdef shared_ptr[CEncryptionConfiguration] pyarrow_unwrap_encryptionconfig(object encryptionconfig) except * cdef shared_ptr[CDecryptionConfiguration] pyarrow_unwrap_decryptionconfig(object decryptionconfig) except * +cdef shared_ptr[CExternalDecryptionConfiguration] pyarrow_unwrap_external_decryptionconfig(object decryptionconfig) except * cdef shared_ptr[CExternalEncryptionConfiguration] pyarrow_unwrap_external_encryptionconfig(object encryptionconfig) except * diff --git a/python/pyarrow/_parquet_encryption.pyx b/python/pyarrow/_parquet_encryption.pyx index 966e9d187503..87bfcd464435 100644 --- a/python/pyarrow/_parquet_encryption.pyx +++ b/python/pyarrow/_parquet_encryption.pyx @@ -426,7 +426,7 @@ cdef class ExternalDecryptionConfiguration(DecryptionConfiguration): self.app_context = app_context if connection_config is not None: self.connection_config = connection_config - + @property def app_context(self): """Get the application context as a dictionary.""" @@ -759,6 +759,43 @@ cdef class CryptoFactory(_Weakrefable): c_file_decryption_properties) return FileDecryptionProperties.wrap(file_decryption_properties) + def external_file_decryption_properties( + self, + KmsConnectionConfig kms_connection_config, + ExternalDecryptionConfiguration decryption_config=None): + """Create file decryption properties. + + Parameters + ---------- + kms_connection_config : KmsConnectionConfig + Configuration of connection to KMS + + decryption_config : ExternalDecryptionConfiguration, default None + Configuration of the decryption, such as cache timeout. + Can be None. + + Returns + ------- + file_decryption_properties : ExternalFileDecryptionProperties + File decryption properties. + """ + cdef: + CExternalDecryptionConfiguration c_decryption_config + CResult[shared_ptr[CExternalFileDecryptionProperties]] \ + c_file_decryption_properties + if decryption_config is None: + c_decryption_config = CExternalDecryptionConfiguration() + else: + c_decryption_config = deref(decryption_config.unwrap_external().get()) + with nogil: + c_file_decryption_properties = \ + self.factory.get().SafeGetExternalFileDecryptionProperties( + deref(kms_connection_config.unwrap().get()), + c_decryption_config) + file_decryption_properties = GetResultValue( + c_file_decryption_properties) + return ExternalFileDecryptionProperties.wrap(file_decryption_properties) + def remove_cache_entries_for_token(self, access_token): self.factory.get().RemoveCacheEntriesForToken(tobytes(access_token)) @@ -800,4 +837,4 @@ cdef shared_ptr[CDecryptionConfiguration] pyarrow_unwrap_decryptionconfig(object cdef shared_ptr[CExternalDecryptionConfiguration] pyarrow_unwrap_external_decryptionconfig(object decryptionconfig) except *: if isinstance(decryptionconfig, ExternalDecryptionConfiguration): return ( decryptionconfig).unwrap_external() - raise TypeError("Expected DecryptionConfiguration, got %s" % type(decryptionconfig)) \ No newline at end of file + raise TypeError("Expected ExternalDecryptionConfiguration, got %s" % type(decryptionconfig)) \ No newline at end of file diff --git a/python/pyarrow/includes/libparquet_encryption.pxd b/python/pyarrow/includes/libparquet_encryption.pxd index 690c79cece8b..8550a442717d 100644 --- a/python/pyarrow/includes/libparquet_encryption.pxd +++ b/python/pyarrow/includes/libparquet_encryption.pxd @@ -22,6 +22,7 @@ from pyarrow._parquet cimport (ParquetCipher, CFileEncryptionProperties, CExternalFileEncryptionProperties, CFileDecryptionProperties, + CExternalFileDecryptionProperties, ParquetCipher_AES_GCM_V1, ParquetCipher_AES_GCM_CTR_V1, ParquetCipher_EXTERNAL_DBPA_V1) @@ -126,6 +127,9 @@ cdef extern from "parquet/encryption/crypto_factory.h" \ shared_ptr[CFileDecryptionProperties] GetFileDecryptionProperties( const CKmsConnectionConfig& kms_connection_config, const CDecryptionConfiguration& decryption_config) except +* + shared_ptr[CExternalFileDecryptionProperties] GetExternalFileDecryptionProperties( + const CKmsConnectionConfig& kms_connection_config, + const CExternalDecryptionConfiguration& decryption_config) except +* void RemoveCacheEntriesForToken(const c_string& access_token) except + void RemoveCacheEntriesForAllTokens() except + @@ -164,4 +168,8 @@ cdef extern from "arrow/python/parquet_encryption.h" \ CResult[shared_ptr[CFileDecryptionProperties]] \ SafeGetFileDecryptionProperties( const CKmsConnectionConfig& kms_connection_config, - const CDecryptionConfiguration& decryption_config) \ No newline at end of file + const CDecryptionConfiguration& decryption_config) + CResult[shared_ptr[CExternalFileDecryptionProperties]] \ + SafeGetExternalFileDecryptionProperties( + const CKmsConnectionConfig& kms_connection_config, + const CExternalDecryptionConfiguration& decryption_config) \ No newline at end of file From 42f95bd9042db06290be9f9efb53a274095d9181 Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Tue, 19 Aug 2025 17:22:58 -0600 Subject: [PATCH 02/13] solved conflicts --- python/pyarrow/_parquet.pxd | 40 +------------- python/pyarrow/_parquet.pyx | 14 ++--- python/pyarrow/_parquet_encryption.pxd | 5 -- python/pyarrow/_parquet_encryption.pyx | 54 +------------------ .../includes/libparquet_encryption.pxd | 18 +------ .../tests/parquet/test_external_encryption.py | 2 +- 6 files changed, 7 insertions(+), 126 deletions(-) diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd index f134b9d7fc0b..09ef69fbb51f 100644 --- a/python/pyarrow/_parquet.pxd +++ b/python/pyarrow/_parquet.pxd @@ -580,21 +580,6 @@ cdef class FileEncryptionProperties: cdef inline shared_ptr[CFileEncryptionProperties] unwrap(self): return self.properties -cdef class ExternalFileEncryptionProperties(FileEncryptionProperties): - cdef: - shared_ptr[CExternalFileEncryptionProperties] properties - - @staticmethod - cdef inline ExternalFileEncryptionProperties wrap_external( - shared_ptr[CExternalFileEncryptionProperties] properties): - - result = ExternalFileEncryptionProperties() - result.properties = properties - return result - - cdef inline shared_ptr[CExternalFileEncryptionProperties] unwrap_external(self): - return self.properties - cdef shared_ptr[WriterProperties] _create_writer_properties( use_dictionary=*, compression=*, @@ -680,18 +665,11 @@ cdef extern from "parquet/encryption/encryption.h" namespace "parquet" nogil: cdef cppclass CFileDecryptionProperties\ " parquet::FileDecryptionProperties": pass - - cdef cppclass CExternalFileDecryptionProperties\ - " parquet::ExternalFileDecryptionProperties": - pass + cdef cppclass CFileEncryptionProperties\ " parquet::FileEncryptionProperties": pass - cdef cppclass CExternalFileEncryptionProperties\ - " parquet::ExternalFileEncryptionProperties": - pass - cdef class FileDecryptionProperties: """File-level decryption properties for the low-level API""" cdef: @@ -707,19 +685,3 @@ cdef class FileDecryptionProperties: cdef inline shared_ptr[CFileDecryptionProperties] unwrap(self): return self.properties - -cdef class ExternalFileDecryptionProperties: - """File-level decryption properties for the low-level API""" - cdef: - shared_ptr[CExternalFileDecryptionProperties] properties - - @staticmethod - cdef inline ExternalFileDecryptionProperties wrap( - shared_ptr[CExternalFileDecryptionProperties] properties): - - result = ExternalFileDecryptionProperties() - result.properties = properties - return result - - cdef inline shared_ptr[CExternalFileDecryptionProperties] unwrap_external(self): - return self.properties \ No newline at end of file diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx index f92d2f27fcbf..55c2866243b0 100644 --- a/python/pyarrow/_parquet.pyx +++ b/python/pyarrow/_parquet.pyx @@ -45,10 +45,7 @@ from pyarrow.lib import (ArrowException, NativeFile, BufferOutputStream, tobytes, frombytes, is_threading_enabled) cimport cpython as cp -from libcpp.memory cimport shared_ptr -cdef extern from "memory" namespace "std": - shared_ptr[T] static_pointer_cast[T, U](shared_ptr[U] r) _DEFAULT_ROW_GROUP_SIZE = 1024*1024 _MAX_ROW_GROUP_SIZE = 64*1024*1024 @@ -2008,14 +2005,9 @@ cdef shared_ptr[WriterProperties] _create_writer_properties( # encryption if encryption_properties is not None: - if isinstance(encryption_properties, ExternalFileEncryptionProperties): - props.encryption( - static_pointer_cast[CFileEncryptionProperties, CExternalFileEncryptionProperties] ( - (encryption_properties).unwrap_external())) - else: - - props.encryption((encryption_properties).unwrap()) - + props.encryption( + (encryption_properties).unwrap()) + # For backwards compatibility reasons we cap the maximum row group size # at 64Mi rows. This could be changed in the future, though it would be # a breaking change. diff --git a/python/pyarrow/_parquet_encryption.pxd b/python/pyarrow/_parquet_encryption.pxd index 09f24a61ab98..e4182608b079 100644 --- a/python/pyarrow/_parquet_encryption.pxd +++ b/python/pyarrow/_parquet_encryption.pxd @@ -22,13 +22,9 @@ from pyarrow.includes.common cimport * from pyarrow.includes.libparquet_encryption cimport * from pyarrow._parquet cimport (ParquetCipher, CFileEncryptionProperties, - CExternalFileEncryptionProperties, CFileDecryptionProperties, - CExternalFileDecryptionProperties, FileEncryptionProperties, - ExternalFileEncryptionProperties, FileDecryptionProperties, - ExternalFileDecryptionProperties, ParquetCipher_AES_GCM_V1, ParquetCipher_AES_GCM_CTR_V1, ParquetCipher_EXTERNAL_DBPA_V1) @@ -59,7 +55,6 @@ cdef shared_ptr[CCryptoFactory] pyarrow_unwrap_cryptofactory(object crypto_facto cdef shared_ptr[CKmsConnectionConfig] pyarrow_unwrap_kmsconnectionconfig(object kmsconnectionconfig) except * cdef shared_ptr[CEncryptionConfiguration] pyarrow_unwrap_encryptionconfig(object encryptionconfig) except * cdef shared_ptr[CDecryptionConfiguration] pyarrow_unwrap_decryptionconfig(object decryptionconfig) except * -cdef shared_ptr[CExternalDecryptionConfiguration] pyarrow_unwrap_external_decryptionconfig(object decryptionconfig) except * cdef shared_ptr[CExternalEncryptionConfiguration] pyarrow_unwrap_external_encryptionconfig(object encryptionconfig) except * diff --git a/python/pyarrow/_parquet_encryption.pyx b/python/pyarrow/_parquet_encryption.pyx index 87bfcd464435..cbaeb4527af2 100644 --- a/python/pyarrow/_parquet_encryption.pyx +++ b/python/pyarrow/_parquet_encryption.pyx @@ -706,22 +706,7 @@ cdef class CryptoFactory(_Weakrefable): file_encryption_properties = GetResultValue( file_encryption_properties_result) return FileEncryptionProperties.wrap(file_encryption_properties) - - def external_file_encryption_properties(self, - KmsConnectionConfig kms_connection_config, - ExternalEncryptionConfiguration external_encryption_config): - cdef: - CResult[shared_ptr[CExternalFileEncryptionProperties]] \ - external_file_encryption_properties_result - with nogil: - external_file_encryption_properties_result = \ - self.factory.get().SafeGetExternalFileEncryptionProperties( - deref(kms_connection_config.unwrap().get()), - deref(external_encryption_config.unwrap_external().get())) - external_file_encryption_properties = GetResultValue( - external_file_encryption_properties_result) - return ExternalFileEncryptionProperties.wrap_external(external_file_encryption_properties) - + def file_decryption_properties( self, KmsConnectionConfig kms_connection_config, @@ -759,43 +744,6 @@ cdef class CryptoFactory(_Weakrefable): c_file_decryption_properties) return FileDecryptionProperties.wrap(file_decryption_properties) - def external_file_decryption_properties( - self, - KmsConnectionConfig kms_connection_config, - ExternalDecryptionConfiguration decryption_config=None): - """Create file decryption properties. - - Parameters - ---------- - kms_connection_config : KmsConnectionConfig - Configuration of connection to KMS - - decryption_config : ExternalDecryptionConfiguration, default None - Configuration of the decryption, such as cache timeout. - Can be None. - - Returns - ------- - file_decryption_properties : ExternalFileDecryptionProperties - File decryption properties. - """ - cdef: - CExternalDecryptionConfiguration c_decryption_config - CResult[shared_ptr[CExternalFileDecryptionProperties]] \ - c_file_decryption_properties - if decryption_config is None: - c_decryption_config = CExternalDecryptionConfiguration() - else: - c_decryption_config = deref(decryption_config.unwrap_external().get()) - with nogil: - c_file_decryption_properties = \ - self.factory.get().SafeGetExternalFileDecryptionProperties( - deref(kms_connection_config.unwrap().get()), - c_decryption_config) - file_decryption_properties = GetResultValue( - c_file_decryption_properties) - return ExternalFileDecryptionProperties.wrap(file_decryption_properties) - def remove_cache_entries_for_token(self, access_token): self.factory.get().RemoveCacheEntriesForToken(tobytes(access_token)) diff --git a/python/pyarrow/includes/libparquet_encryption.pxd b/python/pyarrow/includes/libparquet_encryption.pxd index 8550a442717d..13b2a3aee071 100644 --- a/python/pyarrow/includes/libparquet_encryption.pxd +++ b/python/pyarrow/includes/libparquet_encryption.pxd @@ -20,9 +20,7 @@ from pyarrow.includes.common cimport * from pyarrow._parquet cimport (ParquetCipher, CFileEncryptionProperties, - CExternalFileEncryptionProperties, CFileDecryptionProperties, - CExternalFileDecryptionProperties, ParquetCipher_AES_GCM_V1, ParquetCipher_AES_GCM_CTR_V1, ParquetCipher_EXTERNAL_DBPA_V1) @@ -121,15 +119,9 @@ cdef extern from "parquet/encryption/crypto_factory.h" \ shared_ptr[CFileEncryptionProperties] GetFileEncryptionProperties( const CKmsConnectionConfig& kms_connection_config, const CEncryptionConfiguration& encryption_config) except +* - shared_ptr[CExternalFileEncryptionProperties] GetExternalFileEncryptionProperties( - const CKmsConnectionConfig& kms_connection_config, - const CExternalEncryptionConfiguration& external_encryption_config) except +* shared_ptr[CFileDecryptionProperties] GetFileDecryptionProperties( const CKmsConnectionConfig& kms_connection_config, const CDecryptionConfiguration& decryption_config) except +* - shared_ptr[CExternalFileDecryptionProperties] GetExternalFileDecryptionProperties( - const CKmsConnectionConfig& kms_connection_config, - const CExternalDecryptionConfiguration& decryption_config) except +* void RemoveCacheEntriesForToken(const c_string& access_token) except + void RemoveCacheEntriesForAllTokens() except + @@ -161,15 +153,7 @@ cdef extern from "arrow/python/parquet_encryption.h" \ SafeGetFileEncryptionProperties( const CKmsConnectionConfig& kms_connection_config, const CEncryptionConfiguration& encryption_config) - CResult[shared_ptr[CExternalFileEncryptionProperties]] \ - SafeGetExternalFileEncryptionProperties( - const CKmsConnectionConfig& kms_connection_config, - const CExternalEncryptionConfiguration& external_encryption_config) CResult[shared_ptr[CFileDecryptionProperties]] \ SafeGetFileDecryptionProperties( const CKmsConnectionConfig& kms_connection_config, - const CDecryptionConfiguration& decryption_config) - CResult[shared_ptr[CExternalFileDecryptionProperties]] \ - SafeGetExternalFileDecryptionProperties( - const CKmsConnectionConfig& kms_connection_config, - const CExternalDecryptionConfiguration& decryption_config) \ No newline at end of file + const CDecryptionConfiguration& decryption_config) \ No newline at end of file diff --git a/python/pyarrow/tests/parquet/test_external_encryption.py b/python/pyarrow/tests/parquet/test_external_encryption.py index 76f5c7052713..286f7b022c48 100644 --- a/python/pyarrow/tests/parquet/test_external_encryption.py +++ b/python/pyarrow/tests/parquet/test_external_encryption.py @@ -290,7 +290,7 @@ def test_external_decryption_connection_config_invalid_types(): "config_file": "should-fail" } } - + # Outer value is not a dict with pytest.raises(TypeError, match="Inner value for cipher AES_GCM_V1 must be a dict"): config = pe.ExternalDecryptionConfiguration() From 6af4c5f52e8b7fd4c2ecbcefaf1c44bf4bb1537d Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Tue, 19 Aug 2025 17:29:02 -0600 Subject: [PATCH 03/13] solved conflicts --- python/pyarrow/_parquet.pxd | 19 +++++++++++++++++++ python/pyarrow/_parquet.pyx | 14 +++++++++++--- python/pyarrow/_parquet_encryption.pxd | 2 ++ python/pyarrow/_parquet_encryption.pyx | 19 +++++++++++++++++-- .../includes/libparquet_encryption.pxd | 8 ++++++++ .../tests/parquet/test_external_encryption.py | 2 +- 6 files changed, 58 insertions(+), 6 deletions(-) diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd index 09ef69fbb51f..69226bf0c344 100644 --- a/python/pyarrow/_parquet.pxd +++ b/python/pyarrow/_parquet.pxd @@ -580,6 +580,21 @@ cdef class FileEncryptionProperties: cdef inline shared_ptr[CFileEncryptionProperties] unwrap(self): return self.properties +cdef class ExternalFileEncryptionProperties(FileEncryptionProperties): + cdef: + shared_ptr[CExternalFileEncryptionProperties] properties + + @staticmethod + cdef inline ExternalFileEncryptionProperties wrap_external( + shared_ptr[CExternalFileEncryptionProperties] properties): + + result = ExternalFileEncryptionProperties() + result.properties = properties + return result + + cdef inline shared_ptr[CExternalFileEncryptionProperties] unwrap_external(self): + return self.properties + cdef shared_ptr[WriterProperties] _create_writer_properties( use_dictionary=*, compression=*, @@ -670,6 +685,10 @@ cdef extern from "parquet/encryption/encryption.h" namespace "parquet" nogil: " parquet::FileEncryptionProperties": pass + cdef cppclass CExternalFileEncryptionProperties\ + " parquet::ExternalFileEncryptionProperties": + pass + cdef class FileDecryptionProperties: """File-level decryption properties for the low-level API""" cdef: diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx index 55c2866243b0..f92d2f27fcbf 100644 --- a/python/pyarrow/_parquet.pyx +++ b/python/pyarrow/_parquet.pyx @@ -45,7 +45,10 @@ from pyarrow.lib import (ArrowException, NativeFile, BufferOutputStream, tobytes, frombytes, is_threading_enabled) cimport cpython as cp +from libcpp.memory cimport shared_ptr +cdef extern from "memory" namespace "std": + shared_ptr[T] static_pointer_cast[T, U](shared_ptr[U] r) _DEFAULT_ROW_GROUP_SIZE = 1024*1024 _MAX_ROW_GROUP_SIZE = 64*1024*1024 @@ -2005,9 +2008,14 @@ cdef shared_ptr[WriterProperties] _create_writer_properties( # encryption if encryption_properties is not None: - props.encryption( - (encryption_properties).unwrap()) - + if isinstance(encryption_properties, ExternalFileEncryptionProperties): + props.encryption( + static_pointer_cast[CFileEncryptionProperties, CExternalFileEncryptionProperties] ( + (encryption_properties).unwrap_external())) + else: + + props.encryption((encryption_properties).unwrap()) + # For backwards compatibility reasons we cap the maximum row group size # at 64Mi rows. This could be changed in the future, though it would be # a breaking change. diff --git a/python/pyarrow/_parquet_encryption.pxd b/python/pyarrow/_parquet_encryption.pxd index e4182608b079..97897eb243fd 100644 --- a/python/pyarrow/_parquet_encryption.pxd +++ b/python/pyarrow/_parquet_encryption.pxd @@ -22,8 +22,10 @@ from pyarrow.includes.common cimport * from pyarrow.includes.libparquet_encryption cimport * from pyarrow._parquet cimport (ParquetCipher, CFileEncryptionProperties, + CExternalFileEncryptionProperties, CFileDecryptionProperties, FileEncryptionProperties, + ExternalFileEncryptionProperties, FileDecryptionProperties, ParquetCipher_AES_GCM_V1, ParquetCipher_AES_GCM_CTR_V1, diff --git a/python/pyarrow/_parquet_encryption.pyx b/python/pyarrow/_parquet_encryption.pyx index cbaeb4527af2..fa3825a50b07 100644 --- a/python/pyarrow/_parquet_encryption.pyx +++ b/python/pyarrow/_parquet_encryption.pyx @@ -426,7 +426,7 @@ cdef class ExternalDecryptionConfiguration(DecryptionConfiguration): self.app_context = app_context if connection_config is not None: self.connection_config = connection_config - + @property def app_context(self): """Get the application context as a dictionary.""" @@ -706,7 +706,22 @@ cdef class CryptoFactory(_Weakrefable): file_encryption_properties = GetResultValue( file_encryption_properties_result) return FileEncryptionProperties.wrap(file_encryption_properties) - + + def external_file_encryption_properties(self, + KmsConnectionConfig kms_connection_config, + ExternalEncryptionConfiguration external_encryption_config): + cdef: + CResult[shared_ptr[CExternalFileEncryptionProperties]] \ + external_file_encryption_properties_result + with nogil: + external_file_encryption_properties_result = \ + self.factory.get().SafeGetExternalFileEncryptionProperties( + deref(kms_connection_config.unwrap().get()), + deref(external_encryption_config.unwrap_external().get())) + external_file_encryption_properties = GetResultValue( + external_file_encryption_properties_result) + return ExternalFileEncryptionProperties.wrap_external(external_file_encryption_properties) + def file_decryption_properties( self, KmsConnectionConfig kms_connection_config, diff --git a/python/pyarrow/includes/libparquet_encryption.pxd b/python/pyarrow/includes/libparquet_encryption.pxd index 13b2a3aee071..690c79cece8b 100644 --- a/python/pyarrow/includes/libparquet_encryption.pxd +++ b/python/pyarrow/includes/libparquet_encryption.pxd @@ -20,6 +20,7 @@ from pyarrow.includes.common cimport * from pyarrow._parquet cimport (ParquetCipher, CFileEncryptionProperties, + CExternalFileEncryptionProperties, CFileDecryptionProperties, ParquetCipher_AES_GCM_V1, ParquetCipher_AES_GCM_CTR_V1, @@ -119,6 +120,9 @@ cdef extern from "parquet/encryption/crypto_factory.h" \ shared_ptr[CFileEncryptionProperties] GetFileEncryptionProperties( const CKmsConnectionConfig& kms_connection_config, const CEncryptionConfiguration& encryption_config) except +* + shared_ptr[CExternalFileEncryptionProperties] GetExternalFileEncryptionProperties( + const CKmsConnectionConfig& kms_connection_config, + const CExternalEncryptionConfiguration& external_encryption_config) except +* shared_ptr[CFileDecryptionProperties] GetFileDecryptionProperties( const CKmsConnectionConfig& kms_connection_config, const CDecryptionConfiguration& decryption_config) except +* @@ -153,6 +157,10 @@ cdef extern from "arrow/python/parquet_encryption.h" \ SafeGetFileEncryptionProperties( const CKmsConnectionConfig& kms_connection_config, const CEncryptionConfiguration& encryption_config) + CResult[shared_ptr[CExternalFileEncryptionProperties]] \ + SafeGetExternalFileEncryptionProperties( + const CKmsConnectionConfig& kms_connection_config, + const CExternalEncryptionConfiguration& external_encryption_config) CResult[shared_ptr[CFileDecryptionProperties]] \ SafeGetFileDecryptionProperties( const CKmsConnectionConfig& kms_connection_config, diff --git a/python/pyarrow/tests/parquet/test_external_encryption.py b/python/pyarrow/tests/parquet/test_external_encryption.py index 286f7b022c48..76f5c7052713 100644 --- a/python/pyarrow/tests/parquet/test_external_encryption.py +++ b/python/pyarrow/tests/parquet/test_external_encryption.py @@ -290,7 +290,7 @@ def test_external_decryption_connection_config_invalid_types(): "config_file": "should-fail" } } - + # Outer value is not a dict with pytest.raises(TypeError, match="Inner value for cipher AES_GCM_V1 must be a dict"): config = pe.ExternalDecryptionConfiguration() From bb658ddafd8f0f39810d9f82535168550250bd71 Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Tue, 19 Aug 2025 17:31:49 -0600 Subject: [PATCH 04/13] solved conflicts --- python/pyarrow/_parquet.pxd | 20 +++++++++++ python/pyarrow/_parquet_encryption.pxd | 3 ++ python/pyarrow/_parquet_encryption.pyx | 34 +++++++++++++++++++ .../includes/libparquet_encryption.pxd | 10 +++++- .../tests/parquet/test_external_encryption.py | 26 ++++++++++---- 5 files changed, 86 insertions(+), 7 deletions(-) diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd index 69226bf0c344..d59bdaf0a6b9 100644 --- a/python/pyarrow/_parquet.pxd +++ b/python/pyarrow/_parquet.pxd @@ -681,6 +681,10 @@ cdef extern from "parquet/encryption/encryption.h" namespace "parquet" nogil: " parquet::FileDecryptionProperties": pass + cdef cppclass CExternalFileDecryptionProperties\ + " parquet::ExternalFileDecryptionProperties": + pass + cdef cppclass CFileEncryptionProperties\ " parquet::FileEncryptionProperties": pass @@ -704,3 +708,19 @@ cdef class FileDecryptionProperties: cdef inline shared_ptr[CFileDecryptionProperties] unwrap(self): return self.properties + +cdef class ExternalFileDecryptionProperties: + """File-level decryption properties for the low-level API""" + cdef: + shared_ptr[CExternalFileDecryptionProperties] properties + + @staticmethod + cdef inline ExternalFileDecryptionProperties wrap( + shared_ptr[CExternalFileDecryptionProperties] properties): + + result = ExternalFileDecryptionProperties() + result.properties = properties + return result + + cdef inline shared_ptr[CExternalFileDecryptionProperties] unwrap_external(self): + return self.properties diff --git a/python/pyarrow/_parquet_encryption.pxd b/python/pyarrow/_parquet_encryption.pxd index 97897eb243fd..3063ade1ca31 100644 --- a/python/pyarrow/_parquet_encryption.pxd +++ b/python/pyarrow/_parquet_encryption.pxd @@ -24,9 +24,11 @@ from pyarrow._parquet cimport (ParquetCipher, CFileEncryptionProperties, CExternalFileEncryptionProperties, CFileDecryptionProperties, + CExternalFileDecryptionProperties, FileEncryptionProperties, ExternalFileEncryptionProperties, FileDecryptionProperties, + ExternalFileDecryptionProperties, ParquetCipher_AES_GCM_V1, ParquetCipher_AES_GCM_CTR_V1, ParquetCipher_EXTERNAL_DBPA_V1) @@ -58,6 +60,7 @@ cdef shared_ptr[CKmsConnectionConfig] pyarrow_unwrap_kmsconnectionconfig(object cdef shared_ptr[CEncryptionConfiguration] pyarrow_unwrap_encryptionconfig(object encryptionconfig) except * cdef shared_ptr[CDecryptionConfiguration] pyarrow_unwrap_decryptionconfig(object decryptionconfig) except * cdef shared_ptr[CExternalEncryptionConfiguration] pyarrow_unwrap_external_encryptionconfig(object encryptionconfig) except * +cdef shared_ptr[CExternalDecryptionConfiguration] pyarrow_unwrap_external_decryptionconfig(object decryptionconfig) except * cdef class ExternalEncryptionConfiguration(EncryptionConfiguration): diff --git a/python/pyarrow/_parquet_encryption.pyx b/python/pyarrow/_parquet_encryption.pyx index fa3825a50b07..be8dd26dbbd4 100644 --- a/python/pyarrow/_parquet_encryption.pyx +++ b/python/pyarrow/_parquet_encryption.pyx @@ -759,6 +759,40 @@ cdef class CryptoFactory(_Weakrefable): c_file_decryption_properties) return FileDecryptionProperties.wrap(file_decryption_properties) + def external_file_decryption_properties( + self, + KmsConnectionConfig kms_connection_config, + ExternalDecryptionConfiguration decryption_config=None): + """Create file decryption properties. + Parameters + ---------- + kms_connection_config : KmsConnectionConfig + Configuration of connection to KMS + decryption_config : ExternalDecryptionConfiguration, default None + Configuration of the decryption, such as cache timeout. + Can be None. + Returns + ------- + file_decryption_properties : ExternalFileDecryptionProperties + File decryption properties. + """ + cdef: + CExternalDecryptionConfiguration c_decryption_config + CResult[shared_ptr[CExternalFileDecryptionProperties]] \ + c_file_decryption_properties + if decryption_config is None: + c_decryption_config = CExternalDecryptionConfiguration() + else: + c_decryption_config = deref(decryption_config.unwrap_external().get()) + with nogil: + c_file_decryption_properties = \ + self.factory.get().SafeGetExternalFileDecryptionProperties( + deref(kms_connection_config.unwrap().get()), + c_decryption_config) + file_decryption_properties = GetResultValue( + c_file_decryption_properties) + return ExternalFileDecryptionProperties.wrap(file_decryption_properties) + def remove_cache_entries_for_token(self, access_token): self.factory.get().RemoveCacheEntriesForToken(tobytes(access_token)) diff --git a/python/pyarrow/includes/libparquet_encryption.pxd b/python/pyarrow/includes/libparquet_encryption.pxd index 690c79cece8b..8550a442717d 100644 --- a/python/pyarrow/includes/libparquet_encryption.pxd +++ b/python/pyarrow/includes/libparquet_encryption.pxd @@ -22,6 +22,7 @@ from pyarrow._parquet cimport (ParquetCipher, CFileEncryptionProperties, CExternalFileEncryptionProperties, CFileDecryptionProperties, + CExternalFileDecryptionProperties, ParquetCipher_AES_GCM_V1, ParquetCipher_AES_GCM_CTR_V1, ParquetCipher_EXTERNAL_DBPA_V1) @@ -126,6 +127,9 @@ cdef extern from "parquet/encryption/crypto_factory.h" \ shared_ptr[CFileDecryptionProperties] GetFileDecryptionProperties( const CKmsConnectionConfig& kms_connection_config, const CDecryptionConfiguration& decryption_config) except +* + shared_ptr[CExternalFileDecryptionProperties] GetExternalFileDecryptionProperties( + const CKmsConnectionConfig& kms_connection_config, + const CExternalDecryptionConfiguration& decryption_config) except +* void RemoveCacheEntriesForToken(const c_string& access_token) except + void RemoveCacheEntriesForAllTokens() except + @@ -164,4 +168,8 @@ cdef extern from "arrow/python/parquet_encryption.h" \ CResult[shared_ptr[CFileDecryptionProperties]] \ SafeGetFileDecryptionProperties( const CKmsConnectionConfig& kms_connection_config, - const CDecryptionConfiguration& decryption_config) \ No newline at end of file + const CDecryptionConfiguration& decryption_config) + CResult[shared_ptr[CExternalFileDecryptionProperties]] \ + SafeGetExternalFileDecryptionProperties( + const CKmsConnectionConfig& kms_connection_config, + const CExternalDecryptionConfiguration& decryption_config) \ No newline at end of file diff --git a/python/pyarrow/tests/parquet/test_external_encryption.py b/python/pyarrow/tests/parquet/test_external_encryption.py index 76f5c7052713..531fb9e75346 100644 --- a/python/pyarrow/tests/parquet/test_external_encryption.py +++ b/python/pyarrow/tests/parquet/test_external_encryption.py @@ -15,11 +15,14 @@ # specific language governing permissions and limitations # under the License. +<<<<<<< HEAD import pytest import pyarrow.parquet.encryption as pe import re from datetime import timedelta +======= +>>>>>>> 749fc44a0 (Adding the bases for external file decryptor) @pytest.fixture def dummy_kms_client(): class DummyKmsClient(pe.KmsClient): @@ -40,8 +43,13 @@ def kms_config(): return pe.KmsConnectionConfig( custom_kms_conf={ "footer_key": "012footer_secret", +<<<<<<< HEAD "key_1": "column_secret001", "key_2": "column_secret002" +======= + "orderid_key": "column_secret001", + "productid_key": "column_secret002" +>>>>>>> 749fc44a0 (Adding the bases for external file decryptor) } ) @@ -264,15 +272,13 @@ def test_decryption_configuration_properties(): def test_external_decryption_configuration_properties(external_decryption_config): """Test the ExternalDecryptionConfiguration properties including external-specific fields.""" - config = external_decryption_config - - assert isinstance(config, pe.ExternalDecryptionConfiguration) - assert config.cache_lifetime == timedelta(minutes=5.0) - assert config.app_context == { + assert isinstance(external_decryption_config, pe.ExternalDecryptionConfiguration) + assert external_decryption_config.cache_lifetime == timedelta(minutes=5.0) + assert external_decryption_config.app_context == { "user_id": "Picard1701", "location": "Presidio" } - assert config.connection_config == { + assert external_decryption_config.connection_config == { "AES_GCM_V1": { "config_file": "path/to/config/file", "config_file_decryption_key": "some_key" @@ -315,3 +321,11 @@ def test_external_decryption_connection_config_invalid_types(): "config_file": ["not", "a", "string"] # invalid inner value } } + +def test_external_file_decryption_properties_valid(dummy_kms_client, kms_config, external_decryption_config): + factory = pe.CryptoFactory(dummy_kms_client) + result = factory.external_file_decryption_properties(kms_config, external_decryption_config) + + # Instead of isinstance, check class name and module dynamically because ExternalFileDecryptionProperties is not visbile + assert result.__class__.__name__ == "ExternalFileDecryptionProperties" + assert result.__class__.__module__ == "pyarrow._parquet" From e10e7165c0aa46480da5e535f581eb6fa64bf2cf Mon Sep 17 00:00:00 2001 From: cristekdatum Date: Thu, 14 Aug 2025 21:51:17 -0600 Subject: [PATCH 05/13] code review --- python/pyarrow/_parquet.pxd | 4 ++-- python/pyarrow/_parquet_encryption.pxd | 3 --- python/pyarrow/_parquet_encryption.pyx | 9 ++++----- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd index d59bdaf0a6b9..ff89128a40f5 100644 --- a/python/pyarrow/_parquet.pxd +++ b/python/pyarrow/_parquet.pxd @@ -715,7 +715,7 @@ cdef class ExternalFileDecryptionProperties: shared_ptr[CExternalFileDecryptionProperties] properties @staticmethod - cdef inline ExternalFileDecryptionProperties wrap( + cdef inline ExternalFileDecryptionProperties wrap_external( shared_ptr[CExternalFileDecryptionProperties] properties): result = ExternalFileDecryptionProperties() @@ -723,4 +723,4 @@ cdef class ExternalFileDecryptionProperties: return result cdef inline shared_ptr[CExternalFileDecryptionProperties] unwrap_external(self): - return self.properties + return self.properties diff --git a/python/pyarrow/_parquet_encryption.pxd b/python/pyarrow/_parquet_encryption.pxd index 3063ade1ca31..bf3a8788cd9c 100644 --- a/python/pyarrow/_parquet_encryption.pxd +++ b/python/pyarrow/_parquet_encryption.pxd @@ -59,9 +59,6 @@ cdef shared_ptr[CCryptoFactory] pyarrow_unwrap_cryptofactory(object crypto_facto cdef shared_ptr[CKmsConnectionConfig] pyarrow_unwrap_kmsconnectionconfig(object kmsconnectionconfig) except * cdef shared_ptr[CEncryptionConfiguration] pyarrow_unwrap_encryptionconfig(object encryptionconfig) except * cdef shared_ptr[CDecryptionConfiguration] pyarrow_unwrap_decryptionconfig(object decryptionconfig) except * -cdef shared_ptr[CExternalEncryptionConfiguration] pyarrow_unwrap_external_encryptionconfig(object encryptionconfig) except * -cdef shared_ptr[CExternalDecryptionConfiguration] pyarrow_unwrap_external_decryptionconfig(object decryptionconfig) except * - cdef class ExternalEncryptionConfiguration(EncryptionConfiguration): cdef shared_ptr[CExternalEncryptionConfiguration] external_configuration diff --git a/python/pyarrow/_parquet_encryption.pyx b/python/pyarrow/_parquet_encryption.pyx index be8dd26dbbd4..c64419b3d059 100644 --- a/python/pyarrow/_parquet_encryption.pyx +++ b/python/pyarrow/_parquet_encryption.pyx @@ -762,15 +762,14 @@ cdef class CryptoFactory(_Weakrefable): def external_file_decryption_properties( self, KmsConnectionConfig kms_connection_config, - ExternalDecryptionConfiguration decryption_config=None): + ExternalDecryptionConfiguration decryption_config): """Create file decryption properties. Parameters ---------- kms_connection_config : KmsConnectionConfig Configuration of connection to KMS - decryption_config : ExternalDecryptionConfiguration, default None - Configuration of the decryption, such as cache timeout. - Can be None. + decryption_config : ExternalDecryptionConfiguration + Configuration of the decryption, such as cache timeout and the information on how to connect the external decryption service. Returns ------- file_decryption_properties : ExternalFileDecryptionProperties @@ -791,7 +790,7 @@ cdef class CryptoFactory(_Weakrefable): c_decryption_config) file_decryption_properties = GetResultValue( c_file_decryption_properties) - return ExternalFileDecryptionProperties.wrap(file_decryption_properties) + return ExternalFileDecryptionProperties.wrap_external(file_decryption_properties) def remove_cache_entries_for_token(self, access_token): self.factory.get().RemoveCacheEntriesForToken(tobytes(access_token)) From 483ea2cda402eb76e44f31e66d3b4c03c91786da Mon Sep 17 00:00:00 2001 From: cristekdatum Date: Thu, 14 Aug 2025 21:58:46 -0600 Subject: [PATCH 06/13] code review --- python/pyarrow/_parquet_encryption.pxd | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/python/pyarrow/_parquet_encryption.pxd b/python/pyarrow/_parquet_encryption.pxd index bf3a8788cd9c..81c93a776a8e 100644 --- a/python/pyarrow/_parquet_encryption.pxd +++ b/python/pyarrow/_parquet_encryption.pxd @@ -54,18 +54,16 @@ cdef class KmsConnectionConfig(_Weakrefable): @staticmethod cdef wrap(const CKmsConnectionConfig& config) - -cdef shared_ptr[CCryptoFactory] pyarrow_unwrap_cryptofactory(object crypto_factory) except * -cdef shared_ptr[CKmsConnectionConfig] pyarrow_unwrap_kmsconnectionconfig(object kmsconnectionconfig) except * -cdef shared_ptr[CEncryptionConfiguration] pyarrow_unwrap_encryptionconfig(object encryptionconfig) except * -cdef shared_ptr[CDecryptionConfiguration] pyarrow_unwrap_decryptionconfig(object decryptionconfig) except * - cdef class ExternalEncryptionConfiguration(EncryptionConfiguration): cdef shared_ptr[CExternalEncryptionConfiguration] external_configuration cdef inline shared_ptr[CExternalEncryptionConfiguration] unwrap_external(self) nogil -cdef shared_ptr[CExternalEncryptionConfiguration] pyarrow_unwrap_external_encryptionconfig(object externalencryptionconfig) except * - cdef class ExternalDecryptionConfiguration(DecryptionConfiguration): cdef shared_ptr[CExternalDecryptionConfiguration] external_configuration cdef inline shared_ptr[CExternalDecryptionConfiguration] unwrap_external(self) nogil + +cdef shared_ptr[CCryptoFactory] pyarrow_unwrap_cryptofactory(object crypto_factory) except * +cdef shared_ptr[CKmsConnectionConfig] pyarrow_unwrap_kmsconnectionconfig(object kmsconnectionconfig) except * +cdef shared_ptr[CEncryptionConfiguration] pyarrow_unwrap_encryptionconfig(object encryptionconfig) except * +cdef shared_ptr[CDecryptionConfiguration] pyarrow_unwrap_decryptionconfig(object decryptionconfig) except * +cdef shared_ptr[CExternalEncryptionConfiguration] pyarrow_unwrap_external_encryptionconfig(object externalencryptionconfig) except * cdef shared_ptr[CExternalDecryptionConfiguration] pyarrow_unwrap_external_decryptionconfig(object externaldecryptionconfig) except * \ No newline at end of file From 1faa390e76b280499bcb973df78cf9ff4385ea90 Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Tue, 19 Aug 2025 17:36:53 -0600 Subject: [PATCH 07/13] solved conflicts --- python/pyarrow/_parquet.pxd | 2 +- python/pyarrow/_parquet.pyx | 8 ++- .../tests/parquet/test_external_encryption.py | 71 ++++++++++++++++--- 3 files changed, 70 insertions(+), 11 deletions(-) diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd index ff89128a40f5..d361c5dc6a26 100644 --- a/python/pyarrow/_parquet.pxd +++ b/python/pyarrow/_parquet.pxd @@ -709,7 +709,7 @@ cdef class FileDecryptionProperties: cdef inline shared_ptr[CFileDecryptionProperties] unwrap(self): return self.properties -cdef class ExternalFileDecryptionProperties: +cdef class ExternalFileDecryptionProperties(FileDecryptionProperties): """File-level decryption properties for the low-level API""" cdef: shared_ptr[CExternalFileDecryptionProperties] properties diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx index f92d2f27fcbf..bf3d0181e0c2 100644 --- a/python/pyarrow/_parquet.pyx +++ b/python/pyarrow/_parquet.pyx @@ -1507,8 +1507,12 @@ cdef class ParquetReader(_Weakrefable): thrift_container_size_limit) if decryption_properties is not None: - properties.file_decryption_properties( - decryption_properties.unwrap()) + if isinstance(decryption_properties, ExternalFileEncryptionProperties): + properties.file_decryption_properties( + static_pointer_cast[CFileDecryptionProperties, CExternalFileDecryptionProperties] ( + (decryption_properties).unwrap_external())) + else: + properties.file_decryption_properties((decryption_properties).unwrap()) arrow_props.set_pre_buffer(pre_buffer) diff --git a/python/pyarrow/tests/parquet/test_external_encryption.py b/python/pyarrow/tests/parquet/test_external_encryption.py index 531fb9e75346..aa66ac4ebe54 100644 --- a/python/pyarrow/tests/parquet/test_external_encryption.py +++ b/python/pyarrow/tests/parquet/test_external_encryption.py @@ -15,14 +15,11 @@ # specific language governing permissions and limitations # under the License. -<<<<<<< HEAD import pytest import pyarrow.parquet.encryption as pe import re from datetime import timedelta -======= ->>>>>>> 749fc44a0 (Adding the bases for external file decryptor) @pytest.fixture def dummy_kms_client(): class DummyKmsClient(pe.KmsClient): @@ -43,13 +40,8 @@ def kms_config(): return pe.KmsConnectionConfig( custom_kms_conf={ "footer_key": "012footer_secret", -<<<<<<< HEAD "key_1": "column_secret001", "key_2": "column_secret002" -======= - "orderid_key": "column_secret001", - "productid_key": "column_secret002" ->>>>>>> 749fc44a0 (Adding the bases for external file decryptor) } ) @@ -102,6 +94,43 @@ def external_decryption_config(): config.cache_lifetime=timedelta(minutes=5.0) return config + +@pytest.fixture +def external_decryption_config(): + config = pe.ExternalDecryptionConfiguration( + cache_lifetime=timedelta(minutes=5.0), + app_context={ + "user_id": "Picard1701", + "location": "Presidio" + }, + connection_config={ + "AES_GCM_V1": { + "config_file": "path/to/config/file", + "config_file_decryption_key": "some_key" + } + } + ) + config.cache_lifetime=timedelta(minutes=5.0) + return config + +@pytest.fixture +def encrypted_parquet_file(tmp_path, dummy_kms_client, kms_config, external_decryption_config): + """ + Fixture that creates a Parquet file encrypted with external encryption. + Other tests can depend on this fixture. + """ + file_path = tmp_path / "test_ext.parquet" + table = pa.Table.from_arrays([[1, 2, 3]], names=["col"]) + + factory = pe.CryptoFactory(dummy_kms_client) + encryption_props = factory.external_file_encryption_properties(kms_config, external_decryption_config) + + writer = pq.ParquetWriter(str(file_path), table.schema, encryption_properties=encryption_props) + writer.write_table(table) + writer.close() + + return file_path, table # Return the path and the original table for assertions + def test_encryption_configuration_properties(): """Test the standard EncryptionConfiguration properties to avoid regressions.""" @@ -260,6 +289,12 @@ def test_external_file_encryption_properties_valid( assert result.__class__.__name__ == "ExternalFileEncryptionProperties" assert result.__class__.__module__ == "pyarrow._parquet" +def test_write_table_with_external_encryption(encrypted_parquet_file): + file_path, table = encrypted_parquet_file + + # Check the file exists + assert file_path.exists() + def test_decryption_configuration_properties(): """Test the standard DecryptionConfiguration properties to avoid regressions.""" @@ -329,3 +364,23 @@ def test_external_file_decryption_properties_valid(dummy_kms_client, kms_config, # Instead of isinstance, check class name and module dynamically because ExternalFileDecryptionProperties is not visbile assert result.__class__.__name__ == "ExternalFileDecryptionProperties" assert result.__class__.__module__ == "pyarrow._parquet" + +def test_open_with_external_decryption(encrypted_parquet_file, dummy_kms_client, kms_config, external_encryption_config): + file_path, table = encrypted_parquet_file + + # Create decryption properties + factory = pe.CryptoFactory(dummy_kms_client) + decryption_props = factory.external_file_decryption_properties(kms_config, external_encryption_config) + + # Read the file + reader = ParquetReader() + reader.open(str(file_path), decryption_properties=decryption_props) + read_table = reader.read_table() + + # Assertions + assert read_table.num_rows == table.num_rows + assert read_table.num_columns == table.num_columns + assert read_table.schema.equals(table.schema) + + reader.close() + assert reader.closed From 2606f06d07dc9e9fb4ee28b7656ea156eff7ceb0 Mon Sep 17 00:00:00 2001 From: cristekdatum Date: Mon, 18 Aug 2025 12:19:00 -0600 Subject: [PATCH 08/13] missing files --- python/pyarrow/_parquet.pyx | 2 +- .../tests/parquet/test_external_encryption.py | 43 ++++++++----------- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx index bf3d0181e0c2..f89d2b07bd7a 100644 --- a/python/pyarrow/_parquet.pyx +++ b/python/pyarrow/_parquet.pyx @@ -1507,7 +1507,7 @@ cdef class ParquetReader(_Weakrefable): thrift_container_size_limit) if decryption_properties is not None: - if isinstance(decryption_properties, ExternalFileEncryptionProperties): + if isinstance(decryption_properties, ExternalFileDecryptionProperties): properties.file_decryption_properties( static_pointer_cast[CFileDecryptionProperties, CExternalFileDecryptionProperties] ( (decryption_properties).unwrap_external())) diff --git a/python/pyarrow/tests/parquet/test_external_encryption.py b/python/pyarrow/tests/parquet/test_external_encryption.py index aa66ac4ebe54..25e9d2e732e0 100644 --- a/python/pyarrow/tests/parquet/test_external_encryption.py +++ b/python/pyarrow/tests/parquet/test_external_encryption.py @@ -96,34 +96,24 @@ def external_decryption_config(): @pytest.fixture -def external_decryption_config(): - config = pe.ExternalDecryptionConfiguration( - cache_lifetime=timedelta(minutes=5.0), - app_context={ - "user_id": "Picard1701", - "location": "Presidio" - }, - connection_config={ - "AES_GCM_V1": { - "config_file": "path/to/config/file", - "config_file_decryption_key": "some_key" - } - } - ) - config.cache_lifetime=timedelta(minutes=5.0) - return config - -@pytest.fixture -def encrypted_parquet_file(tmp_path, dummy_kms_client, kms_config, external_decryption_config): +def encrypted_parquet_file(tmp_path, dummy_kms_client, kms_config, external_encryption_config): """ Fixture that creates a Parquet file encrypted with external encryption. Other tests can depend on this fixture. """ file_path = tmp_path / "test_ext.parquet" - table = pa.Table.from_arrays([[1, 2, 3]], names=["col"]) + + # Create table with columns matching the per_column_encryption keys in the config + table = pa.Table.from_arrays( + [ + [1, 2, 3], # column "a" + ["A", "B", "C"] # column "b" + ], + names=["a", "b"] + ) factory = pe.CryptoFactory(dummy_kms_client) - encryption_props = factory.external_file_encryption_properties(kms_config, external_decryption_config) + encryption_props = factory.external_file_encryption_properties(kms_config, external_encryption_config) writer = pq.ParquetWriter(str(file_path), table.schema, encryption_properties=encryption_props) writer.write_table(table) @@ -365,22 +355,27 @@ def test_external_file_decryption_properties_valid(dummy_kms_client, kms_config, assert result.__class__.__name__ == "ExternalFileDecryptionProperties" assert result.__class__.__module__ == "pyarrow._parquet" -def test_open_with_external_decryption(encrypted_parquet_file, dummy_kms_client, kms_config, external_encryption_config): +def test_open_with_external_decryption( + encrypted_parquet_file, dummy_kms_client, kms_config, external_decryption_config +): file_path, table = encrypted_parquet_file # Create decryption properties factory = pe.CryptoFactory(dummy_kms_client) - decryption_props = factory.external_file_decryption_properties(kms_config, external_encryption_config) + decryption_props = factory.external_file_decryption_properties( + kms_config, external_decryption_config + ) # Read the file reader = ParquetReader() reader.open(str(file_path), decryption_properties=decryption_props) read_table = reader.read_table() - # Assertions + # Assertions: check schema, number of rows and columns assert read_table.num_rows == table.num_rows assert read_table.num_columns == table.num_columns assert read_table.schema.equals(table.schema) + assert read_table.column_names == table.column_names reader.close() assert reader.closed From a858935bd53426ddac3a1622496c575f12863ce1 Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Tue, 19 Aug 2025 20:35:16 -0600 Subject: [PATCH 09/13] Fixed problems with bindings for ExternalDecryptionConfig. Also fixed external encryption Python test --- cpp/src/parquet/encryption/crypto_factory.cc | 13 +- python/pyarrow/_parquet_encryption.pyx | 17 +- .../tests/parquet/test_external_encryption.py | 363 ++++++++++-------- python/scripts/base_app.py | 14 + 4 files changed, 248 insertions(+), 159 deletions(-) diff --git a/cpp/src/parquet/encryption/crypto_factory.cc b/cpp/src/parquet/encryption/crypto_factory.cc index f13a58d610e3..7e5e08fd61b1 100644 --- a/cpp/src/parquet/encryption/crypto_factory.cc +++ b/cpp/src/parquet/encryption/crypto_factory.cc @@ -315,9 +315,9 @@ ColumnPathToEncryptionPropertiesMap CryptoFactory::GetColumnEncryptionProperties } std::shared_ptr CryptoFactory::GetFileDecryptionProperties( - const KmsConnectionConfig& kms_connection_config, - const DecryptionConfiguration& decryption_config, const std::string& file_path, - const std::shared_ptr<::arrow::fs::FileSystem>& file_system) { + const KmsConnectionConfig& kms_connection_config, + const DecryptionConfiguration& decryption_config, const std::string& file_path, + const std::shared_ptr<::arrow::fs::FileSystem>& file_system) { auto key_retriever = std::make_shared( key_toolkit_, kms_connection_config, decryption_config.cache_lifetime_seconds, file_path, file_system); @@ -330,10 +330,9 @@ std::shared_ptr CryptoFactory::GetFileDecryptionProper std::shared_ptr CryptoFactory::GetExternalFileDecryptionProperties( - const KmsConnectionConfig& kms_connection_config, - const ExternalDecryptionConfiguration& external_decryption_config, - const std::string& file_path, const std::shared_ptr<::arrow::fs::FileSystem>& file_system) { - + const KmsConnectionConfig& kms_connection_config, + const ExternalDecryptionConfiguration& external_decryption_config, + const std::string& file_path, const std::shared_ptr<::arrow::fs::FileSystem>& file_system) { // Use the same FileKeyUnwrapper as in the FileDecryptionProperties. // TODO(sbrenes): Check what needs to change in the case of external decryptors (Issue #52). auto key_retriever = std::make_shared( diff --git a/python/pyarrow/_parquet_encryption.pyx b/python/pyarrow/_parquet_encryption.pyx index c64419b3d059..d8adafa00767 100644 --- a/python/pyarrow/_parquet_encryption.pyx +++ b/python/pyarrow/_parquet_encryption.pyx @@ -419,13 +419,28 @@ cdef class ExternalDecryptionConfiguration(DecryptionConfiguration): __slots__ = () def __init__(self, *, cache_lifetime=None, app_context=None, connection_config=None): - super().__init__(cache_lifetime=cache_lifetime) + # Initialize the pointer first so the get/set forwards work. self.external_configuration.reset(new CExternalDecryptionConfiguration()) + super().__init__(cache_lifetime=cache_lifetime) + + self.external_configuration.get().cache_lifetime_seconds = \ + self.configuration.get().cache_lifetime_seconds if app_context is not None: self.app_context = app_context if connection_config is not None: self.connection_config = connection_config + + """ Forward all attributes get/set methods to the superclass """ + """ The superclass already converts to/from bytes and does additional processing needed """ + @property + def cache_lifetime(self): + return DecryptionConfiguration.cache_lifetime.__get__(self) + + @cache_lifetime.setter + def cache_lifetime(self, value): + DecryptionConfiguration.cache_lifetime.__set__(self, value) + self.external_configuration.get().cache_lifetime_seconds = value.total_seconds() @property def app_context(self): diff --git a/python/pyarrow/tests/parquet/test_external_encryption.py b/python/pyarrow/tests/parquet/test_external_encryption.py index 25e9d2e732e0..70607efe1275 100644 --- a/python/pyarrow/tests/parquet/test_external_encryption.py +++ b/python/pyarrow/tests/parquet/test_external_encryption.py @@ -15,60 +15,101 @@ # specific language governing permissions and limitations # under the License. +import base64 +import datetime +import pyarrow +import pyarrow.parquet as pp +import pyarrow.parquet.encryption as ppe import pytest -import pyarrow.parquet.encryption as pe import re -from datetime import timedelta - -@pytest.fixture -def dummy_kms_client(): - class DummyKmsClient(pe.KmsClient): - def __init__(self, kms_connection_config): - super().__init__() - - def wrap_key(self, key_bytes, master_key_identifier): - # dummy wrap just returns key_bytes - return key_bytes - - def unwrap_key(self, wrapped_key, master_key_identifier): - # dummy unwrap just returns wrapped_key - return wrapped_key - return DummyKmsClient - -@pytest.fixture -def kms_config(): - return pe.KmsConnectionConfig( + +class FooKmsClient(ppe.KmsClient): + + def __init__(self, kms_connection_config): + ppe.KmsClient.__init__(self) + self.master_keys_map = kms_connection_config.custom_kms_conf + + def wrap_key(self, key_bytes, master_key_identifier): + master_key_bytes = self.master_keys_map[master_key_identifier].encode('utf-8') + joint_key = b"".join([master_key_bytes, key_bytes]) + return base64.b64encode(joint_key) + + def unwrap_key(self, wrapped_key, master_key_identifier): + expected_master = self.master_keys_map[master_key_identifier] + decoded_key = base64.b64decode(wrapped_key) + master_key_bytes = decoded_key[:16] + decrypted_key = decoded_key[16:] + if (expected_master == master_key_bytes.decode('utf-8')): + return decrypted_key + raise ValueError(f"Bad master key used [{master_key_bytes}] - [{decrypted_key}]") + + +def get_data_table(): + sample_data = { + "orderId": [1001, 1002, 1003], + "productId": [152, 268, 6548], + "price": [3.25, 6.48, 2.12], + "vat": [0.0, 0.2, 0.05] + } + return pyarrow.Table.from_pydict(sample_data) + + +def kms_client_factory(kms_connection_config): + return FooKmsClient(kms_connection_config) + + +def get_kms_connection_config(): + return ppe.KmsConnectionConfig( custom_kms_conf={ "footer_key": "012footer_secret", - "key_1": "column_secret001", - "key_2": "column_secret002" + "orderid_key": "column_secret001", + "productid_key": "column_secret002" } ) -@pytest.fixture -def external_encryption_config(): - return pe.ExternalEncryptionConfiguration( - footer_key="footer_key", - column_keys={ - "key_1": ["a"], + +def get_encryption_config(): + return ppe.EncryptionConfiguration( + footer_key = "footer_key", + column_keys = { + "orderid_key": ["orderId"], + "productid_key": ["productId"] }, - encryption_algorithm="AES_GCM_V1", - plaintext_footer=True, - double_wrapping=True, - cache_lifetime=timedelta(minutes=5.0), - internal_key_material=True, - data_key_length_bits=256, - per_column_encryption={ - "b": { + encryption_algorithm = "AES_GCM_V1", + cache_lifetime=datetime.timedelta(minutes=2.0), + data_key_length_bits = 128, + plaintext_footer=True + ) + + +def get_encryption_properties(): + encryption_config = get_encryption_config() + crypto_factory = ppe.CryptoFactory(kms_client_factory) + return crypto_factory.file_encryption_properties( + get_kms_connection_config(), encryption_config) + + +def get_external_encryption_config(plaintext_footer=True): + return ppe.ExternalEncryptionConfiguration( + footer_key = "footer_key", + column_keys = { + "productid_key": ["productId"] + }, + encryption_algorithm = "AES_GCM_V1", + cache_lifetime=datetime.timedelta(minutes=2.0), + data_key_length_bits = 128, + plaintext_footer=plaintext_footer, + per_column_encryption = { + "orderId": { "encryption_algorithm": "AES_GCM_CTR_V1", - "encryption_key": "key_2" - } + "encryption_key": "orderid_key" + }, }, - app_context={ + app_context = { "user_id": "Picard1701", "location": "Presidio" }, - connection_config={ + connection_config = { "EXTERNAL_DBPA_V1": { "config_file": "path/to/config/file", "config_file_decryption_key": "some_key" @@ -76,95 +117,104 @@ def external_encryption_config(): } ) -@pytest.fixture -def external_decryption_config(): - config = pe.ExternalDecryptionConfiguration( - cache_lifetime=timedelta(minutes=5.0), - app_context={ + +def get_external_encryption_properties(): + encryption_config = get_external_encryption_config() + crypto_factory = ppe.CryptoFactory(kms_client_factory) + return crypto_factory.external_file_encryption_properties( + get_kms_connection_config(), encryption_config) + + +def get_decryption_config(): + return ppe.DecryptionConfiguration(cache_lifetime=datetime.timedelta(minutes=10.0)) + + +def get_decryption_properties(): + decryption_config = get_decryption_config() + crypto_factory = ppe.CryptoFactory(kms_client_factory) + return crypto_factory.file_decryption_properties( + get_kms_connection_config(), decryption_config) + + +def get_external_decryption_config(): + return ppe.ExternalDecryptionConfiguration( + cache_lifetime=datetime.timedelta(minutes=10.0), + app_context = { "user_id": "Picard1701", "location": "Presidio" }, - connection_config={ - "AES_GCM_V1": { + connection_config = { + "EXTERNAL_DBPA_V1": { "config_file": "path/to/config/file", "config_file_decryption_key": "some_key" } } ) - config.cache_lifetime=timedelta(minutes=5.0) - return config -@pytest.fixture -def encrypted_parquet_file(tmp_path, dummy_kms_client, kms_config, external_encryption_config): - """ - Fixture that creates a Parquet file encrypted with external encryption. - Other tests can depend on this fixture. - """ - file_path = tmp_path / "test_ext.parquet" - - # Create table with columns matching the per_column_encryption keys in the config - table = pa.Table.from_arrays( - [ - [1, 2, 3], # column "a" - ["A", "B", "C"] # column "b" - ], - names=["a", "b"] - ) +def get_external_decryption_properties(): + decryption_config = get_external_decryption_config() + crypto_factory = ppe.CryptoFactory(kms_client_factory) + return crypto_factory.external_file_decryption_properties( + get_kms_connection_config(), decryption_config) - factory = pe.CryptoFactory(dummy_kms_client) - encryption_props = factory.external_file_encryption_properties(kms_config, external_encryption_config) - writer = pq.ParquetWriter(str(file_path), table.schema, encryption_properties=encryption_props) +def write_parquet(table, location, encryption_properties): + writer = pp.ParquetWriter(location, table.schema, encryption_properties=encryption_properties) writer.write_table(table) - writer.close() - return file_path, table # Return the path and the original table for assertions + +def read_parquet(location, decryption_properties): + reader = pp.ParquetFile(location, decryption_properties=decryption_properties) + return reader.read() + def test_encryption_configuration_properties(): """Test the standard EncryptionConfiguration properties to avoid regressions.""" - config = pe.EncryptionConfiguration( + config = ppe.EncryptionConfiguration( footer_key="footer-key-name", column_keys={ "key_1": ["a"], }, - encryption_algorithm="AES_GCM_V1", + encryption_algorithm="EXTERNAL_DBPA_V1", plaintext_footer=True, double_wrapping=True, - cache_lifetime=timedelta(minutes=5.0), + cache_lifetime=datetime.timedelta(minutes=5.0), internal_key_material=True, data_key_length_bits=256 ) - assert isinstance(config, pe.EncryptionConfiguration) + assert isinstance(config, ppe.EncryptionConfiguration) assert config.footer_key == "footer-key-name" assert config.column_keys == { "key_1": ["a"] } - assert config.encryption_algorithm == "AES_GCM_V1" + assert config.encryption_algorithm == "EXTERNAL_DBPA_V1" assert config.plaintext_footer is True assert config.double_wrapping is True - assert config.cache_lifetime == timedelta(minutes=5.0) + assert config.cache_lifetime == datetime.timedelta(minutes=5.0) assert config.internal_key_material is True assert config.data_key_length_bits == 256 -def test_external_encryption_configuration_properties(external_encryption_config): + +def test_external_encryption_configuration_properties(): """Test the ExternalEncryptionConfiguration properties including external-specific fields.""" - assert isinstance(external_encryption_config, pe.ExternalEncryptionConfiguration) + external_encryption_config = get_external_encryption_config() + assert isinstance(external_encryption_config, ppe.ExternalEncryptionConfiguration) assert external_encryption_config.footer_key == "footer_key" assert external_encryption_config.column_keys == { - "key_1": ["a"] + "productid_key": ["productId"] } assert external_encryption_config.encryption_algorithm == "AES_GCM_V1" assert external_encryption_config.plaintext_footer is True assert external_encryption_config.double_wrapping is True - assert external_encryption_config.cache_lifetime == timedelta(minutes=5.0) + assert external_encryption_config.cache_lifetime == datetime.timedelta(minutes=2.0) assert external_encryption_config.internal_key_material is True - assert external_encryption_config.data_key_length_bits == 256 + assert external_encryption_config.data_key_length_bits == 128 assert external_encryption_config.app_context == { "user_id": "Picard1701", @@ -179,25 +229,27 @@ def test_external_encryption_configuration_properties(external_encryption_config } assert external_encryption_config.per_column_encryption == { - "b": { + "orderId": { "encryption_algorithm": "AES_GCM_CTR_V1", - "encryption_key": "key_2" - } + "encryption_key": "orderid_key" + }, } + def test_external_encryption_app_context_invalid_json(): """Ensure app_context raises TypeError for non-JSON-serializable input.""" with pytest.raises(TypeError, match="Failed to serialize app_context: {'invalid': {1, 2, 3}}"): - pe.ExternalEncryptionConfiguration( + ppe.ExternalEncryptionConfiguration( footer_key="key", app_context={"invalid": set([1, 2, 3])} # sets are not JSON-serializable ) + def test_external_encryption_per_column_encryption_invalid_algorithm(): """Ensure invalid encryption_algorithm raises a ValueError or is rejected.""" with pytest.raises(ValueError, match="Invalid cipher name: INVALID"): - pe.ExternalEncryptionConfiguration( + ppe.ExternalEncryptionConfiguration( footer_key="key", per_column_encryption={ "a": { @@ -207,10 +259,11 @@ def test_external_encryption_per_column_encryption_invalid_algorithm(): } ) + def test_external_encryption_per_column_encryption_new_algorithm(): """Ensure new encryption_algorithm is accepted.""" - pe.ExternalEncryptionConfiguration( + ppe.ExternalEncryptionConfiguration( footer_key="key", per_column_encryption={ "a": { @@ -220,10 +273,11 @@ def test_external_encryption_per_column_encryption_new_algorithm(): } ) + def test_external_encryption_connection_config_invalid_types(): """Ensure connection_config rejects non-string keys or values.""" with pytest.raises(TypeError, match="All inner config keys/values must be str"): - config=pe.ExternalEncryptionConfiguration( + config=ppe.ExternalEncryptionConfiguration( footer_key="key" ) config.connection_config={ @@ -234,7 +288,7 @@ def test_external_encryption_connection_config_invalid_types(): } with pytest.raises(TypeError, match="All inner config keys/values must be str"): - config = pe.ExternalEncryptionConfiguration( + config = ppe.ExternalEncryptionConfiguration( footer_key="key" ) config.connection_config={ @@ -244,7 +298,8 @@ def test_external_encryption_connection_config_invalid_types(): } def test_external_encryption_rejects_none_values(): - config = pe.ExternalEncryptionConfiguration(footer_key="key") + """Ensure None values are rejected.""" + config = ppe.ExternalEncryptionConfiguration(footer_key="key") # per_column_encryption: expect ValueError with pytest.raises(TypeError, match="per_column_encryption cannot be None"): @@ -258,64 +313,64 @@ def test_external_encryption_rejects_none_values(): with pytest.raises(ValueError, match="Connection config value cannot be None"): config.connection_config = None -def test_external_file_encryption_properties_rejects_column_in_two_places( - dummy_kms_client, kms_config): - config = pe.ExternalEncryptionConfiguration( - footer_key="key", - column_keys={"key_1": ["a"]}, - per_column_encryption={"a": {"encryption_algorithm": "AES_GCM_V1", "encryption_key": "key_2"}}, + +def test_external_file_encryption_properties_rejects_column_in_two_places(): + """Ensure a column cannot be defined in both column_keys and per_column_encryption.""" + config = ppe.ExternalEncryptionConfiguration( + footer_key="footer_key", + column_keys={"orderid_key": ["a"]}, + per_column_encryption={"a": { + "encryption_algorithm": "AES_GCM_V1", + "encryption_key": "key_2" + }}, ) - factory = pe.CryptoFactory(dummy_kms_client) - with pytest.raises(OSError, match=re.escape("Multiple keys defined for column [a]")): - factory.external_file_encryption_properties(kms_config, config) + factory = ppe.CryptoFactory(kms_client_factory) + with pytest.raises(OSError, match=re.escape("Multiple keys defined for column [a]")): + factory.external_file_encryption_properties(get_kms_connection_config(), config) -def test_external_file_encryption_properties_valid( - dummy_kms_client, kms_config, external_encryption_config): - factory = pe.CryptoFactory(dummy_kms_client) - result = factory.external_file_encryption_properties(kms_config, external_encryption_config) - # Instead of isinstance, check class name and module dynamically because - # ExternalFileEncryptionProperties is not visible - assert result.__class__.__name__ == "ExternalFileEncryptionProperties" - assert result.__class__.__module__ == "pyarrow._parquet" +def test_external_file_encryption_properties_valid(): + """Check class name and module because ExternalFileEncryptionProperties is not visible.""" + external_encryption_properties = get_external_encryption_properties() -def test_write_table_with_external_encryption(encrypted_parquet_file): - file_path, table = encrypted_parquet_file + assert external_encryption_properties.__class__.__name__ == "ExternalFileEncryptionProperties" + assert external_encryption_properties.__class__.__module__ == "pyarrow._parquet" - # Check the file exists - assert file_path.exists() def test_decryption_configuration_properties(): """Test the standard DecryptionConfiguration properties to avoid regressions.""" - config = pe.DecryptionConfiguration() - config.cache_lifetime=timedelta(minutes=5.0) + config = ppe.DecryptionConfiguration() + config.cache_lifetime=datetime.timedelta(minutes=5.0) - assert isinstance(config, pe.DecryptionConfiguration) - assert config.cache_lifetime == timedelta(minutes=5.0) + assert isinstance(config, ppe.DecryptionConfiguration) + assert config.cache_lifetime == datetime.timedelta(minutes=5.0) -def test_external_decryption_configuration_properties(external_decryption_config): + +def test_external_decryption_configuration_properties(): """Test the ExternalDecryptionConfiguration properties including external-specific fields.""" - assert isinstance(external_decryption_config, pe.ExternalDecryptionConfiguration) - assert external_decryption_config.cache_lifetime == timedelta(minutes=5.0) + external_decryption_config = get_external_decryption_config() + assert isinstance(external_decryption_config, ppe.ExternalDecryptionConfiguration) + assert external_decryption_config.cache_lifetime == datetime.timedelta(minutes=10.0) assert external_decryption_config.app_context == { "user_id": "Picard1701", "location": "Presidio" } assert external_decryption_config.connection_config == { - "AES_GCM_V1": { + "EXTERNAL_DBPA_V1": { "config_file": "path/to/config/file", "config_file_decryption_key": "some_key" } } - + + def test_external_decryption_connection_config_invalid_types(): """Ensure connection_config rejects non-string keys or values.""" # Outer key is not a string (int instead of cipher name string) with pytest.raises(AttributeError, match="'int' object has no attribute 'upper'"): - config = pe.ExternalDecryptionConfiguration() + config = ppe.ExternalDecryptionConfiguration() config.connection_config = { 123: { # invalid outer key "config_file": "should-fail" @@ -324,14 +379,14 @@ def test_external_decryption_connection_config_invalid_types(): # Outer value is not a dict with pytest.raises(TypeError, match="Inner value for cipher AES_GCM_V1 must be a dict"): - config = pe.ExternalDecryptionConfiguration() + config = ppe.ExternalDecryptionConfiguration() config.connection_config = { "AES_GCM_V1": ["not", "a", "dict"] # invalid outer value (should be dict) } # Inner key is not a string with pytest.raises(TypeError, match="All inner config keys/values must be str"): - config = pe.ExternalDecryptionConfiguration() + config = ppe.ExternalDecryptionConfiguration() config.connection_config = { "AES_GCM_V1": { 123: "should-fail" # invalid inner key @@ -340,42 +395,48 @@ def test_external_decryption_connection_config_invalid_types(): # Inner value is not a string with pytest.raises(TypeError, match="All inner config keys/values must be str"): - config = pe.ExternalDecryptionConfiguration() + config = ppe.ExternalDecryptionConfiguration() config.connection_config = { "AES_GCM_V1": { "config_file": ["not", "a", "string"] # invalid inner value } } -def test_external_file_decryption_properties_valid(dummy_kms_client, kms_config, external_decryption_config): - factory = pe.CryptoFactory(dummy_kms_client) - result = factory.external_file_decryption_properties(kms_config, external_decryption_config) - # Instead of isinstance, check class name and module dynamically because ExternalFileDecryptionProperties is not visbile - assert result.__class__.__name__ == "ExternalFileDecryptionProperties" - assert result.__class__.__module__ == "pyarrow._parquet" +def test_external_file_decryption_properties_valid(): + """Check class name and module because ExternalFileDecryptionProperties is not visible.""" -def test_open_with_external_decryption( - encrypted_parquet_file, dummy_kms_client, kms_config, external_decryption_config -): - file_path, table = encrypted_parquet_file + external_decryption_properties = get_external_decryption_properties() + + assert external_decryption_properties.__class__.__name__ == "ExternalFileDecryptionProperties" + assert external_decryption_properties.__class__.__module__ == "pyarrow._parquet" + + +def test_read_and_write_standard_encryption(tmp_path): + """ Test a roundtrip encryption and decryption using standard encryption. """ + + data_table = get_data_table() + parquet_path = tmp_path / "test.parquet" + write_parquet(data_table, parquet_path, get_encryption_properties()) + + read_data_table = read_parquet(parquet_path, get_decryption_properties()) + assert read_data_table.equals(data_table) + assert read_data_table.num_rows == data_table.num_rows + assert read_data_table.num_columns == data_table.num_columns + assert read_data_table.schema.equals(data_table.schema) + assert read_data_table.column_names == data_table.column_names - # Create decryption properties - factory = pe.CryptoFactory(dummy_kms_client) - decryption_props = factory.external_file_decryption_properties( - kms_config, external_decryption_config - ) - # Read the file - reader = ParquetReader() - reader.open(str(file_path), decryption_properties=decryption_props) - read_table = reader.read_table() +def test_read_and_write_external_encryption(tmp_path): + """ Test a roundtrip encryption and decryption using external encryption. """ - # Assertions: check schema, number of rows and columns - assert read_table.num_rows == table.num_rows - assert read_table.num_columns == table.num_columns - assert read_table.schema.equals(table.schema) - assert read_table.column_names == table.column_names + data_table = get_data_table() + parquet_path = tmp_path / "test.parquet" + write_parquet(data_table, parquet_path, get_external_encryption_properties()) - reader.close() - assert reader.closed + read_data_table = read_parquet(parquet_path, get_external_decryption_properties()) + assert read_data_table.equals(data_table) + assert read_data_table.num_rows == data_table.num_rows + assert read_data_table.num_columns == data_table.num_columns + assert read_data_table.schema.equals(data_table.schema) + assert read_data_table.column_names == data_table.column_names diff --git a/python/scripts/base_app.py b/python/scripts/base_app.py index 59858f25eb21..c95dd12d5e29 100644 --- a/python/scripts/base_app.py +++ b/python/scripts/base_app.py @@ -158,6 +158,20 @@ def get_encryption_config(plaintext_footer=True): def get_decryption_config(): return ppe.DecryptionConfiguration(cache_lifetime=datetime.timedelta(minutes=2.0)) +def get_external_decryption_config(): + return ppe.ExternalDecryptionConfiguration( + cache_lifetime=datetime.timedelta(minutes=2.0), + app_context = { + "user_id": "Picard1701", + "location": "Presidio" + }, + connection_config = { + "EXTERNAL_DBPA_V1": { + "config_file": "path/to/config/file", + "config_file_decryption_key": "some_key" + } + } +) if __name__ == "__main__": create_and_encrypt_parquet() From 03bccda5c2747e7eb8f8aa0fd48629f1ede4a634 Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Tue, 19 Aug 2025 20:54:23 -0600 Subject: [PATCH 10/13] restoring crypto_factory --- cpp/src/parquet/encryption/crypto_factory.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cpp/src/parquet/encryption/crypto_factory.cc b/cpp/src/parquet/encryption/crypto_factory.cc index 7e5e08fd61b1..837d8c0377e4 100644 --- a/cpp/src/parquet/encryption/crypto_factory.cc +++ b/cpp/src/parquet/encryption/crypto_factory.cc @@ -315,9 +315,9 @@ ColumnPathToEncryptionPropertiesMap CryptoFactory::GetColumnEncryptionProperties } std::shared_ptr CryptoFactory::GetFileDecryptionProperties( - const KmsConnectionConfig& kms_connection_config, - const DecryptionConfiguration& decryption_config, const std::string& file_path, - const std::shared_ptr<::arrow::fs::FileSystem>& file_system) { + const KmsConnectionConfig& kms_connection_config, + const DecryptionConfiguration& decryption_config, const std::string& file_path, + const std::shared_ptr<::arrow::fs::FileSystem>& file_system) { auto key_retriever = std::make_shared( key_toolkit_, kms_connection_config, decryption_config.cache_lifetime_seconds, file_path, file_system); @@ -330,9 +330,9 @@ std::shared_ptr CryptoFactory::GetFileDecryptionProper std::shared_ptr CryptoFactory::GetExternalFileDecryptionProperties( - const KmsConnectionConfig& kms_connection_config, - const ExternalDecryptionConfiguration& external_decryption_config, - const std::string& file_path, const std::shared_ptr<::arrow::fs::FileSystem>& file_system) { + const KmsConnectionConfig& kms_connection_config, + const ExternalDecryptionConfiguration& external_decryption_config, + const std::string& file_path, const std::shared_ptr<::arrow::fs::FileSystem>& file_system) { // Use the same FileKeyUnwrapper as in the FileDecryptionProperties. // TODO(sbrenes): Check what needs to change in the case of external decryptors (Issue #52). auto key_retriever = std::make_shared( From bf9a0eaba9cb131d33004c4d15effb5138b7e77b Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Tue, 19 Aug 2025 20:54:56 -0600 Subject: [PATCH 11/13] restoring crypto_factory --- cpp/src/parquet/encryption/crypto_factory.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/src/parquet/encryption/crypto_factory.cc b/cpp/src/parquet/encryption/crypto_factory.cc index 837d8c0377e4..c8bfa0b07787 100644 --- a/cpp/src/parquet/encryption/crypto_factory.cc +++ b/cpp/src/parquet/encryption/crypto_factory.cc @@ -333,6 +333,7 @@ CryptoFactory::GetExternalFileDecryptionProperties( const KmsConnectionConfig& kms_connection_config, const ExternalDecryptionConfiguration& external_decryption_config, const std::string& file_path, const std::shared_ptr<::arrow::fs::FileSystem>& file_system) { + // Use the same FileKeyUnwrapper as in the FileDecryptionProperties. // TODO(sbrenes): Check what needs to change in the case of external decryptors (Issue #52). auto key_retriever = std::make_shared( From 2269958fd890a0276ec8aa00c09c716d12273bf7 Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Tue, 19 Aug 2025 20:55:25 -0600 Subject: [PATCH 12/13] restoring crypto_factory --- cpp/src/parquet/encryption/crypto_factory.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/parquet/encryption/crypto_factory.cc b/cpp/src/parquet/encryption/crypto_factory.cc index c8bfa0b07787..f13a58d610e3 100644 --- a/cpp/src/parquet/encryption/crypto_factory.cc +++ b/cpp/src/parquet/encryption/crypto_factory.cc @@ -333,7 +333,7 @@ CryptoFactory::GetExternalFileDecryptionProperties( const KmsConnectionConfig& kms_connection_config, const ExternalDecryptionConfiguration& external_decryption_config, const std::string& file_path, const std::shared_ptr<::arrow::fs::FileSystem>& file_system) { - + // Use the same FileKeyUnwrapper as in the FileDecryptionProperties. // TODO(sbrenes): Check what needs to change in the case of external decryptors (Issue #52). auto key_retriever = std::make_shared( From eb30bd9a23c4ff4b94ff32e15821d41ace8ab4de Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Wed, 20 Aug 2025 09:06:01 -0600 Subject: [PATCH 13/13] Added comment --- python/pyarrow/_parquet_encryption.pyx | 1 + 1 file changed, 1 insertion(+) diff --git a/python/pyarrow/_parquet_encryption.pyx b/python/pyarrow/_parquet_encryption.pyx index d8adafa00767..88fc908e9558 100644 --- a/python/pyarrow/_parquet_encryption.pyx +++ b/python/pyarrow/_parquet_encryption.pyx @@ -420,6 +420,7 @@ cdef class ExternalDecryptionConfiguration(DecryptionConfiguration): def __init__(self, *, cache_lifetime=None, app_context=None, connection_config=None): # Initialize the pointer first so the get/set forwards work. + # Super init will run the setters/getters below so we need the pointer to exist. self.external_configuration.reset(new CExternalDecryptionConfiguration()) super().__init__(cache_lifetime=cache_lifetime)