From 26d1086a763123b1f5f184bea4a8f2d9f541d939 Mon Sep 17 00:00:00 2001 From: Marco Arguedas Date: Wed, 7 Jan 2026 12:12:31 -0600 Subject: [PATCH 1/2] Ensuring that a bool column has encryption (for testing purposes) --- python/scripts/base_app.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/python/scripts/base_app.py b/python/scripts/base_app.py index a8858ca9e59e..c302772e6924 100644 --- a/python/scripts/base_app.py +++ b/python/scripts/base_app.py @@ -48,7 +48,11 @@ def write_parquet(table, location, encryption_config=None): # Change scenario ID to test different cases. # https://github.com/protegrity/arrow/issues/204 for more details. - scenario_id = 5 + scenario_id_raw = os.getenv("BASE_APP_SCENARIO_ID", "5") + try: + scenario_id = int(scenario_id_raw) + except ValueError as exc: + raise ValueError(f"Invalid SCENARIO_ID: {scenario_id_raw!r} (must be an integer)") from exc match scenario_id: case 1: @@ -254,11 +258,11 @@ def get_external_encryption_config(plaintext_footer=True): "customer_name": { "encryption_algorithm": "EXTERNAL_DBPA_V1", "encryption_key": "customer_key" - }#, intentionally left out to test per-column encryption for the 'has_subscription' column. - # "has_subscription": { - # "encryption_algorithm": "EXTERNAL_DBPA_V1", - # "encryption_key": "has_subscription_key" - # } + }, + "has_subscription": { + "encryption_algorithm": "EXTERNAL_DBPA_V1", + "encryption_key": "has_subscription_key" + } }, app_context = { "user_id": "Picard1701", From 28341e2932924616d4f3d2b6d2bad4cc99e64117 Mon Sep 17 00:00:00 2001 From: Marco Arguedas Date: Wed, 7 Jan 2026 13:02:48 -0600 Subject: [PATCH 2/2] Renaming 'connection_config' to 'configuration_properties' (C++, Python, Cython code) --- .../parquet/arrow/arrow_reader_writer_test.cc | 8 +- cpp/src/parquet/column_writer_test.cc | 8 +- cpp/src/parquet/encryption/crypto_factory.cc | 22 ++--- cpp/src/parquet/encryption/crypto_factory.h | 4 +- .../parquet/encryption/crypto_factory_test.cc | 38 ++++----- cpp/src/parquet/encryption/encryption.cc | 28 +++---- cpp/src/parquet/encryption/encryption.h | 26 +++--- .../encryption/external/dbpa_executor.cc | 12 +-- .../encryption/external/dbpa_executor.h | 2 +- .../encryption/external/dbpa_executor_test.cc | 10 +-- .../external/dbpa_library_wrapper.h | 4 +- .../external/dbpa_library_wrapper_test.cc | 14 ++-- .../encryption/external/dbpa_test_agent.cc | 4 +- .../encryption/external/dbpa_test_agent.h | 6 +- .../encryption/external_dbpa_encryption.cc | 80 +++++++++---------- .../encryption/external_dbpa_encryption.h | 12 +-- cpp/src/parquet/encryption/properties_test.cc | 22 ++--- cpp/src/parquet/file_deserialize_test.cc | 4 +- python/pyarrow/_parquet_encryption.pyx | 46 +++++------ .../includes/libparquet_encryption.pxd | 4 +- .../tests/parquet/test_external_encryption.py | 36 ++++----- python/scripts/base_app.py | 14 ++-- .../scripts/use_external_dbpa_encryption.py | 22 ++--- 23 files changed, 213 insertions(+), 213 deletions(-) diff --git a/cpp/src/parquet/arrow/arrow_reader_writer_test.cc b/cpp/src/parquet/arrow/arrow_reader_writer_test.cc index 8e2e99303f46..3814859c1c99 100644 --- a/cpp/src/parquet/arrow/arrow_reader_writer_test.cc +++ b/cpp/src/parquet/arrow/arrow_reader_writer_test.cc @@ -1990,8 +1990,8 @@ TEST(ExternalDbpaConcurrencyTest, FailsWhenUseThreadsTrue) { parquet::ExternalFileEncryptionProperties::Builder fep_builder(footer_key); fep_builder.footer_key_metadata("kf") ->encrypted_columns(enc_cols) - ->algorithm(parquet::ParquetCipher::AES_GCM_V1) - ->connection_config({{parquet::ParquetCipher::EXTERNAL_DBPA_V1, + ->algorithm(parquet::ParquetCipher::AES_GCM_V1) + ->configuration_properties({{parquet::ParquetCipher::EXTERNAL_DBPA_V1, {{"agent_library_path", lib_path}, {"file_path", "/tmp/test"}}}}); @@ -2010,8 +2010,8 @@ TEST(ExternalDbpaConcurrencyTest, FailsWhenUseThreadsTrue) { parquet::ExternalFileDecryptionProperties::Builder dep_builder; dep_builder.key_retriever(kr) - ->app_context("{}") - ->connection_config({{parquet::ParquetCipher::EXTERNAL_DBPA_V1, + ->app_context("{}") + ->configuration_properties({{parquet::ParquetCipher::EXTERNAL_DBPA_V1, {{"agent_library_path", lib_path}, {"file_path", "/tmp/test"}}}}); parquet::ReaderProperties rp = parquet::default_reader_properties(); diff --git a/cpp/src/parquet/column_writer_test.cc b/cpp/src/parquet/column_writer_test.cc index e301370206c2..0e8ae2f17b09 100644 --- a/cpp/src/parquet/column_writer_test.cc +++ b/cpp/src/parquet/column_writer_test.cc @@ -1989,7 +1989,7 @@ TEST_F(TestColumnWriterEncryption, ExternalDBPAEncryption) { ->set_plaintext_footer() ->algorithm(ParquetCipher::AES_GCM_V1) ->app_context(app_context_) - ->connection_config({{ParquetCipher::EXTERNAL_DBPA_V1, connection_config_}}); + ->configuration_properties({{ParquetCipher::EXTERNAL_DBPA_V1, connection_config_}}); auto file_encryption_properties = fep_builder.build_external(); auto writer_properties_builder = WriterProperties::Builder(); @@ -2017,7 +2017,7 @@ TEST_F(TestColumnWriterEncryption, ExternalDBPAEncryption) { decryption_properties_builder.footer_key(kFooterEncryptionKey_) ->column_keys(decryption_columns) ->app_context(app_context_) - ->connection_config({{ParquetCipher::EXTERNAL_DBPA_V1, connection_config_}}); + ->configuration_properties({{ParquetCipher::EXTERNAL_DBPA_V1, connection_config_}}); reader_properties.file_decryption_properties(decryption_properties_builder.build_external()); auto file_reader = ParquetFileReader::Open( @@ -2062,7 +2062,7 @@ TEST_F(TestColumnWriterEncryption, ExternalDBPAEncryption_MultiplePagesNoConflic ->set_plaintext_footer() ->algorithm(ParquetCipher::AES_GCM_V1) ->app_context(app_context_) - ->connection_config({{ParquetCipher::EXTERNAL_DBPA_V1, connection_config_}}); + ->configuration_properties({{ParquetCipher::EXTERNAL_DBPA_V1, connection_config_}}); auto file_encryption_properties = fep_builder.build_external(); auto writer_properties_builder = WriterProperties::Builder(); @@ -2113,7 +2113,7 @@ TEST_F(TestColumnWriterEncryption, ExternalDBPAEncryption_ConflictingMetadataThr ->set_plaintext_footer() ->algorithm(ParquetCipher::AES_GCM_V1) ->app_context(app_context_) - ->connection_config({{ParquetCipher::EXTERNAL_DBPA_V1, { + ->configuration_properties({{ParquetCipher::EXTERNAL_DBPA_V1, { {"dbpa_test_force_conflicting_metadata", "1"}, {"agent_library_path", library_path_} }}}); diff --git a/cpp/src/parquet/encryption/crypto_factory.cc b/cpp/src/parquet/encryption/crypto_factory.cc index 4e07a5eb7bd2..ae7f0b54a39b 100644 --- a/cpp/src/parquet/encryption/crypto_factory.cc +++ b/cpp/src/parquet/encryption/crypto_factory.cc @@ -73,13 +73,13 @@ int ValidateAndGetKeyLength(int32_t dek_length_bits) { return dek_length_bits / 8; } -std::map> ConvertConnectionConfig( +std::map> ConvertConfigurationProperties( const std::unordered_map>& connection_config) { + std::unordered_map>& configuration_properties) { std::map> converted_config; - for (const auto& [cipher_type, inner_config] : connection_config) { + for (const auto& [cipher_type, inner_config] : configuration_properties) { if (!IsParquetCipherSupported(cipher_type)) { throw ParquetException("Invalid ParquetCipher type: " + std::to_string(static_cast(cipher_type))); @@ -88,10 +88,10 @@ std::map> ConvertConnect std::map converted_inner; for (const auto& [key, value] : inner_config) { if (key.empty()) { - throw ParquetException("Empty key in connection config"); + throw ParquetException("Empty key in configuration properties"); } if (value.empty()) { - throw ParquetException("Empty value for key '" + key + "' in connection config"); + throw ParquetException("Empty value for key '" + key + "' in configuration properties"); } converted_inner[key] = value; } @@ -239,9 +239,9 @@ CryptoFactory::GetExternalFileEncryptionProperties( external_properties_builder.app_context(external_encryption_config.app_context); } - if (!external_encryption_config.connection_config.empty()) { - external_properties_builder.connection_config(ConvertConnectionConfig( - external_encryption_config.connection_config)); + if (!external_encryption_config.configuration_properties.empty()) { + external_properties_builder.configuration_properties(ConvertConfigurationProperties( + external_encryption_config.configuration_properties)); } if (key_material_store != nullptr) { @@ -353,9 +353,9 @@ CryptoFactory::GetExternalFileDecryptionProperties( builder.app_context(external_decryption_config.app_context); } - if (!external_decryption_config.connection_config.empty()) { - builder.connection_config(ConvertConnectionConfig( - external_decryption_config.connection_config)); + if (!external_decryption_config.configuration_properties.empty()) { + builder.configuration_properties(ConvertConfigurationProperties( + external_decryption_config.configuration_properties)); } return builder.build_external(); diff --git a/cpp/src/parquet/encryption/crypto_factory.h b/cpp/src/parquet/encryption/crypto_factory.h index ff12eccdfb30..61945115c86c 100644 --- a/cpp/src/parquet/encryption/crypto_factory.h +++ b/cpp/src/parquet/encryption/crypto_factory.h @@ -136,7 +136,7 @@ std::string app_context; /// For security, these values should never be sent in this config, only the locations of /// the files that the external encryptor will know how to access. std::unordered_map> - connection_config; + configuration_properties; }; struct PARQUET_EXPORT DecryptionConfiguration { @@ -163,7 +163,7 @@ struct PARQUET_EXPORT ExternalDecryptionConfiguration : public DecryptionConfigu /// For security, these values should never be sent in this config, only the locations of /// the files that the external decryptor will know how to access. std::unordered_map> - connection_config; + configuration_properties; }; /// This is a core class, that translates the parameters of high level encryption (like diff --git a/cpp/src/parquet/encryption/crypto_factory_test.cc b/cpp/src/parquet/encryption/crypto_factory_test.cc index c738684e8f87..11b1a4d9ea84 100644 --- a/cpp/src/parquet/encryption/crypto_factory_test.cc +++ b/cpp/src/parquet/encryption/crypto_factory_test.cc @@ -142,7 +142,7 @@ TEST_F(CryptoFactoryTest, ExternalEncryptionConfig) { config.per_column_encryption = per_column_encryption; config.app_context = "{\"user_id\": \"abc123\", \"location\": {\"lat\": 9.7489, \"lon\": -83.7534}}"; - config.connection_config = { + config.configuration_properties = { {ParquetCipher::EXTERNAL_DBPA_V1, {{"file_path", "path/to/file"}}} }; @@ -177,15 +177,15 @@ TEST_F(CryptoFactoryTest, ExternalEncryptionConfig) { EXPECT_FALSE(column_properties_3->parquet_cipher().has_value()); EXPECT_FALSE(properties->app_context().empty()); - EXPECT_FALSE(properties->connection_config().empty()); + EXPECT_FALSE(properties->configuration_properties().empty()); EXPECT_EQ(properties->app_context(), config.app_context); - EXPECT_NE(properties->connection_config().find(ParquetCipher::EXTERNAL_DBPA_V1), - properties->connection_config().end()); - EXPECT_EQ(properties->connection_config().find(ParquetCipher::AES_GCM_CTR_V1), - properties->connection_config().end()); - EXPECT_NE(properties->connection_config().at(ParquetCipher::EXTERNAL_DBPA_V1).find("file_path"), - properties->connection_config().at(ParquetCipher::EXTERNAL_DBPA_V1).end()); - EXPECT_EQ(properties->connection_config().at(ParquetCipher::EXTERNAL_DBPA_V1).at("file_path"), + EXPECT_NE(properties->configuration_properties().find(ParquetCipher::EXTERNAL_DBPA_V1), + properties->configuration_properties().end()); + EXPECT_EQ(properties->configuration_properties().find(ParquetCipher::AES_GCM_CTR_V1), + properties->configuration_properties().end()); + EXPECT_NE(properties->configuration_properties().at(ParquetCipher::EXTERNAL_DBPA_V1).find("file_path"), + properties->configuration_properties().at(ParquetCipher::EXTERNAL_DBPA_V1).end()); + EXPECT_EQ(properties->configuration_properties().at(ParquetCipher::EXTERNAL_DBPA_V1).at("file_path"), "path/to/file"); } @@ -244,7 +244,7 @@ TEST_F(CryptoFactoryTest, BasicDecryptionConfig) { EXPECT_TRUE(properties->plaintext_files_allowed()); EXPECT_THAT(properties->key_retriever(), testing::NotNull()); EXPECT_TRUE(properties->app_context().empty()); - EXPECT_TRUE(properties->connection_config().empty()); + EXPECT_TRUE(properties->configuration_properties().empty()); } TEST_F(CryptoFactoryTest, ExternalDecryptionConfig) { @@ -252,7 +252,7 @@ TEST_F(CryptoFactoryTest, ExternalDecryptionConfig) { config.cache_lifetime_seconds = 600; config.app_context = "{\"user_id\": \"abc123\", \"location\": {\"lat\": 9.7489, \"lon\": -83.7534}}"; - config.connection_config = { + config.configuration_properties = { {ParquetCipher::EXTERNAL_DBPA_V1, {{"file_path", "path/to/file"}}} }; @@ -261,15 +261,15 @@ TEST_F(CryptoFactoryTest, ExternalDecryptionConfig) { EXPECT_TRUE(properties->plaintext_files_allowed()); EXPECT_THAT(properties->key_retriever(), testing::NotNull()); EXPECT_FALSE(properties->app_context().empty()); - EXPECT_FALSE(properties->connection_config().empty()); + EXPECT_FALSE(properties->configuration_properties().empty()); EXPECT_EQ(properties->app_context(), config.app_context); - EXPECT_NE(properties->connection_config().find(ParquetCipher::EXTERNAL_DBPA_V1), - properties->connection_config().end()); - EXPECT_EQ(properties->connection_config().find(ParquetCipher::AES_GCM_CTR_V1), - properties->connection_config().end()); - EXPECT_NE(properties->connection_config().at(ParquetCipher::EXTERNAL_DBPA_V1).find("file_path"), - properties->connection_config().at(ParquetCipher::EXTERNAL_DBPA_V1).end()); - EXPECT_EQ(properties->connection_config().at(ParquetCipher::EXTERNAL_DBPA_V1).at("file_path"), + EXPECT_NE(properties->configuration_properties().find(ParquetCipher::EXTERNAL_DBPA_V1), + properties->configuration_properties().end()); + EXPECT_EQ(properties->configuration_properties().find(ParquetCipher::AES_GCM_CTR_V1), + properties->configuration_properties().end()); + EXPECT_NE(properties->configuration_properties().at(ParquetCipher::EXTERNAL_DBPA_V1).find("file_path"), + properties->configuration_properties().at(ParquetCipher::EXTERNAL_DBPA_V1).end()); + EXPECT_EQ(properties->configuration_properties().at(ParquetCipher::EXTERNAL_DBPA_V1).at("file_path"), "path/to/file"); } diff --git a/cpp/src/parquet/encryption/encryption.cc b/cpp/src/parquet/encryption/encryption.cc index 6ee8a6db932f..b8a07d87f3ee 100644 --- a/cpp/src/parquet/encryption/encryption.cc +++ b/cpp/src/parquet/encryption/encryption.cc @@ -160,16 +160,16 @@ ExternalFileDecryptionProperties::Builder* ExternalFileDecryptionProperties::Bui } ExternalFileDecryptionProperties::Builder* -ExternalFileDecryptionProperties::Builder::connection_config( +ExternalFileDecryptionProperties::Builder::configuration_properties( std::map> config) { - if (connection_config_.size() != 0) { - throw ParquetException("Connection config already set"); + if (configuration_properties_.size() != 0) { + throw ParquetException("Configuration properties already set"); } if (config.size() == 0) { return this; } - connection_config_ = std::move(config); + configuration_properties_ = std::move(config); return this; } @@ -178,7 +178,7 @@ ExternalFileDecryptionProperties::Builder::build_external() { return std::shared_ptr(new ExternalFileDecryptionProperties( footer_key_, key_retriever_, check_plaintext_footer_integrity_, aad_prefix_, aad_prefix_verifier_, column_decryption_properties_, plaintext_files_allowed_, - app_context_, connection_config_)); + app_context_, configuration_properties_)); } ExternalFileDecryptionProperties::ExternalFileDecryptionProperties( @@ -189,12 +189,12 @@ ExternalFileDecryptionProperties::ExternalFileDecryptionProperties( ColumnPathToDecryptionPropertiesMap column_decryption_properties, bool plaintext_files_allowed, std::string app_context, - std::map> connection_config) + std::map> configuration_properties) : FileDecryptionProperties(footer_key, key_retriever, check_plaintext_footer_integrity, aad_prefix, aad_prefix_verifier, column_decryption_properties, plaintext_files_allowed), app_context_(app_context), - connection_config_(connection_config) {} + configuration_properties_(configuration_properties) {} ColumnDecryptionProperties::Builder* ColumnDecryptionProperties::Builder::key( SecureString key) { @@ -404,17 +404,17 @@ ExternalFileEncryptionProperties::Builder* ExternalFileEncryptionProperties::Bui } ExternalFileEncryptionProperties::Builder* -ExternalFileEncryptionProperties::Builder::connection_config( +ExternalFileEncryptionProperties::Builder::configuration_properties( std::map> config) { - if (connection_config_.size() != 0) { - throw ParquetException("Connection config already set"); + if (configuration_properties_.size() != 0) { + throw ParquetException("Configuration properties already set"); } if (config.size() == 0) { return this; } - connection_config_ = std::move(config); + configuration_properties_ = std::move(config); return this; } @@ -422,7 +422,7 @@ std::shared_ptr ExternalFileEncryptionProperties::Builder::build_external() { return std::shared_ptr(new ExternalFileEncryptionProperties( parquet_cipher_, footer_key_, footer_key_metadata_, encrypted_footer_, aad_prefix_, - store_aad_prefix_in_file_, encrypted_columns_, app_context_, connection_config_)); + store_aad_prefix_in_file_, encrypted_columns_, app_context_, configuration_properties_)); } ExternalFileEncryptionProperties::ExternalFileEncryptionProperties( @@ -431,11 +431,11 @@ ExternalFileEncryptionProperties::ExternalFileEncryptionProperties( std::string aad_prefix, bool store_aad_prefix_in_file, ColumnPathToEncryptionPropertiesMap encrypted_columns, std::string app_context, - std::map> connection_config) + std::map> configuration_properties) : FileEncryptionProperties(cipher, footer_key, footer_key_metadata, encrypted_footer, aad_prefix, store_aad_prefix_in_file, encrypted_columns), app_context_(app_context), - connection_config_(connection_config) {} + configuration_properties_(configuration_properties) {} } // namespace parquet diff --git a/cpp/src/parquet/encryption/encryption.h b/cpp/src/parquet/encryption/encryption.h index 140550acc23d..18dc230e17a9 100644 --- a/cpp/src/parquet/encryption/encryption.h +++ b/cpp/src/parquet/encryption/encryption.h @@ -378,7 +378,7 @@ class PARQUET_EXPORT ExternalFileDecryptionProperties : public FileDecryptionPro Builder* app_context(std::string context); - Builder* connection_config( + Builder* configuration_properties( std::map> config); /// Forward all base class property methods to the base class Builder so we can return the @@ -423,21 +423,21 @@ class PARQUET_EXPORT ExternalFileDecryptionProperties : public FileDecryptionPro private: std::string app_context_; - std::map> connection_config_; + std::map> configuration_properties_; }; const std::string& app_context() const { return app_context_; } - const std::map>& - connection_config() const { - return connection_config_; + const std::map>& + configuration_properties() const { + return configuration_properties_; } private: std::string app_context_; - std::map> connection_config_; + std::map> configuration_properties_; ExternalFileDecryptionProperties( ::arrow::util::SecureString footer_key, @@ -447,7 +447,7 @@ class PARQUET_EXPORT ExternalFileDecryptionProperties : public FileDecryptionPro ColumnPathToDecryptionPropertiesMap column_decryption_properties, bool plaintext_files_allowed, std::string app_context, - std::map> connection_config); + std::map> configuration_properties); }; class PARQUET_EXPORT FileEncryptionProperties { @@ -565,7 +565,7 @@ class PARQUET_EXPORT ExternalFileEncryptionProperties : public FileEncryptionPro /// needed by the external encryptors, including location of a dynamically-linked library, /// or config files where the external encryptors can find urls, certificates, and parameters /// needed to make a remote call. - Builder* connection_config( + Builder* configuration_properties( std::map> config); /// Forward all base class property methods to the base class Builder so we can return the @@ -609,7 +609,7 @@ class PARQUET_EXPORT ExternalFileEncryptionProperties : public FileEncryptionPro private: std::string app_context_; - std::map> connection_config_; + std::map> configuration_properties_; }; const std::string& app_context() const { @@ -617,13 +617,13 @@ class PARQUET_EXPORT ExternalFileEncryptionProperties : public FileEncryptionPro } const std::map>& - connection_config() const { - return connection_config_; + configuration_properties() const { + return configuration_properties_; } private: std::string app_context_; - std::map> connection_config_; + std::map> configuration_properties_; ExternalFileEncryptionProperties(ParquetCipher::type cipher, ::arrow::util::SecureString footer_key, @@ -631,7 +631,7 @@ class PARQUET_EXPORT ExternalFileEncryptionProperties : public FileEncryptionPro std::string aad_prefix, bool store_aad_prefix_in_file, ColumnPathToEncryptionPropertiesMap encrypted_columns, std::string app_context, - std::map> connection_config); + std::map> configuration_properties); }; } // namespace parquet diff --git a/cpp/src/parquet/encryption/external/dbpa_executor.cc b/cpp/src/parquet/encryption/external/dbpa_executor.cc index 1f7f665f115d..628e644141f4 100644 --- a/cpp/src/parquet/encryption/external/dbpa_executor.cc +++ b/cpp/src/parquet/encryption/external/dbpa_executor.cc @@ -89,11 +89,11 @@ auto ExecuteWithTimeout(const std::string& operation_name, // they will be re-thrown by future.get() (original exception is thrown unchanged, no wrapping)) if constexpr (std::is_void_v) { future.get(); - ARROW_LOG(DEBUG) << "[DBPAExecutor] SUCCESS: " << operation_name << " completed successfully"; + ARROW_LOG(DEBUG) << "[DBPAExecutor] COMPLETED: " << operation_name << " operation."; return; } else { auto result = future.get(); - ARROW_LOG(DEBUG) << "[DBPAExecutor] SUCCESS: " << operation_name << " completed successfully"; + ARROW_LOG(DEBUG) << "[DBPAExecutor] COMPLETED: " << operation_name << " operation."; return result; } } @@ -138,7 +138,7 @@ DBPAExecutor::DBPAExecutor(std::unique_ptr ag void DBPAExecutor::init( std::string column_name, - std::map connection_config, + std::map configuration_properties, std::string app_context, std::string column_key_id, Type::type data_type, @@ -151,18 +151,18 @@ void DBPAExecutor::init( ExecuteWithTimeout("init", init_timeout_milliseconds_, [this](std::string col_name, - std::map conn_config, + std::map config_props, std::string app_ctx, std::string col_key_id, Type::type dt, std::optional dt_len, CompressionCodec::type comp_type, std::optional> col_enc_metadata) { - wrapped_agent_->init(std::move(col_name), std::move(conn_config), + wrapped_agent_->init(std::move(col_name), std::move(config_props), std::move(app_ctx), std::move(col_key_id), dt, dt_len, comp_type, std::move(col_enc_metadata)); }, - std::move(column_name), std::move(connection_config), + std::move(column_name), std::move(configuration_properties), std::move(app_context), std::move(column_key_id), data_type, datatype_length, compression_type, std::move(column_encryption_metadata)); } diff --git a/cpp/src/parquet/encryption/external/dbpa_executor.h b/cpp/src/parquet/encryption/external/dbpa_executor.h index 2e890bcf2ad7..ac4b0ff967dc 100644 --- a/cpp/src/parquet/encryption/external/dbpa_executor.h +++ b/cpp/src/parquet/encryption/external/dbpa_executor.h @@ -70,7 +70,7 @@ class DBPAExecutor : public DataBatchProtectionAgentInterface { */ void init( std::string column_name, - std::map connection_config, + std::map configuration_properties, std::string app_context, std::string column_key_id, Type::type data_type, diff --git a/cpp/src/parquet/encryption/external/dbpa_executor_test.cc b/cpp/src/parquet/encryption/external/dbpa_executor_test.cc index 4cecd9fdbfd5..7d15301e37d5 100644 --- a/cpp/src/parquet/encryption/external/dbpa_executor_test.cc +++ b/cpp/src/parquet/encryption/external/dbpa_executor_test.cc @@ -93,7 +93,7 @@ class MockDBPAAgent : public DataBatchProtectionAgentInterface { // Track parameters from last calls std::string last_init_column_name_; - std::map last_init_connection_config_; + std::map last_init_configuration_properties_; std::string last_init_app_context_; std::string last_init_column_key_id_; Type::type last_init_data_type_; @@ -117,7 +117,7 @@ class MockDBPAAgent : public DataBatchProtectionAgentInterface { std::string throw_message_ = "Mock agent error"; void init(std::string column_name, - std::map connection_config, + std::map configuration_properties, std::string app_context, std::string column_key_id, Type::type data_type, @@ -126,7 +126,7 @@ class MockDBPAAgent : public DataBatchProtectionAgentInterface { std::optional> column_encryption_metadata) override { init_call_count_++; last_init_column_name_ = column_name; - last_init_connection_config_ = connection_config; + last_init_configuration_properties_ = configuration_properties; last_init_app_context_ = app_context; last_init_column_key_id_ = column_key_id; last_init_data_type_ = data_type; @@ -246,7 +246,7 @@ TEST_F(DBPAExecutorTest, InitForwardsToWrappedAgent) { // Verify all parameters were forwarded correctly EXPECT_EQ(mock_agent_ptr_->last_init_column_name_, column_name); - EXPECT_EQ(mock_agent_ptr_->last_init_connection_config_, connection_config); + EXPECT_EQ(mock_agent_ptr_->last_init_configuration_properties_, connection_config); EXPECT_EQ(mock_agent_ptr_->last_init_app_context_, app_context); EXPECT_EQ(mock_agent_ptr_->last_init_column_key_id_, column_key_id); EXPECT_EQ(mock_agent_ptr_->last_init_data_type_, data_type); @@ -332,7 +332,7 @@ TEST_F(DBPAExecutorTest, MultipleCallsAreProperlyForwarded) { // Verify the last call parameters (second call) EXPECT_EQ(mock_agent_ptr_->last_init_column_name_, "column1"); - EXPECT_EQ(mock_agent_ptr_->last_init_connection_config_["key1"], "value1"); + EXPECT_EQ(mock_agent_ptr_->last_init_configuration_properties_["key1"], "value1"); EXPECT_EQ(mock_agent_ptr_->last_init_app_context_, "context1"); EXPECT_EQ(mock_agent_ptr_->last_init_column_key_id_, "key1"); EXPECT_EQ(mock_agent_ptr_->last_init_data_type_, Type::type::INT32); diff --git a/cpp/src/parquet/encryption/external/dbpa_library_wrapper.h b/cpp/src/parquet/encryption/external/dbpa_library_wrapper.h index c233a884608a..823c38aa5967 100644 --- a/cpp/src/parquet/encryption/external/dbpa_library_wrapper.h +++ b/cpp/src/parquet/encryption/external/dbpa_library_wrapper.h @@ -72,14 +72,14 @@ class DBPALibraryWrapper : public DataBatchProtectionAgentInterface { // Decorator implementation of init method inline void init( std::string column_name, - std::map connection_config, + std::map configuration_properties, std::string app_context, std::string column_key_id, Type::type data_type, std::optional datatype_length, CompressionCodec::type compression_type, std::optional> column_encryption_metadata) override { - wrapped_agent_->init(std::move(column_name), std::move(connection_config), + wrapped_agent_->init(std::move(column_name), std::move(configuration_properties), std::move(app_context), std::move(column_key_id), data_type, datatype_length, compression_type, std::move(column_encryption_metadata)); } diff --git a/cpp/src/parquet/encryption/external/dbpa_library_wrapper_test.cc b/cpp/src/parquet/encryption/external/dbpa_library_wrapper_test.cc index 2404ed7dab64..eccb5ecc336c 100644 --- a/cpp/src/parquet/encryption/external/dbpa_library_wrapper_test.cc +++ b/cpp/src/parquet/encryption/external/dbpa_library_wrapper_test.cc @@ -189,7 +189,7 @@ class MockCompanionDBPA { // Init tracking methods const std::string& GetInitColumnName() const { return init_column_name_; } - const std::map& GetInitConnectionConfig() const { return init_connection_config_; } + const std::map& GetInitConfigurationProperties() const { return init_configuration_properties_; } const std::string& GetInitAppContext() const { return init_app_context_; } const std::string& GetInitColumnKeyId() const { return init_column_key_id_; } Type::type GetInitDataType() const { return init_data_type_; } @@ -223,7 +223,7 @@ class MockCompanionDBPA { // Init parameter tracking void SetInitParameters( std::string column_name, - std::map connection_config, + std::map configuration_properties, std::string app_context, std::string column_key_id, Type::type data_type, @@ -231,7 +231,7 @@ class MockCompanionDBPA { std::optional datatype_length = std::nullopt, std::optional> column_encryption_metadata = std::nullopt) { init_column_name_ = std::move(column_name); - init_connection_config_ = std::move(connection_config); + init_configuration_properties_ = std::move(configuration_properties); init_app_context_ = std::move(app_context); init_column_key_id_ = std::move(column_key_id); init_data_type_ = data_type; @@ -255,7 +255,7 @@ class MockCompanionDBPA { // Init parameters std::string init_column_name_; - std::map init_connection_config_; + std::map init_configuration_properties_; std::string init_app_context_; std::string init_column_key_id_; Type::type init_data_type_; @@ -550,7 +550,7 @@ TEST_F(DBPALibraryWrapperTest, InitDelegation) { // Verify the correct parameters were passed to the mock EXPECT_EQ(mock_companion_->GetInitColumnName(), column_name); - EXPECT_EQ(mock_companion_->GetInitConnectionConfig(), connection_config); + EXPECT_EQ(mock_companion_->GetInitConfigurationProperties(), connection_config); EXPECT_EQ(mock_companion_->GetInitAppContext(), app_context); EXPECT_EQ(mock_companion_->GetInitColumnKeyId(), column_key_id); EXPECT_EQ(mock_companion_->GetInitDataType(), data_type); @@ -597,7 +597,7 @@ TEST_F(DBPALibraryWrapperTest, InitDelegationWithEmptyParameters) { // Verify the empty parameters were passed correctly EXPECT_EQ(mock_companion_->GetInitColumnName(), empty_column_name); - EXPECT_EQ(mock_companion_->GetInitConnectionConfig(), empty_connection_config); + EXPECT_EQ(mock_companion_->GetInitConfigurationProperties(), empty_connection_config); EXPECT_EQ(mock_companion_->GetInitAppContext(), empty_app_context); EXPECT_EQ(mock_companion_->GetInitColumnKeyId(), empty_column_key_id); EXPECT_EQ(mock_companion_->GetInitDataType(), data_type); @@ -794,7 +794,7 @@ TEST_F(DBPALibraryWrapperTest, InitWithEncryptDecryptOperations) { // Verify init parameters are still accessible EXPECT_EQ(mock_companion_->GetInitColumnName(), column_name); - EXPECT_EQ(mock_companion_->GetInitConnectionConfig(), connection_config); + EXPECT_EQ(mock_companion_->GetInitConfigurationProperties(), connection_config); EXPECT_EQ(mock_companion_->GetInitAppContext(), app_context); EXPECT_EQ(mock_companion_->GetInitColumnKeyId(), column_key_id); EXPECT_EQ(mock_companion_->GetInitDataType(), data_type); diff --git a/cpp/src/parquet/encryption/external/dbpa_test_agent.cc b/cpp/src/parquet/encryption/external/dbpa_test_agent.cc index a14f8ab198a2..d94fdfaa5c94 100644 --- a/cpp/src/parquet/encryption/external/dbpa_test_agent.cc +++ b/cpp/src/parquet/encryption/external/dbpa_test_agent.cc @@ -110,8 +110,8 @@ std::unique_ptr DBPATestAgent::Encrypt( } // For tests, optionally force a conflicting metadata value on subsequent calls - auto it = connection_config_.find("dbpa_test_force_conflicting_metadata"); - bool force_conflict = (it != connection_config_.end() && it->second == "1"); + auto it = configuration_properties_.find("dbpa_test_force_conflicting_metadata"); + bool force_conflict = (it != configuration_properties_.end() && it->second == "1"); encrypt_calls_++; if (force_conflict && encrypt_calls_ >= 2) { // Return a different value for test_key1 to trigger conflict in writer diff --git a/cpp/src/parquet/encryption/external/dbpa_test_agent.h b/cpp/src/parquet/encryption/external/dbpa_test_agent.h index da8af599ff9c..af4b417cf2dc 100644 --- a/cpp/src/parquet/encryption/external/dbpa_test_agent.h +++ b/cpp/src/parquet/encryption/external/dbpa_test_agent.h @@ -42,7 +42,7 @@ class DBPATestAgent : public DataBatchProtectionAgentInterface { void init( std::string column_name, - std::map connection_config, + std::map configuration_properties, std::string app_context, std::string column_key_id, Type::type data_type, @@ -55,7 +55,7 @@ class DBPATestAgent : public DataBatchProtectionAgentInterface { } // Store the key id so we can use it for simple test XOR encryption/decryption key_ = std::move(column_key_id); - connection_config_ = std::move(connection_config); + configuration_properties_ = std::move(configuration_properties); } std::unique_ptr Encrypt( @@ -72,7 +72,7 @@ class DBPATestAgent : public DataBatchProtectionAgentInterface { // Used as a simple XOR key for test encryption/decryption std::string key_; // Stored connection configuration from init(); used to toggle test behaviors - std::map connection_config_; + std::map configuration_properties_; // Count Encrypt() calls to allow staged behavior in tests size_t encrypt_calls_ = 0; }; diff --git a/cpp/src/parquet/encryption/external_dbpa_encryption.cc b/cpp/src/parquet/encryption/external_dbpa_encryption.cc index bf35d649efd4..cd1db043d032 100644 --- a/cpp/src/parquet/encryption/external_dbpa_encryption.cc +++ b/cpp/src/parquet/encryption/external_dbpa_encryption.cc @@ -49,7 +49,7 @@ namespace { // Shared between the encryptor and decryptor. std::unique_ptr LoadAndInitializeAgent( const std::string& column_name, - const std::map& connection_config, + const std::map& configuration_properties, const std::string& app_context, const std::string& key_id, Type::type data_type, @@ -64,9 +64,9 @@ std::unique_ptr LoadAndInitia const std::string DECRYPT_TIMEOUT_KEY = "agent_decrypt_timeout_ms"; // Step 1: Get path to the shared library - auto it = connection_config.find(SHARED_LIBRARY_PATH_KEY); - if (it == connection_config.end()) { - auto const msg = "Required configuration key '" + SHARED_LIBRARY_PATH_KEY + "' not found in connection_config"; + auto it = configuration_properties.find(SHARED_LIBRARY_PATH_KEY); + if (it == configuration_properties.end()) { + auto const msg = "Required configuration key '" + SHARED_LIBRARY_PATH_KEY + "' not found in configuration_properties"; ARROW_LOG(ERROR) << msg; throw ParquetException(msg); } @@ -89,20 +89,20 @@ std::unique_ptr LoadAndInitia int64_t init_timeout_ms = 10*1000; //10 seconds int64_t encrypt_timeout_ms = 30*1000; //30 seconds int64_t decrypt_timeout_ms = 30*1000; //30 seconds. - // Override the default values if they are present in the connection_config. + // Override the default values if they are present in the configuration_properties. try { - if (connection_config.find(INIT_TIMEOUT_KEY) != connection_config.end()) { - init_timeout_ms = std::stoi(connection_config.at(INIT_TIMEOUT_KEY)); + if (configuration_properties.find(INIT_TIMEOUT_KEY) != configuration_properties.end()) { + init_timeout_ms = std::stoi(configuration_properties.at(INIT_TIMEOUT_KEY)); } - if (connection_config.find(ENCRYPT_TIMEOUT_KEY) != connection_config.end()) { - encrypt_timeout_ms = std::stoi(connection_config.at(ENCRYPT_TIMEOUT_KEY)); + if (configuration_properties.find(ENCRYPT_TIMEOUT_KEY) != configuration_properties.end()) { + encrypt_timeout_ms = std::stoi(configuration_properties.at(ENCRYPT_TIMEOUT_KEY)); } - if (connection_config.find(DECRYPT_TIMEOUT_KEY) != connection_config.end()) { - decrypt_timeout_ms = std::stoi(connection_config.at(DECRYPT_TIMEOUT_KEY)); + if (configuration_properties.find(DECRYPT_TIMEOUT_KEY) != configuration_properties.end()) { + decrypt_timeout_ms = std::stoi(configuration_properties.at(DECRYPT_TIMEOUT_KEY)); } } catch (const std::exception& e) { - ARROW_LOG(ERROR) << "Failed to parse timeout values from connection_config: " << e.what(); - throw ParquetException("Failed to parse timeout values from connection_config"); + ARROW_LOG(ERROR) << "Failed to parse timeout values from configuration_properties: " << e.what(); + throw ParquetException("Failed to parse timeout values from configuration_properties"); } ARROW_LOG(DEBUG) << "init_timeout_ms = " << init_timeout_ms; @@ -125,7 +125,7 @@ std::unique_ptr LoadAndInitia executor_wrapped_agent->init( /*column_name*/ column_name, - /*connection_config*/ connection_config, + /*configuration_properties*/ configuration_properties, /*app_context*/ app_context, /*column_key_id*/ key_id, /*data_type*/ DBPAEnumUtils::ParquetTypeToDBPA(data_type), @@ -203,12 +203,12 @@ return metadata_map; ExternalDBPAEncryptorAdapter::ExternalDBPAEncryptorAdapter( ParquetCipher::type algorithm, std::string column_name, std::string key_id, Type::type data_type, Compression::type compression_type, Encoding::type encoding_type, - std::optional datatype_length, std::string app_context, std::map connection_config, + std::optional datatype_length, std::string app_context, std::map configuration_properties, std::unique_ptr agent_instance) : algorithm_(algorithm), column_name_(column_name), key_id_(key_id), data_type_(data_type), compression_type_(compression_type), encoding_type_(encoding_type), datatype_length_(datatype_length), app_context_(app_context), - connection_config_(connection_config), + configuration_properties_(configuration_properties), agent_instance_(std::move(agent_instance)) { if (algorithm != ParquetCipher::EXTERNAL_DBPA_V1) { @@ -219,7 +219,7 @@ ExternalDBPAEncryptorAdapter::ExternalDBPAEncryptorAdapter( std::unique_ptr ExternalDBPAEncryptorAdapter::Make( ParquetCipher::type algorithm, std::string column_name, std::string key_id, Type::type data_type, Compression::type compression_type, Encoding::type encoding_type, - std::string app_context, std::map connection_config, + std::string app_context, std::map configuration_properties, std::optional datatype_length) { // Ensure DBPA logging threshold is configured before any logs here @@ -235,8 +235,8 @@ std::unique_ptr ExternalDBPAEncryptorAdapter::Make ARROW_LOG(DEBUG) << " compression_type = " << compression_type; ARROW_LOG(DEBUG) << " encoding_type = " << encoding_type; ARROW_LOG(DEBUG) << " app_context = " << app_context; - ARROW_LOG(DEBUG) << " connection_config:"; - for (const auto& [key, value] : connection_config) { + ARROW_LOG(DEBUG) << " configuration_properties:"; + for (const auto& [key, value] : configuration_properties) { ARROW_LOG(DEBUG) << " " << key << " = " << value; } } @@ -248,7 +248,7 @@ std::unique_ptr ExternalDBPAEncryptorAdapter::Make ARROW_LOG(DEBUG) << "ExternalDBPAEncryptorAdapter::ExternalDBPAEncryptorAdapter() -- loading and initializing agent"; // Load and initialize the agent using the utility function auto agent_instance = LoadAndInitializeAgent( - column_name, connection_config, app_context, key_id, data_type, compression_type, datatype_length, + column_name, configuration_properties, app_context, key_id, data_type, compression_type, datatype_length, /*key_value_metadata*/ nullptr); //if we got to this point, the agent was initialized successfully @@ -265,7 +265,7 @@ std::unique_ptr ExternalDBPAEncryptorAdapter::Make /*encoding_type*/ encoding_type, /*datatype_length*/ datatype_length, /*app_context*/ app_context, - /*connection_config*/ connection_config, + /*configuration_properties*/ configuration_properties, /*agent_instance*/ std::move(agent_instance)) ); @@ -337,8 +337,8 @@ int32_t ExternalDBPAEncryptorAdapter::InvokeExternalEncrypt( ARROW_LOG(DEBUG) << "Compression Type: [" << compression_type_ << "]"; ARROW_LOG(DEBUG) << "Encoding Type: [" << encoding_type_ << "]"; ARROW_LOG(DEBUG) << "App Context: [" << app_context_ << "]"; - ARROW_LOG(DEBUG) << "Connection Config:"; - for (const auto& [cfg_key, cfg_value] : connection_config_) { + ARROW_LOG(DEBUG) << "Configuration Properties:"; + for (const auto& [cfg_key, cfg_value] : configuration_properties_) { ARROW_LOG(DEBUG) << " [" << cfg_key << "]: [" << cfg_value << "]"; } } @@ -389,9 +389,9 @@ ExternalDBPAEncryptorAdapter* ExternalDBPAEncryptorAdapterFactory::GetEncryptor( } auto column_path = column_chunk_metadata->descr()->path(); if (encryptor_cache_.find(column_path->ToDotString()) == encryptor_cache_.end()) { - auto connection_config = external_file_encryption_properties->connection_config(); - if (connection_config.find(algorithm) == connection_config.end()) { - throw ParquetException("External DBPA encryption requires its connection configuration"); + auto configuration_properties = external_file_encryption_properties->configuration_properties(); + if (configuration_properties.find(algorithm) == configuration_properties.end()) { + throw ParquetException("External DBPA encryption requires its configuration properties"); } auto column_encryption_properties = external_file_encryption_properties @@ -411,7 +411,7 @@ ExternalDBPAEncryptorAdapter* ExternalDBPAEncryptorAdapterFactory::GetEncryptor( auto compression_type = column_chunk_metadata->properties()->compression(column_path); auto encoding_type = column_chunk_metadata->properties()->encoding(column_path); auto app_context = external_file_encryption_properties->app_context(); - auto connection_config_for_algorithm = connection_config.at(algorithm); + auto connection_config_for_algorithm = configuration_properties.at(algorithm); std::string key_id; try { @@ -441,13 +441,13 @@ ExternalDBPADecryptorAdapter::ExternalDBPADecryptorAdapter( ParquetCipher::type algorithm, std::string column_name, std::string key_id, Type::type data_type, Compression::type compression_type, std::vector encoding_types, std::optional datatype_length, std::string app_context, - std::map connection_config, + std::map configuration_properties, std::unique_ptr agent_instance, std::shared_ptr key_value_metadata) : algorithm_(algorithm), column_name_(column_name), key_id_(key_id), data_type_(data_type), compression_type_(compression_type), encoding_types_(encoding_types), datatype_length_(datatype_length), app_context_(app_context), - connection_config_(connection_config), + configuration_properties_(configuration_properties), agent_instance_(std::move(agent_instance)) { if (algorithm != ParquetCipher::EXTERNAL_DBPA_V1) { @@ -462,7 +462,7 @@ std::unique_ptr ExternalDBPADecryptorAdapter::Make ParquetCipher::type algorithm, std::string column_name, std::string key_id, Type::type data_type, Compression::type compression_type, std::vector encoding_types, std::string app_context, - std::map connection_config, + std::map configuration_properties, std::optional datatype_length, std::shared_ptr key_value_metadata) { @@ -489,8 +489,8 @@ std::unique_ptr ExternalDBPADecryptorAdapter::Make ARROW_LOG(DEBUG) << ss.str(); } ARROW_LOG(DEBUG) << " app_context = " << app_context; - ARROW_LOG(DEBUG) << " connection_config:"; - for (const auto& [key, value] : connection_config) { + ARROW_LOG(DEBUG) << " configuration_properties:"; + for (const auto& [key, value] : configuration_properties) { ARROW_LOG(DEBUG) << " " << key << " = " << value; } ARROW_LOG(DEBUG) << " key_value_metadata:"; @@ -503,7 +503,7 @@ std::unique_ptr ExternalDBPADecryptorAdapter::Make ARROW_LOG(DEBUG) << "ExternalDBPADecryptorAdapter::ExternalDBPADecryptorAdapter() -- loading and initializing agent"; // Load and initialize the agent using the utility function auto agent_instance = LoadAndInitializeAgent( - column_name, connection_config, app_context, key_id, data_type, compression_type, datatype_length, + column_name, configuration_properties, app_context, key_id, data_type, compression_type, datatype_length, /*key_value_metadata*/ key_value_metadata); //if we got to this point, the agent was initialized successfully @@ -519,7 +519,7 @@ std::unique_ptr ExternalDBPADecryptorAdapter::Make /*encoding_types*/ encoding_types, /*datatype_length*/ datatype_length, /*app_context*/ app_context, - /*connection_config*/ connection_config, + /*configuration_properties*/ configuration_properties, /*agent_instance*/ std::move(agent_instance), /*key_value_metadata*/ key_value_metadata) ); @@ -584,8 +584,8 @@ int32_t ExternalDBPADecryptorAdapter::InvokeExternalDecrypt( ARROW_LOG(DEBUG) << ss.str(); } ARROW_LOG(DEBUG) << "App Context: [" << app_context_ << "]"; - ARROW_LOG(DEBUG) << "Connection Config:"; - for (const auto& [key, value] : connection_config_) { + ARROW_LOG(DEBUG) << "Configuration Properties:"; + for (const auto& [key, value] : configuration_properties_) { ARROW_LOG(DEBUG) << " [" << key << "]: [" << value << "]"; } } @@ -633,9 +633,9 @@ std::unique_ptr ExternalDBPADecryptorAdapterFactory::GetDecr if (column_chunk_metadata == nullptr || crypto_metadata == nullptr) { throw ParquetException("External DBPA decryption requires column chunk and crypto metadata"); } - auto connection_config = external_file_decryption_properties->connection_config(); - if (connection_config.find(algorithm) == connection_config.end()) { - throw ParquetException("External DBPA decryption requires its connection configuration"); + auto configuration_properties = external_file_decryption_properties->configuration_properties(); + if (configuration_properties.find(algorithm) == configuration_properties.end()) { + throw ParquetException("External DBPA decryption requires its configuration properties"); } auto column_path = column_chunk_metadata->descr()->path(); auto data_type = column_chunk_metadata->descr()->physical_type(); @@ -646,7 +646,7 @@ std::unique_ptr ExternalDBPADecryptorAdapterFactory::GetDecr auto compression_type = column_chunk_metadata->compression(); auto encoding_types = column_chunk_metadata->encodings(); auto app_context = external_file_decryption_properties->app_context(); - auto connection_config_for_algorithm = connection_config.at(algorithm); + auto connection_config_for_algorithm = configuration_properties.at(algorithm); auto key_value_metadata = column_chunk_metadata->key_value_metadata(); std::string key_id; diff --git a/cpp/src/parquet/encryption/external_dbpa_encryption.h b/cpp/src/parquet/encryption/external_dbpa_encryption.h index edf88fd55c9c..f2c67ec3dd27 100644 --- a/cpp/src/parquet/encryption/external_dbpa_encryption.h +++ b/cpp/src/parquet/encryption/external_dbpa_encryption.h @@ -41,7 +41,7 @@ class ExternalDBPAEncryptorAdapter : public EncryptorInterface { ParquetCipher::type algorithm, std::string column_name, std::string key_id, Type::type data_type, Compression::type compression_type, Encoding::type encoding_type, std::string app_context, - std::map connection_config, + std::map configuration_properties, std::optional datatype_length); ~ExternalDBPAEncryptorAdapter() = default; @@ -88,7 +88,7 @@ class ExternalDBPAEncryptorAdapter : public EncryptorInterface { ParquetCipher::type algorithm, std::string column_name, std::string key_id, Type::type data_type, Compression::type compression_type, Encoding::type encoding_type, std::optional datatype_length, std::string app_context, - std::map connection_config, + std::map configuration_properties, std::unique_ptr agent_instance); int32_t InvokeExternalEncrypt( @@ -104,7 +104,7 @@ class ExternalDBPAEncryptorAdapter : public EncryptorInterface { Encoding::type encoding_type_; std::optional datatype_length_; std::string app_context_; - std::map connection_config_; + std::map configuration_properties_; std::unique_ptr agent_instance_; @@ -153,7 +153,7 @@ class ExternalDBPADecryptorAdapter : public DecryptorInterface { ParquetCipher::type algorithm, std::string column_name, std::string key_id, Type::type data_type, Compression::type compression_type, std::vector encoding_types, std::string app_context, - std::map connection_config, + std::map configuration_properties, std::optional datatype_length, std::shared_ptr key_value_metadata); @@ -197,7 +197,7 @@ class ExternalDBPADecryptorAdapter : public DecryptorInterface { ParquetCipher::type algorithm, std::string column_name, std::string key_id, Type::type data_type, Compression::type compression_type, std::vector encoding_types, std::optional datatype_length, - std::string app_context, std::map connection_config, + std::string app_context, std::map configuration_properties, std::unique_ptr agent_instance, std::shared_ptr key_value_metadata); @@ -215,7 +215,7 @@ class ExternalDBPADecryptorAdapter : public DecryptorInterface { std::vector encoding_types_; std::optional datatype_length_; std::string app_context_; - std::map connection_config_; + std::map configuration_properties_; std::unique_ptr agent_instance_; diff --git a/cpp/src/parquet/encryption/properties_test.cc b/cpp/src/parquet/encryption/properties_test.cc index ca4086594486..b955f09d7dd7 100644 --- a/cpp/src/parquet/encryption/properties_test.cc +++ b/cpp/src/parquet/encryption/properties_test.cc @@ -298,7 +298,7 @@ TEST(TestExternalFileEncryptionProperties, SuperClassFieldsSetCorrectly) { ASSERT_EQ(NULLPTR, out_col_props_3); ASSERT_EQ(true, props->app_context().empty()); - ASSERT_EQ(true, props->connection_config().size() == 0); + ASSERT_EQ(true, props->configuration_properties().size() == 0); } // The subclass adds two additional fields @@ -310,19 +310,19 @@ TEST(TestExternalFileEncryptionProperties, SetExternalContextAndConfig) { " \"lon\": -84.0\n" " }\n" "}"; - std::map> connection_config; - connection_config[ParquetCipher::AES_GCM_V1]["lib_location"] = "path/to/lib.so"; - connection_config[ParquetCipher::AES_GCM_V1]["config_file"] = "path/to/config/file"; + std::map> configuration_properties; + configuration_properties[ParquetCipher::AES_GCM_V1]["lib_location"] = "path/to/lib.so"; + configuration_properties[ParquetCipher::AES_GCM_V1]["config_file"] = "path/to/config/file"; ExternalFileEncryptionProperties::Builder builder(kFooterEncryptionKey); builder.app_context(app_context); - builder.connection_config(connection_config); + builder.configuration_properties(configuration_properties); std::shared_ptr props = builder.build_external(); ASSERT_EQ(false, props->app_context().empty()); ASSERT_EQ(app_context, props->app_context()); - ASSERT_EQ(false, props->connection_config().size() == 0); - ASSERT_EQ(connection_config, props->connection_config()); + ASSERT_EQ(false, props->configuration_properties().size() == 0); + ASSERT_EQ(configuration_properties, props->configuration_properties()); } TEST(TestExternalFileEncryptionProperties, EncryptTwoColumnsWithDifferentAlgorithms) { @@ -450,7 +450,7 @@ TEST(TestExternalFileDecryptionProperties, SuperClassFieldsSetCorrectly) { ASSERT_EQ(true, props->plaintext_files_allowed()); ASSERT_EQ(kFooterEncryptionKey, props->footer_key()); ASSERT_EQ(true, props->app_context().empty()); - ASSERT_EQ(true, props->connection_config().size() == 0); + ASSERT_EQ(true, props->configuration_properties().size() == 0); auto out_key_retriever = props->key_retriever(); ASSERT_EQ(kFooterEncryptionKey, out_key_retriever->GetKey("kf")); @@ -477,13 +477,13 @@ TEST(TestExternalFileDecryptionProperties, SetExternalContextAndConfig) { auto builder = parquet::ExternalFileDecryptionProperties::Builder(); builder.footer_key(kFooterEncryptionKey); builder.app_context(app_context); - builder.connection_config(connection_config); + builder.configuration_properties(connection_config); std::shared_ptr props = builder.build_external(); ASSERT_EQ(false, props->app_context().empty()); ASSERT_EQ(app_context, props->app_context()); - ASSERT_EQ(false, props->connection_config().size() == 0); - ASSERT_EQ(connection_config, props->connection_config()); + ASSERT_EQ(false, props->configuration_properties().size() == 0); + ASSERT_EQ(connection_config, props->configuration_properties()); } TEST(TestExternalFileDecryptionProperties, SetInvalidAppContext) { diff --git a/cpp/src/parquet/file_deserialize_test.cc b/cpp/src/parquet/file_deserialize_test.cc index 36f7c97c2ab2..9fb0369111b2 100644 --- a/cpp/src/parquet/file_deserialize_test.cc +++ b/cpp/src/parquet/file_deserialize_test.cc @@ -1298,7 +1298,7 @@ namespace parquet { ->algorithm(parquet::ParquetCipher::AES_GCM_CTR_V1) ->encrypted_columns(enc_cols) ->app_context(app_context) - ->connection_config({{parquet::ParquetCipher::EXTERNAL_DBPA_V1, + ->configuration_properties({{parquet::ParquetCipher::EXTERNAL_DBPA_V1, {{"agent_library_path", library_path}}}}); auto file_enc_props = fep_builder.build_external(); @@ -1323,7 +1323,7 @@ namespace parquet { dep_builder.footer_key(footer_key) ->column_keys(dec_cols) ->app_context(app_context) - ->connection_config({{parquet::ParquetCipher::EXTERNAL_DBPA_V1, + ->configuration_properties({{parquet::ParquetCipher::EXTERNAL_DBPA_V1, {{"agent_library_path", library_path}}}}); reader_props.file_decryption_properties(dep_builder.build_external()); diff --git a/python/pyarrow/_parquet_encryption.pyx b/python/pyarrow/_parquet_encryption.pyx index 048be845e554..3249ac842e80 100644 --- a/python/pyarrow/_parquet_encryption.pyx +++ b/python/pyarrow/_parquet_encryption.pyx @@ -203,8 +203,8 @@ cdef class ExternalEncryptionConfiguration(EncryptionConfiguration): encryption_algorithm=None, plaintext_footer=None, double_wrapping=None, cache_lifetime=None, internal_key_material=None, - data_key_length_bits=None, per_column_encryption=None, - app_context=None, connection_config=None): + data_key_length_bits=None, per_column_encryption=None, + app_context=None, configuration_properties=None): # Initialize pointer first so the get/set forwards work. self.external_configuration.reset( @@ -224,8 +224,8 @@ cdef class ExternalEncryptionConfiguration(EncryptionConfiguration): if app_context is not None: self.app_context = app_context - if connection_config is not None: - self.connection_config = connection_config + if configuration_properties is not None: + self.configuration_properties = configuration_properties if per_column_encryption is not None: self.per_column_encryption = per_column_encryption @@ -319,14 +319,14 @@ cdef class ExternalEncryptionConfiguration(EncryptionConfiguration): raise TypeError(f"Failed to serialize app_context: {repr(value)}") @property - def connection_config(self): - """Get the connection configuration as a Python dictionary.""" + def configuration_properties(self): + """Get the configuration properties as a Python dictionary.""" cdef pair[ParquetCipher, unordered_map[c_string, c_string]] outer_pair cdef pair[c_string, c_string] inner_pair result = {} - for outer_pair in self.external_configuration.get().connection_config: + for outer_pair in self.external_configuration.get().configuration_properties: cipher_name = cipher_to_name(outer_pair.first) inner_map = {} for inner_pair in outer_pair.second: @@ -335,11 +335,11 @@ cdef class ExternalEncryptionConfiguration(EncryptionConfiguration): return result - @connection_config.setter - def connection_config(self, dict value): - """Set the connection configuration from a Python dictionary.""" + @configuration_properties.setter + def configuration_properties(self, dict value): + """Set the configuration properties from a Python dictionary.""" if value is None: - raise ValueError("Connection config value cannot be None") + raise ValueError("Configuration properties value cannot be None") cdef unordered_map[ParquetCipher, unordered_map[c_string, c_string]] cpp_map cdef unordered_map[c_string, c_string] inner_cpp_map @@ -358,7 +358,7 @@ cdef class ExternalEncryptionConfiguration(EncryptionConfiguration): inner_cpp_map[tobytes(k)] = tobytes(v) cpp_map[cipher_enum] = inner_cpp_map - self.external_configuration.get().connection_config = cpp_map + self.external_configuration.get().configuration_properties = cpp_map @property def per_column_encryption(self): @@ -432,7 +432,7 @@ cdef class ExternalDecryptionConfiguration(DecryptionConfiguration): # Avoid mistakingly creating attributes __slots__ = () - def __init__(self, *, cache_lifetime=None, app_context=None, connection_config=None): + def __init__(self, *, cache_lifetime=None, app_context=None, configuration_properties=None): # Initialize the pointer first so the get/set forwards work. # Super init will run the setters/getters below so we need the pointer to exist. self.external_configuration.reset(new CExternalDecryptionConfiguration()) @@ -443,8 +443,8 @@ cdef class ExternalDecryptionConfiguration(DecryptionConfiguration): if app_context is not None: self.app_context = app_context - if connection_config is not None: - self.connection_config = connection_config + if configuration_properties is not None: + self.configuration_properties = configuration_properties """ Forward all attributes get/set methods to the superclass """ """ The superclass already converts to/from bytes and does additional processing needed """ @@ -481,14 +481,14 @@ cdef class ExternalDecryptionConfiguration(DecryptionConfiguration): raise TypeError(f"Failed to serialize app_context: {repr(value)}") @property - def connection_config(self): - """Get the connection configuration as a Python dictionary.""" + def configuration_properties(self): + """Get the configuration properties as a Python dictionary.""" cdef pair[ParquetCipher, unordered_map[c_string, c_string]] outer_pair cdef pair[c_string, c_string] inner_pair result = {} - for outer_pair in self.external_configuration.get().connection_config: + for outer_pair in self.external_configuration.get().configuration_properties: cipher_name = cipher_to_name(outer_pair.first) inner_map = {} for inner_pair in outer_pair.second: @@ -497,11 +497,11 @@ cdef class ExternalDecryptionConfiguration(DecryptionConfiguration): return result - @connection_config.setter - def connection_config(self, dict value): - """Set the connection configuration from a Python dictionary.""" + @configuration_properties.setter + def configuration_properties(self, dict value): + """Set the configuration properties from a Python dictionary.""" if value is None: - raise ValueError("Connection config value cannot be None") + raise ValueError("Configuration properties value cannot be None") cdef unordered_map[ParquetCipher, unordered_map[c_string, c_string]] cpp_map cdef unordered_map[c_string, c_string] inner_cpp_map @@ -518,7 +518,7 @@ cdef class ExternalDecryptionConfiguration(DecryptionConfiguration): inner_cpp_map[tobytes(k)] = tobytes(v) cpp_map[cipher_enum] = inner_cpp_map - self.external_configuration.get().connection_config = cpp_map + self.external_configuration.get().configuration_properties = cpp_map cdef inline shared_ptr[CExternalDecryptionConfiguration] unwrap_external(self) nogil: return self.external_configuration diff --git a/python/pyarrow/includes/libparquet_encryption.pxd b/python/pyarrow/includes/libparquet_encryption.pxd index 0ac688cc4358..4828248c8e75 100644 --- a/python/pyarrow/includes/libparquet_encryption.pxd +++ b/python/pyarrow/includes/libparquet_encryption.pxd @@ -103,7 +103,7 @@ cdef extern from "parquet/encryption/crypto_factory.h" \ int32_t data_key_length_bits unordered_map[c_string, CColumnEncryptionAttributes] per_column_encryption c_string app_context - unordered_map[ParquetCipher, unordered_map[c_string, c_string]] connection_config + unordered_map[ParquetCipher, unordered_map[c_string, c_string]] configuration_properties cdef cppclass CDecryptionConfiguration\ " parquet::encryption::DecryptionConfiguration": @@ -115,7 +115,7 @@ cdef extern from "parquet/encryption/crypto_factory.h" \ CExternalDecryptionConfiguration() except + double cache_lifetime_seconds c_string app_context - unordered_map[ParquetCipher, unordered_map[c_string, c_string]] connection_config + unordered_map[ParquetCipher, unordered_map[c_string, c_string]] configuration_properties cdef cppclass CCryptoFactory" parquet::encryption::CryptoFactory": void RegisterKmsClientFactory( diff --git a/python/pyarrow/tests/parquet/test_external_encryption.py b/python/pyarrow/tests/parquet/test_external_encryption.py index 708b5807406a..acdbefc794cd 100644 --- a/python/pyarrow/tests/parquet/test_external_encryption.py +++ b/python/pyarrow/tests/parquet/test_external_encryption.py @@ -123,7 +123,7 @@ def get_external_encryption_config(plaintext_footer=True): "user_id": "Picard1701", "location": "Presidio" }, - connection_config={ + configuration_properties={ "EXTERNAL_DBPA_V1": { "config_file": "path/to/config/file", "config_file_decryption_key": "some_key", @@ -158,7 +158,7 @@ def get_external_decryption_config(): "user_id": "Picard1701", "location": "Presidio" }, - connection_config={ + configuration_properties={ "EXTERNAL_DBPA_V1": { "config_file": "path/to/config/file", "config_file_decryption_key": "some_key", @@ -253,7 +253,7 @@ def test_external_encryption_configuration_properties(): "location": "Presidio" } - assert external_encryption_config.connection_config == { + assert external_encryption_config.configuration_properties == { "EXTERNAL_DBPA_V1": { "config_file": "path/to/config/file", "config_file_decryption_key": "some_key", @@ -310,15 +310,15 @@ def test_external_encryption_per_column_encryption_new_algorithm(): ) -def test_external_encryption_connection_config_invalid_types(): - """Ensure connection_config rejects non-string keys or values.""" +def test_external_encryption_configuration_properties_invalid_types(): + """Ensure configuration_properties rejects non-string keys or values.""" with pytest.raises( TypeError, match="All inner config keys/values must be str"): config = ppe.ExternalEncryptionConfiguration( footer_key="key" ) - config.connection_config = { + config.configuration_properties = { "EXTERNAL_DBPA_V1": { "config_file": "path/to/file", 123: "should-fail" # Invalid: key is not a string @@ -331,7 +331,7 @@ def test_external_encryption_connection_config_invalid_types(): config = ppe.ExternalEncryptionConfiguration( footer_key="key" ) - config.connection_config = { + config.configuration_properties = { "EXTERNAL_DBPA_V1": { "config_file": ["not", "a", "string"] # Invalid: value is not a string } @@ -350,9 +350,9 @@ def test_external_encryption_rejects_none_values(): with pytest.raises(ValueError, match="app_context must be JSON-serializable"): config.app_context = None - # connection_config: expect ValueError due to None not being iterable - with pytest.raises(ValueError, match="Connection config value cannot be None"): - config.connection_config = None + # configuration_properties: expect ValueError due to None not being iterable + with pytest.raises(ValueError, match="Configuration properties value cannot be None"): + config.configuration_properties = None def test_external_file_encryption_properties_rejects_column_in_two_places(): @@ -407,7 +407,7 @@ def test_external_decryption_configuration_properties(): "user_id": "Picard1701", "location": "Presidio" } - assert external_decryption_config.connection_config == { + assert external_decryption_config.configuration_properties == { "EXTERNAL_DBPA_V1": { "config_file": "path/to/config/file", "config_file_decryption_key": "some_key", @@ -416,13 +416,13 @@ def test_external_decryption_configuration_properties(): } -def test_external_decryption_connection_config_invalid_types(): - """Ensure connection_config rejects non-string keys or values.""" +def test_external_decryption_configuration_properties_invalid_types(): + """Ensure configuration_properties rejects non-string keys or values.""" # Outer key is not a string (int instead of cipher name string) with pytest.raises(AttributeError, match="'int' object has no attribute 'upper'"): config = ppe.ExternalDecryptionConfiguration() - config.connection_config = { + config.configuration_properties = { 123: { # invalid outer key "config_file": "should-fail" } @@ -434,14 +434,14 @@ def test_external_decryption_connection_config_invalid_types(): match="Inner value for cipher AES_GCM_V1 must be a dict" ): config = ppe.ExternalDecryptionConfiguration() - config.connection_config = { + config.configuration_properties = { "AES_GCM_V1": ["not", "a", "dict"] # invalid outer value (should be dict) } # Inner key is not a string with pytest.raises(TypeError, match="All inner config keys/values must be str"): config = ppe.ExternalDecryptionConfiguration() - config.connection_config = { + config.configuration_properties = { "AES_GCM_V1": { 123: "should-fail" # invalid inner key } @@ -450,7 +450,7 @@ def test_external_decryption_connection_config_invalid_types(): # Inner value is not a string with pytest.raises(TypeError, match="All inner config keys/values must be str"): config = ppe.ExternalDecryptionConfiguration() - config.connection_config = { + config.configuration_properties = { "AES_GCM_V1": { "config_file": ["not", "a", "string"] # invalid inner value } @@ -498,7 +498,7 @@ def get_custom_external_encryption_properties(encryption_algorithm, per_column_e "user_id": "Picard1701", "location": "Presidio" }, - connection_config={ + configuration_properties={ "EXTERNAL_DBPA_V1": { "config_file": "path/to/config/file", "config_file_decryption_key": "some_key", diff --git a/python/scripts/base_app.py b/python/scripts/base_app.py index c302772e6924..64c112f5ed0a 100644 --- a/python/scripts/base_app.py +++ b/python/scripts/base_app.py @@ -268,7 +268,7 @@ def get_external_encryption_config(plaintext_footer=True): "user_id": "Picard1701", "location": "Presidio" }, - connection_config = get_dbpa_connection_config() + configuration_properties = get_dbpa_configuration_properties() ) def get_encryption_config(plaintext_footer=True): @@ -294,7 +294,7 @@ def get_external_decryption_config(): "user_id": "Picard1701", "location": "Presidio" }, - connection_config = get_dbpa_connection_config() + configuration_properties = get_dbpa_configuration_properties() ) def get_config_file(): @@ -313,10 +313,10 @@ def get_config_file(): #did not find the file. return None and let the caller handle it. #throw an error - raise FileNotFoundError(f"Connection config [{config_file_name}] file not found") + raise FileNotFoundError(f"Configuration properties file [{config_file_name}] not found") -def get_dbpa_connection_config(): +def get_dbpa_configuration_properties(): #we read the name of the external DBPA agent library from the environment variable DBPA_LIBRARY_PATH. #if not available, we use the default to 'libDBPATestAgent.so'. #this library performs key-independent, XOR encryption/decryption, and is built as part of the Parquet Arrow tests. @@ -325,7 +325,7 @@ def get_dbpa_connection_config(): 'DBPA_LIBRARY_PATH', 'libDBPATestAgent.so' if platform.system() == 'Linux' else 'libDBPATestAgent.dylib') - connection_config = { + configuration_properties = { "EXTERNAL_DBPA_V1": { "agent_library_path": agent_library_path, "agent_init_timeout_ms": "15000", @@ -339,9 +339,9 @@ def get_dbpa_connection_config(): if (config_file_required): config_path = get_config_file() - connection_config["EXTERNAL_DBPA_V1"]["connection_config_file_path"] = config_path + configuration_properties["EXTERNAL_DBPA_V1"]["connection_config_file_path"] = config_path - return connection_config + return configuration_properties if __name__ == "__main__": diff --git a/python/scripts/use_external_dbpa_encryption.py b/python/scripts/use_external_dbpa_encryption.py index 4e4c7dc19160..64907fdd5da3 100644 --- a/python/scripts/use_external_dbpa_encryption.py +++ b/python/scripts/use_external_dbpa_encryption.py @@ -109,8 +109,8 @@ def get_external_encryption_config(use_remote_service): # Additional context for the external encryptor. Arrow will just forward this value to # the external encryptor, and does not read its contents. app_context=get_app_context(), - # Connection configuration for the external encryptor. - connection_config=get_dbpa_connection_config(use_remote_service) + # Configuration properties for the external encryptor. + configuration_properties=get_dbpa_configuration_properties(use_remote_service) ) def get_external_file_encryption_properties(external_encryption_config): @@ -133,8 +133,8 @@ def get_external_decryption_config(use_remote_service): cache_lifetime=datetime.timedelta(minutes=2.0), # Additional context for the external decryptor. app_context=get_app_context(), - # Connection configuration for the external decryptor. - connection_config=get_dbpa_connection_config(use_remote_service) + # Configuration properties for the external decryptor. + configuration_properties=get_dbpa_configuration_properties(use_remote_service) ) def get_external_file_decryption_properties(external_decryption_config): @@ -157,7 +157,7 @@ def get_app_context(): # ################################################################################################ """ -Set up the connection configuration for the external DBPA encryptor. +Set up the configuration properties for the external DBPA encryptor. Each application can provide its own timeout values for the external DBPA encryptor operations. If none are provided, default values are used on the encryptor side. These timeout values are not network related, but rather a protection mechanism to avoid the @@ -171,8 +171,8 @@ def get_app_context(): This includes the server URL and the authentication credentials, which the application must procure on its own. """ -def get_dbpa_connection_config(use_remote_service): - connection_config = { +def get_dbpa_configuration_properties(use_remote_service): + configuration_properties = { "EXTERNAL_DBPA_V1": { "agent_init_timeout_ms": "15000", "agent_encrypt_timeout_ms": "35000", @@ -182,16 +182,16 @@ def get_dbpa_connection_config(use_remote_service): if use_remote_service: agent_library_path = ( 'libdbpsRemoteAgent.so' if platform.system() == 'Linux' else 'libdbpsRemoteAgent.dylib') - connection_config["EXTERNAL_DBPA_V1"]["agent_library_path"] = agent_library_path + configuration_properties["EXTERNAL_DBPA_V1"]["agent_library_path"] = agent_library_path # Make sure this is the absolute path to the connection config file. remote_file_path = '/arrowdev/python/scripts/test_connection_config_file.json' - connection_config["EXTERNAL_DBPA_V1"]["connection_config_file_path"] = remote_file_path + configuration_properties["EXTERNAL_DBPA_V1"]["connection_config_file_path"] = remote_file_path else: agent_library_path = ( 'libdbpsLocalAgent.so' if platform.system() == 'Linux' else 'libdbpsLocalAgent.dylib') - connection_config["EXTERNAL_DBPA_V1"]["agent_library_path"] = agent_library_path + configuration_properties["EXTERNAL_DBPA_V1"]["agent_library_path"] = agent_library_path - return connection_config + return configuration_properties # ################################################################################################