diff --git a/cpp/src/parquet/CMakeLists.txt b/cpp/src/parquet/CMakeLists.txt index 09d00c069d02..fa9e918cccf3 100644 --- a/cpp/src/parquet/CMakeLists.txt +++ b/cpp/src/parquet/CMakeLists.txt @@ -239,10 +239,11 @@ endif() if(PARQUET_REQUIRE_ENCRYPTION) list(APPEND PARQUET_SHARED_PRIVATE_LINK_LIBS ${ARROW_OPENSSL_LIBS}) set(PARQUET_SRCS ${PARQUET_SRCS} encryption/aes_encryption.cc + encryption/openssl_internal.cc encryption/encryption_utils.cc encryption/external_dbpa_encryption.cc - encryption/openssl_internal.cc encryption/external/dbpa_library_wrapper.cc + encryption/external/dbpa_utils.cc encryption/external/loadable_encryptor_utils.cc ) # Encryption key management diff --git a/cpp/src/parquet/encryption/external/dbpa_utils.h b/cpp/src/parquet/encryption/external/dbpa_utils.h index 66cad649a793..b1ad22c7057e 100644 --- a/cpp/src/parquet/encryption/external/dbpa_utils.h +++ b/cpp/src/parquet/encryption/external/dbpa_utils.h @@ -5,7 +5,7 @@ #include -#include "parquet/encryption/external/borrowed/dbpa_interface.h" +#include "parquet/encryption/external/third_party/dbpa_interface.h" #include "parquet/types.h" #include "arrow/type_fwd.h" // For arrow::Compression diff --git a/cpp/src/parquet/encryption/external_dbpa_encryption.cc b/cpp/src/parquet/encryption/external_dbpa_encryption.cc index 4caef16157ca..902c7b7747cf 100644 --- a/cpp/src/parquet/encryption/external_dbpa_encryption.cc +++ b/cpp/src/parquet/encryption/external_dbpa_encryption.cc @@ -1,15 +1,75 @@ // What license shall we use for this file? #include +#include #include "parquet/encryption/external_dbpa_encryption.h" #include "parquet/encryption/key_metadata.h" +#include "parquet/encryption/external/loadable_encryptor_utils.h" +#include "parquet/encryption/external/dbpa_utils.h" +#include "parquet/exception.h" +#include "parquet/types.h" -/// TODO(sbrenes): Add proper implementation. Right now we are just going to return -/// the plaintext as the ciphertext. +using parquet::encryption::external::LoadableEncryptorUtils; +using parquet::encryption::external::dbpa_utils; + +using dbps::external::EncryptionResult; +using dbps::external::DecryptionResult; namespace parquet::encryption { +// Utility function to load and initialize a DataBatchProtectionAgentInterface instance +// Shared between the encryptor and decryptor. +std::unique_ptr LoadAndInitializeAgent( + const std::string& column_name, + const std::map& connection_config, + const std::string& app_context, + const std::string& key_id, + Type::type data_type, + Compression::type compression_type) { + + // Load a new DataBatchProtectionAgentInterface instance from the shared library + std::cout << "[DEBUG] Loading agent from library..." << std::endl; + + const std::string SHARED_LIBRARY_PATH_KEY = "agent_library_path"; + + // Step 1: Get path to the shared library + auto it = connection_config.find(SHARED_LIBRARY_PATH_KEY); + if (it == connection_config.end()) { + auto const msg = "Required configuration key '" + SHARED_LIBRARY_PATH_KEY + "' not found in connection_config"; + std::cout << "[ERROR] " << msg << std::endl; + throw ParquetException(msg); + } + auto library_path = it->second; + std::cout << "[DEBUG] library_path = " << library_path << std::endl; + + // Step 2: Load an instance of the DataBatchProtectionAgentInterface + std::cout << "[DEBUG] Loading agent instance from library..." << std::endl; + auto agent_instance = LoadableEncryptorUtils::LoadFromLibrary(library_path); + if (!agent_instance) { + std::cout << "[ERROR] Failed to create instance of DataBatchProtectionAgentInterface" << std::endl; + throw ParquetException("Failed to create instance of DataBatchProtectionAgentInterface"); + } + std::cout << "[DEBUG] Successfully loaded agent_instance" << std::endl; + std::cout << "[DEBUG] Initializing agent instance" << std::endl; + + // Step 3: Initialize the agent. + agent_instance->init( + /*column_name*/ column_name, + /*connection_config*/ connection_config, + /*app_context*/ app_context, + /*column_key_id*/ key_id, + /*data_type*/ dbpa_utils::ParquetTypeToExternal(data_type), + /*compression_type*/ dbpa_utils::ArrowCompressionToExternal(compression_type) + ); //LoadAndInitializeAgent() + + //TODO: what to do if agent was not initialized? should exception be thrown? + + std::cout << "[DEBUG] Successfully initialized agent instance" << std::endl; + + return agent_instance; +} + ExternalDBPAEncryptorAdapter::ExternalDBPAEncryptorAdapter( ParquetCipher::type algorithm, std::string column_name, std::string key_id, Type::type data_type, Compression::type compression_type, Encoding::type encoding_type, @@ -17,7 +77,32 @@ ExternalDBPAEncryptorAdapter::ExternalDBPAEncryptorAdapter( : algorithm_(algorithm), column_name_(column_name), key_id_(key_id), data_type_(data_type), compression_type_(compression_type), encoding_type_(encoding_type), app_context_(app_context), - connection_config_(connection_config) {} + connection_config_(connection_config) { + + //TODO: figure out logging + std::cout << "[DEBUG] ExternalDBPAEncryptorAdapter::ExternalDBPAEncryptorAdapter() -- constructor" << std::endl; + std::cout << "[DEBUG] algorithm = " << algorithm << std::endl; + std::cout << "[DEBUG] column_name = " << column_name << std::endl; + std::cout << "[DEBUG] key_id = " << key_id << std::endl; + std::cout << "[DEBUG] data_type = " << data_type << std::endl; + std::cout << "[DEBUG] compression_type = " << compression_type << std::endl; + std::cout << "[DEBUG] encoding_type = " << encoding_type << std::endl; + std::cout << "[DEBUG] app_context = " << app_context << std::endl; + std::cout << "[DEBUG] connection_config:" << std::endl; + for (const auto& [key, value] : connection_config) { + std::cout << "[DEBUG] " << key << " = " << value << std::endl; + } + + std::cout << "[DEBUG] ExternalDBPAEncryptorAdapter::ExternalDBPAEncryptorAdapter() -- loading and initializing agent" << std::endl; + // Load and initialize the agent using the utility function + agent_instance_ = LoadAndInitializeAgent( + column_name, connection_config, app_context, key_id, data_type, compression_type); + + //TODO: what to do if agent was not initialized? + agent_initialized_ = true; + + std::cout << "[DEBUG] Successfully loaded and initialized agent" << std::endl; + } std::unique_ptr ExternalDBPAEncryptorAdapter::Make( ParquetCipher::type algorithm, std::string column_name, std::string key_id, @@ -29,47 +114,71 @@ std::unique_ptr ExternalDBPAEncryptorAdapter::Make } int32_t ExternalDBPAEncryptorAdapter::CiphertextLength(int64_t plaintext_len) const { + std::cout << "ExternalDBPAEncryptorAdapter::CiphertextLength" << std::endl; + std::cout << " plaintext_len = " << plaintext_len << std::endl; + int return_value = plaintext_len; + std::cout << " return_value = " << return_value << std::endl; return plaintext_len; } int32_t ExternalDBPAEncryptorAdapter::Encrypt( ::arrow::util::span plaintext, ::arrow::util::span key, ::arrow::util::span aad, ::arrow::util::span ciphertext) { - return CallExternalDBPA(plaintext, ciphertext); + + return InvokeExternalEncrypt(plaintext, ciphertext); } int32_t ExternalDBPAEncryptorAdapter::SignedFooterEncrypt( ::arrow::util::span footer, ::arrow::util::span key, ::arrow::util::span aad, ::arrow::util::span nonce, ::arrow::util::span encrypted_footer) { - return CallExternalDBPA(footer, encrypted_footer); + return InvokeExternalEncrypt(footer, encrypted_footer); } -int32_t ExternalDBPAEncryptorAdapter::CallExternalDBPA( +int32_t ExternalDBPAEncryptorAdapter::InvokeExternalEncrypt( ::arrow::util::span plaintext, ::arrow::util::span ciphertext) { - std::cout << "\n*-*-*- START: ExternalDBPAEncryptor::Encrypt Hello World! *-*-*-" << std::endl; - std::cout << "Encryption Algorithm: [" << algorithm_ << "]" << std::endl; - std::cout << "Column Name: [" << column_name_ << "]" << std::endl; - std::cout << "Key ID: [" << key_id_ << "]" << std::endl; - std::cout << "Data Type: [" << data_type_ << "]" << std::endl; - std::cout << "Compression Type: [" << compression_type_ << "]" << std::endl; - std::cout << "Encoding Type: [" << encoding_type_ << "]" << std::endl; - std::cout << "App Context: [" << app_context_ << "]" << std::endl; - std::cout << "Connection Config:" << std::endl; - for (const auto& [key, value] : connection_config_) { - std::cout << " [" << key << "]: [" << value << "]" << std::endl; - } - std::copy(plaintext.begin(), plaintext.end(), ciphertext.begin()); - - std::string plaintext_str(plaintext.begin(), plaintext.end()); - std::string ciphertext_str(ciphertext.begin(), ciphertext.end()); - std::cout << "Plaintext: [" << plaintext_str << "]" << std::endl; - std::cout << "Ciphertext: [" << ciphertext_str << "]" << std::endl; - std::cout << "*-*-*- END: ExternalDBPAEncryptor::Encrypt Hello World! *-*-*-\n" << std::endl; + std::cout << "\n*-*-*- START: ExternalDBPAEncryptor::Encrypt Hello World! *-*-*-" << std::endl; + std::cout << "Encryption Algorithm: [" << algorithm_ << "]" << std::endl; + std::cout << "Column Name: [" << column_name_ << "]" << std::endl; + std::cout << "Key ID: [" << key_id_ << "]" << std::endl; + std::cout << "Data Type: [" << data_type_ << "]" << std::endl; + std::cout << "Compression Type: [" << compression_type_ << "]" << std::endl; + std::cout << "Encoding Type: [" << encoding_type_ << "]" << std::endl; + std::cout << "App Context: [" << app_context_ << "]" << std::endl; + std::cout << "Connection Config:" << std::endl; + for (const auto& [cfg_key, cfg_value] : connection_config_) { + std::cout << " [" << cfg_key << "]: [" << cfg_value << "]" << std::endl; + } + + if (!agent_initialized_) { + throw ParquetException("Underlying DBPA Agent was not initialized"); + } - return ciphertext.size(); -} + std::cout << "[DEBUG] Calling agent_instance_->Encrypt..." << std::endl; + std::unique_ptr result = agent_instance_->Encrypt(plaintext); + + if (!result->success()) { + std::cout << "[ERROR] Encryption failed: " << result->error_message() << std::endl; + throw ParquetException(result->error_message()); + } + + std::cout << "[DEBUG] Encryption successful" << std::endl; + std::cout << " result size: " << result->size() << " bytes" << std::endl; + std::cout << " result ciphertext size: " << result->ciphertext().size() << " bytes" << std::endl; + + if (ciphertext.size() < result->ciphertext().size()) { + std::cout << "[ERROR] Ciphertext buffer too small. Need " << result->ciphertext().size() + << " bytes, have " << ciphertext.size() << " bytes" << std::endl; + throw ParquetException("Ciphertext buffer too small for encrypted result"); + } + + std::cout << "[DEBUG] Copying result to ciphertext buffer..." << std::endl; + std::copy(result->ciphertext().begin(), result->ciphertext().end(), ciphertext.begin()); + std::cout << "[DEBUG] Encryption completed successfully" << std::endl; + + return static_cast(result->size()); + } ExternalDBPAEncryptorAdapter* ExternalDBPAEncryptorAdapterFactory::GetEncryptor( ParquetCipher::type algorithm, const ColumnChunkMetaDataBuilder* column_chunk_metadata, @@ -126,7 +235,36 @@ ExternalDBPADecryptorAdapter::ExternalDBPADecryptorAdapter( : algorithm_(algorithm), column_name_(column_name), key_id_(key_id), data_type_(data_type), compression_type_(compression_type), encoding_types_(encoding_types), app_context_(app_context), - connection_config_(connection_config) {} + connection_config_(connection_config) { + + std::cout << "[DEBUG] ExternalDBPADecryptorAdapter::ExternalDBPADecryptorAdapter() -- constructor" << std::endl; + std::cout << "[DEBUG] algorithm = " << algorithm << std::endl; + std::cout << "[DEBUG] column_name = " << column_name << std::endl; + std::cout << "[DEBUG] key_id = " << key_id << std::endl; + std::cout << "[DEBUG] data_type = " << data_type << std::endl; + std::cout << "[DEBUG] compression_type = " << compression_type << std::endl; + std::cout << "[DEBUG] encoding_types = ["; + for (const auto& encoding : encoding_types) { + std::cout << static_cast(encoding) << " "; + } + std::cout << "]" << std::endl; + std::cout << "[DEBUG] app_context = " << app_context << std::endl; + std::cout << "[DEBUG] connection_config:" << std::endl; + for (const auto& [key, value] : connection_config) { + std::cout << "[DEBUG] " << key << " = " << value << std::endl; + } + + std::cout << "[DEBUG] ExternalDBPADecryptorAdapter::ExternalDBPADecryptorAdapter() -- loading and initializing agent" << std::endl; + + // Load and initialize the agent using the utility function + agent_instance_ = LoadAndInitializeAgent( + column_name, connection_config, app_context, key_id, data_type, compression_type); + + //TODO: what to do if agent was not initialized? + agent_initialized_ = true; + + std::cout << "[DEBUG] Successfully loaded and initialized agent" << std::endl; + } std::unique_ptr ExternalDBPADecryptorAdapter::Make( ParquetCipher::type algorithm, std::string column_name, std::string key_id, @@ -149,38 +287,58 @@ int32_t ExternalDBPADecryptorAdapter::CiphertextLength(int32_t plaintext_len) co int32_t ExternalDBPADecryptorAdapter::Decrypt( ::arrow::util::span ciphertext, ::arrow::util::span key, ::arrow::util::span aad, ::arrow::util::span plaintext) { - return CallExternalDBPA(ciphertext, plaintext); + + return InvokeExternalDecrypt(ciphertext, plaintext); } -int32_t ExternalDBPADecryptorAdapter::CallExternalDBPA( +int32_t ExternalDBPADecryptorAdapter::InvokeExternalDecrypt( ::arrow::util::span ciphertext, ::arrow::util::span plaintext) { - std::cout << "\n*-*-*- START: ExternalDBPADecryptor::Decrypt Hello World! *-*-*-" << std::endl; - std::cout << "Decryption Algorithm: [" << algorithm_ << "]" << std::endl; - std::cout << "Column Name: [" << column_name_ << "]" << std::endl; - std::cout << "Key ID: [" << key_id_ << "]" << std::endl; - std::cout << "Data Type: [" << data_type_ << "]" << std::endl; - std::cout << "Compression Type: [" << compression_type_ << "]" << std::endl; - std::cout << "Encoding Types: ["; - for (const auto& encoding_type : encoding_types_) { - std::cout << encoding_type << "\n"; - } - std::cout << "]" << std::endl; - std::cout << "App Context: [" << app_context_ << "]" << std::endl; - std::cout << "Connection Config:" << std::endl; - for (const auto& [key, value] : connection_config_) { - std::cout << " [" << key << "]: [" << value << "]" << std::endl; - } - - std::copy(ciphertext.begin(), ciphertext.end(), plaintext.begin()); - std::string plaintext_str(plaintext.begin(), plaintext.end()); - std::string ciphertext_str(ciphertext.begin(), ciphertext.end()); - std::cout << "Plaintext: [" << plaintext_str << "]" << std::endl; - std::cout << "Ciphertext: [" << ciphertext_str << "]" << std::endl; - std::cout << "*-*-*- END: ExternalDBPADecryptor::Decrypt Hello World! *-*-*-\n" << std::endl; + std::cout << "\n*-*-*- START: ExternalDBPADecryptor::Decrypt Hello World! *-*-*-" << std::endl; + std::cout << "Decryption Algorithm: [" << algorithm_ << "]" << std::endl; + std::cout << "Column Name: [" << column_name_ << "]" << std::endl; + std::cout << "Key ID: [" << key_id_ << "]" << std::endl; + std::cout << "Data Type: [" << data_type_ << "]" << std::endl; + std::cout << "Compression Type: [" << compression_type_ << "]" << std::endl; + std::cout << "Encoding Types: ["; + for (const auto& encoding_type : encoding_types_) { + std::cout << encoding_type << "\n"; + } + std::cout << "]" << std::endl; + std::cout << "App Context: [" << app_context_ << "]" << std::endl; + std::cout << "Connection Config:" << std::endl; + for (const auto& [key, value] : connection_config_) { + std::cout << " [" << key << "]: [" << value << "]" << std::endl; + } + + if (!agent_initialized_) { + throw ParquetException("Underlying DBPA Agent was not initialized"); + } - return plaintext.size(); -} + std::cout << "[DEBUG] Calling agent_instance_->Decrypt..." << std::endl; + std::unique_ptr result = agent_instance_->Decrypt(ciphertext); + + if (!result->success()) { + std::cout << "[ERROR] Decryption failed: " << result->error_message() << std::endl; + throw ParquetException(result->error_message()); + } + + std::cout << "[DEBUG] Decryption successful" << std::endl; + std::cout << " result size: " << result->size() << " bytes" << std::endl; + std::cout << " result plaintext size: " << result->plaintext().size() << " bytes" << std::endl; + + if (plaintext.size() < result->plaintext().size()) { + std::cout << "[ERROR] Plaintext buffer too small. Need " << result->plaintext().size() + << " bytes, have " << plaintext.size() << " bytes" << std::endl; + throw ParquetException("Plaintext buffer too small for decrypted result"); + } + + std::cout << "[DEBUG] Copying result to plaintext buffer..." << std::endl; + std::copy(result->plaintext().begin(), result->plaintext().end(), plaintext.begin()); + std::cout << "[DEBUG] Decryption completed successfully" << std::endl; + + return result->size(); + } std::unique_ptr ExternalDBPADecryptorAdapterFactory::GetDecryptor( ParquetCipher::type algorithm, const ColumnCryptoMetaData* crypto_metadata, diff --git a/cpp/src/parquet/encryption/external_dbpa_encryption.h b/cpp/src/parquet/encryption/external_dbpa_encryption.h index 169e1d5595de..151ef4fd5a00 100644 --- a/cpp/src/parquet/encryption/external_dbpa_encryption.h +++ b/cpp/src/parquet/encryption/external_dbpa_encryption.h @@ -5,6 +5,8 @@ #include #include +#include "parquet/encryption/external/third_party/dbpa_interface.h" + #include "parquet/encryption/encryptor_interface.h" #include "parquet/encryption/decryptor_interface.h" #include "parquet/metadata.h" @@ -46,7 +48,7 @@ class ExternalDBPAEncryptorAdapter : public EncryptorInterface { ::arrow::util::span encrypted_footer) override; private: - int32_t CallExternalDBPA( + int32_t InvokeExternalEncrypt( ::arrow::util::span plaintext, ::arrow::util::span ciphertext); ParquetCipher::type algorithm_; @@ -57,6 +59,9 @@ class ExternalDBPAEncryptorAdapter : public EncryptorInterface { Encoding::type encoding_type_; std::string app_context_; std::map connection_config_; + + std::unique_ptr agent_instance_; + bool agent_initialized_ = false; }; /// Factory for ExternalDBPAEncryptorAdapter instances. The cache exists while the write @@ -104,7 +109,7 @@ class ExternalDBPADecryptorAdapter : public DecryptorInterface { ::arrow::util::span plaintext) override; private: - int32_t CallExternalDBPA( + int32_t InvokeExternalDecrypt( ::arrow::util::span ciphertext, ::arrow::util::span plaintext); ParquetCipher::type algorithm_; @@ -116,6 +121,9 @@ class ExternalDBPADecryptorAdapter : public DecryptorInterface { std::vector encoding_types_; std::string app_context_; std::map connection_config_; + + std::unique_ptr agent_instance_; + bool agent_initialized_ = false; }; /// Factory for ExternalDBPADecryptorAdapter instances. No cache exists for decryptors. diff --git a/cpp/src/parquet/encryption/external_dbpa_encryption_test.cc b/cpp/src/parquet/encryption/external_dbpa_encryption_test.cc index 5f5b16748581..730da0579633 100644 --- a/cpp/src/parquet/encryption/external_dbpa_encryption_test.cc +++ b/cpp/src/parquet/encryption/external_dbpa_encryption_test.cc @@ -17,7 +17,8 @@ class ExternalDBPAEncryptorAdapterTest : public ::testing::Test { "{\"user_id\": \"abc123\", \"location\": {\"lat\": 9.7489, \"lon\": -83.7534}}"; connection_config_ = { {"lib_name", "dbpa_lib.so"}, - {"config_path", "path/to/file"} + {"config_path", "path/to/file"}, + {"agent_library_path", "libDBPATestAgent.so"} }; } @@ -25,9 +26,9 @@ class ExternalDBPAEncryptorAdapterTest : public ::testing::Test { ParquetCipher::type algorithm, std::string column_name, std::string key_id, Type::type data_type, Compression::type compression_type, Encoding::type encoding_type, std::string plaintext) { - ExternalDBPAEncryptorAdapter encryptor( - algorithm, column_name, key_id, data_type, compression_type, encoding_type, - app_context_, connection_config_); + ExternalDBPAEncryptorAdapter encryptor(algorithm, column_name, key_id, data_type, + compression_type, encoding_type, app_context_, + connection_config_); int32_t expected_ciphertext_length = plaintext.size(); int32_t actual_ciphertext_length = encryptor.CiphertextLength(plaintext.size()); @@ -39,7 +40,19 @@ class ExternalDBPAEncryptorAdapterTest : public ::testing::Test { ASSERT_EQ(expected_ciphertext_length, encryption_length); std::string ciphertext_str(ciphertext_buffer.begin(), ciphertext_buffer.end()); - ASSERT_EQ(plaintext, ciphertext_str); + + // We know this uses XOR encryption. Therefore, the ciphertext is the same as the plaintext. + // XOR encrytion encrypts each byte of the plaintext with 0xAA. + // See external/dbpa_test_agent.cc for the implementation. + + // Assert that plaintext and ciphertext have the same length + ASSERT_EQ(plaintext.size(), ciphertext_str.size()); + + // Assert that ciphertext is plaintext XOR'd with 0xAA + for (size_t i = 0; i < plaintext.size(); i++) { + ASSERT_EQ(static_cast(ciphertext_str[i]), + static_cast(plaintext[i]) ^ 0xAA); + } ExternalDBPADecryptorAdapter decryptor(algorithm, column_name, key_id, data_type, compression_type, {encoding_type}, app_context_, @@ -55,6 +68,8 @@ class ExternalDBPAEncryptorAdapterTest : public ::testing::Test { ASSERT_EQ(expected_plaintext_length, decryption_length); std::string plaintext_str(plaintext_buffer.begin(), plaintext_buffer.end()); + + // Assert that the decrypted plaintext matches the original plaintext ASSERT_EQ(plaintext, plaintext_str); } diff --git a/cpp/src/parquet/encryption/read_configurations_test.cc b/cpp/src/parquet/encryption/read_configurations_test.cc index 7e0602ddd079..8479b14b9a51 100644 --- a/cpp/src/parquet/encryption/read_configurations_test.cc +++ b/cpp/src/parquet/encryption/read_configurations_test.cc @@ -184,6 +184,7 @@ class TestDecryptionConfiguration file_decryption_builder_5.key_retriever(kr5); file_decryption_builder_5.connection_config({ {parquet::ParquetCipher::EXTERNAL_DBPA_V1, { + {"agent_library_path", "libDBPATestAgent.so"}, {"file_path", "/tmp/test"}, {"other_config", "value"} }} diff --git a/cpp/src/parquet/encryption/write_configurations_test.cc b/cpp/src/parquet/encryption/write_configurations_test.cc index d032f5296bcd..8d2bcdd59682 100644 --- a/cpp/src/parquet/encryption/write_configurations_test.cc +++ b/cpp/src/parquet/encryption/write_configurations_test.cc @@ -244,6 +244,7 @@ TEST_F(TestEncryptionConfiguration, EncryptWithPerColumnEncryption) { ->algorithm(parquet::ParquetCipher::AES_GCM_V1) ->connection_config({ {parquet::ParquetCipher::EXTERNAL_DBPA_V1, { + {"agent_library_path", "libDBPATestAgent.so"}, {"file_path", "/tmp/test"}, {"other_config", "value"} }}