From 48d2e6dae4bee94d82968fb412f3fe119d5f910f Mon Sep 17 00:00:00 2001 From: Marco Arguedas Date: Wed, 17 Dec 2025 17:29:56 -0600 Subject: [PATCH 01/10] Adding necessary boost packages to Ubuntu images (CI/Github Actions) --- ci/docker/ubuntu-22.04-cpp.dockerfile | 1 + ci/docker/ubuntu-24.04-cpp.dockerfile | 1 + 2 files changed, 2 insertions(+) diff --git a/ci/docker/ubuntu-22.04-cpp.dockerfile b/ci/docker/ubuntu-22.04-cpp.dockerfile index 88a27efe335d..6cc8b4e11ff6 100644 --- a/ci/docker/ubuntu-22.04-cpp.dockerfile +++ b/ci/docker/ubuntu-22.04-cpp.dockerfile @@ -72,6 +72,7 @@ RUN apt-get update -y -q && \ gdb \ git \ libbenchmark-dev \ + libboost-date-time-dev \ libboost-filesystem-dev \ libboost-system-dev \ libbrotli-dev \ diff --git a/ci/docker/ubuntu-24.04-cpp.dockerfile b/ci/docker/ubuntu-24.04-cpp.dockerfile index 0347d452d7bf..126e1d852479 100644 --- a/ci/docker/ubuntu-24.04-cpp.dockerfile +++ b/ci/docker/ubuntu-24.04-cpp.dockerfile @@ -73,6 +73,7 @@ RUN apt-get update -y -q && \ gdb \ git \ libbenchmark-dev \ + libboost-date-time-dev \ libboost-filesystem-dev \ libboost-system-dev \ libbrotli-dev \ From c6727f25dbe759f15ebf9f859b69a1b6ffeef76e Mon Sep 17 00:00:00 2001 From: Marco Arguedas Date: Wed, 17 Dec 2025 18:12:33 -0600 Subject: [PATCH 02/10] Updating to the latest version (SHA/tag) of DBPS --- cpp/src/parquet/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/parquet/CMakeLists.txt b/cpp/src/parquet/CMakeLists.txt index 40b3d08ea243..1faa96d291eb 100644 --- a/cpp/src/parquet/CMakeLists.txt +++ b/cpp/src/parquet/CMakeLists.txt @@ -285,7 +285,7 @@ if(PARQUET_REQUIRE_ENCRYPTION) #TODO: Change to a specific tag/commit when we have one. #https://github.com/protegrity/arrow/issues/179 - GIT_TAG be87857e4d8c40977c3143c57805c7cc1c8394a8 + GIT_TAG 4c808b2233ed0bc04529c3b0dbf7c214c4901043 GIT_SHALLOW FALSE ) From 7f23e432a77a78e8094b919b0e1cc434eb0aa147 Mon Sep 17 00:00:00 2001 From: Marco Arguedas Date: Wed, 17 Dec 2025 19:24:44 -0600 Subject: [PATCH 03/10] Updating dbpa_executor_test.cc and dbpa_library_wrapper_test.cc to latest version of DBPA interface --- .../encryption/external/dbpa_executor_test.cc | 114 +++---- .../external/dbpa_library_wrapper_test.cc | 301 ++++++++++-------- 2 files changed, 230 insertions(+), 185 deletions(-) diff --git a/cpp/src/parquet/encryption/external/dbpa_executor_test.cc b/cpp/src/parquet/encryption/external/dbpa_executor_test.cc index fa527d358222..4cecd9fdbfd5 100644 --- a/cpp/src/parquet/encryption/external/dbpa_executor_test.cc +++ b/cpp/src/parquet/encryption/external/dbpa_executor_test.cc @@ -33,9 +33,11 @@ class TestEncryptionResult : public EncryptionResult { public: TestEncryptionResult(std::vector data, bool success = true, std::string error_msg = "", - std::map error_fields = {}) + std::map error_fields = {}, + std::optional> metadata = std::nullopt) : ciphertext_data_(std::move(data)), success_(success), - error_message_(std::move(error_msg)), error_fields_(std::move(error_fields)) {} + error_message_(std::move(error_msg)), error_fields_(std::move(error_fields)), + metadata_(std::move(metadata)) {} span ciphertext() const override { return span(ciphertext_data_.data(), ciphertext_data_.size()); @@ -43,6 +45,9 @@ class TestEncryptionResult : public EncryptionResult { std::size_t size() const override { return ciphertext_data_.size(); } bool success() const override { return success_; } + const std::optional> encryption_metadata() const override { + return metadata_; + } const std::string& error_message() const override { return error_message_; } const std::map& error_fields() const override { return error_fields_; } @@ -51,6 +56,7 @@ class TestEncryptionResult : public EncryptionResult { bool success_; std::string error_message_; std::map error_fields_; + std::optional> metadata_; }; class TestDecryptionResult : public DecryptionResult { @@ -84,7 +90,7 @@ class MockDBPAAgent : public DataBatchProtectionAgentInterface { int init_call_count_ = 0; int encrypt_call_count_ = 0; int decrypt_call_count_ = 0; - + // Track parameters from last calls std::string last_init_column_name_; std::map last_init_connection_config_; @@ -96,20 +102,20 @@ class MockDBPAAgent : public DataBatchProtectionAgentInterface { std::optional> last_init_column_encryption_metadata_; std::map last_encrypt_encoding_attrs_; std::map last_decrypt_encoding_attrs_; - + std::vector last_encrypt_plaintext_; std::vector last_decrypt_ciphertext_; - + // Mock results std::unique_ptr mock_encrypt_result_; std::unique_ptr mock_decrypt_result_; - + // Control behavior bool should_throw_on_init_ = false; bool should_throw_on_encrypt_ = false; bool should_throw_on_decrypt_ = false; std::string throw_message_ = "Mock agent error"; - + void init(std::string column_name, std::map connection_config, std::string app_context, @@ -127,22 +133,22 @@ class MockDBPAAgent : public DataBatchProtectionAgentInterface { last_init_compression_type_ = compression_type; last_init_datatype_length_ = datatype_length; last_init_column_encryption_metadata_ = std::move(column_encryption_metadata); - + if (should_throw_on_init_) { throw std::runtime_error(throw_message_); } } - + std::unique_ptr Encrypt(span plaintext, std::map encoding_attributes) override { encrypt_call_count_++; last_encrypt_plaintext_.assign(plaintext.begin(), plaintext.end()); last_encrypt_encoding_attrs_ = std::move(encoding_attributes); - + if (should_throw_on_encrypt_) { throw std::runtime_error(throw_message_); } - + // Return a copy of the mock result if available if (mock_encrypt_result_) { // Create a new result with the same data @@ -150,17 +156,17 @@ class MockDBPAAgent : public DataBatchProtectionAgentInterface { } return nullptr; } - + std::unique_ptr Decrypt(span ciphertext, std::map encoding_attributes) override { decrypt_call_count_++; last_decrypt_ciphertext_.assign(ciphertext.begin(), ciphertext.end()); last_decrypt_encoding_attrs_ = std::move(encoding_attributes); - + if (should_throw_on_decrypt_) { throw std::runtime_error(throw_message_); } - + // Return a copy of the mock result if available if (mock_decrypt_result_) { // Create a new result with the same data @@ -168,18 +174,18 @@ class MockDBPAAgent : public DataBatchProtectionAgentInterface { } return nullptr; } - + // Helper methods for test setup void ResetCallCounts() { init_call_count_ = 0; encrypt_call_count_ = 0; decrypt_call_count_ = 0; } - + void SetMockEncryptResult(std::unique_ptr result) { mock_encrypt_result_ = std::move(result); } - + void SetMockDecryptResult(std::unique_ptr result) { mock_decrypt_result_ = std::move(result); } @@ -191,7 +197,7 @@ class DBPAExecutorTest : public ::testing::Test { // Create a mock agent to wrap mock_agent_ = std::make_unique(); mock_agent_ptr_ = mock_agent_.get(); // Keep raw pointer for verification - + // Create the executor that wraps the mock agent with custom timeouts executor_ = std::make_unique(std::move(mock_agent_), 1000, // init timeout: 1 second @@ -210,10 +216,10 @@ TEST_F(DBPAExecutorTest, ConstructorWithNullAgentThrows) { TEST_F(DBPAExecutorTest, ConstructorWithInvalidTimeoutsThrows) { auto test_agent = std::make_unique(); - + // Test negative timeout EXPECT_THROW(DBPAExecutor(std::move(test_agent), -1, 1000, 1000), std::invalid_argument); - + // Test zero timeout test_agent = std::make_unique(); EXPECT_THROW(DBPAExecutor(std::move(test_agent), 0, 1000, 1000), std::invalid_argument); @@ -229,15 +235,15 @@ TEST_F(DBPAExecutorTest, InitForwardsToWrappedAgent) { // Reset call count before test mock_agent_ptr_->ResetCallCounts(); - + // Call init through executor std::optional> column_encryption_metadata = std::map{{"metaKey", "metaValue"}}; EXPECT_NO_THROW(executor_->init(column_name, connection_config, app_context, column_key_id, data_type, std::nullopt, compression_type, column_encryption_metadata)); - + // Verify the mock agent was called exactly once EXPECT_EQ(mock_agent_ptr_->init_call_count_, 1); - + // Verify all parameters were forwarded correctly EXPECT_EQ(mock_agent_ptr_->last_init_column_name_, column_name); EXPECT_EQ(mock_agent_ptr_->last_init_connection_config_, connection_config); @@ -260,18 +266,18 @@ TEST_F(DBPAExecutorTest, EncryptForwardsToWrappedAgent) { // Reset call count before test mock_agent_ptr_->ResetCallCounts(); - + // Encrypt should not throw and should return a result std::map attrs = {{"format", "plain"}}; auto result = executor_->Encrypt(plaintext_span, attrs); - + // Verify the mock agent was called exactly once EXPECT_EQ(mock_agent_ptr_->encrypt_call_count_, 1); - + // Verify the plaintext was forwarded correctly EXPECT_EQ(mock_agent_ptr_->last_encrypt_plaintext_, plaintext); EXPECT_EQ(mock_agent_ptr_->last_encrypt_encoding_attrs_, attrs); - + // Note: result might be nullptr since we didn't set up a mock result // The important thing is that the call was forwarded } @@ -287,18 +293,18 @@ TEST_F(DBPAExecutorTest, DecryptForwardsToWrappedAgent) { // Reset call count before test mock_agent_ptr_->ResetCallCounts(); - + // Decrypt should not throw and should return a result std::map attrs = {{"format", "plain"}}; auto result = executor_->Decrypt(ciphertext_span, attrs); - + // Verify the mock agent was called exactly once EXPECT_EQ(mock_agent_ptr_->decrypt_call_count_, 1); - + // Verify the ciphertext was forwarded correctly EXPECT_EQ(mock_agent_ptr_->last_decrypt_ciphertext_, ciphertext); EXPECT_EQ(mock_agent_ptr_->last_decrypt_encoding_attrs_, attrs); - + // Note: result might be nullptr since we didn't set up a mock result // The important thing is that the call was forwarded } @@ -316,14 +322,14 @@ TEST_F(DBPAExecutorTest, InitForwardsDatatypeLength) { TEST_F(DBPAExecutorTest, MultipleCallsAreProperlyForwarded) { // Reset call counts mock_agent_ptr_->ResetCallCounts(); - + // Make multiple init calls with different parameters executor_->init("column1", {{"key1", "value1"}}, "context1", "key1", Type::type::INT32, std::nullopt, CompressionCodec::type::UNCOMPRESSED, std::nullopt); - + // Verify both calls were made EXPECT_EQ(mock_agent_ptr_->init_call_count_, 1); - + // Verify the last call parameters (second call) EXPECT_EQ(mock_agent_ptr_->last_init_column_name_, "column1"); EXPECT_EQ(mock_agent_ptr_->last_init_connection_config_["key1"], "value1"); @@ -331,34 +337,34 @@ TEST_F(DBPAExecutorTest, MultipleCallsAreProperlyForwarded) { EXPECT_EQ(mock_agent_ptr_->last_init_column_key_id_, "key1"); EXPECT_EQ(mock_agent_ptr_->last_init_data_type_, Type::type::INT32); EXPECT_EQ(mock_agent_ptr_->last_init_compression_type_, CompressionCodec::type::UNCOMPRESSED); - + // Make multiple encrypt calls std::vector data1 = {1, 2, 3}; std::vector data2 = {4, 5, 6, 7}; span span1(data1); span span2(data2); - + executor_->Encrypt(span1, {}); executor_->Encrypt(span2, {}); - + // Verify both encrypt calls were made EXPECT_EQ(mock_agent_ptr_->encrypt_call_count_, 2); - + // Verify the last encrypt call parameters EXPECT_EQ(mock_agent_ptr_->last_encrypt_plaintext_, data2); - + // Make multiple decrypt calls std::vector cipher1 = {8, 9, 10}; std::vector cipher2 = {11, 12, 13, 14, 15}; span cipher_span1(cipher1); span cipher_span2(cipher2); - + executor_->Decrypt(cipher_span1, {}); executor_->Decrypt(cipher_span2, {}); - + // Verify both decrypt calls were made EXPECT_EQ(mock_agent_ptr_->decrypt_call_count_, 2); - + // Verify the last decrypt call parameters EXPECT_EQ(mock_agent_ptr_->last_decrypt_ciphertext_, cipher2); } @@ -374,20 +380,20 @@ TEST_F(DBPAExecutorTest, TimeoutExceptionThrownOnSlowOperation) { // Simulate slow operation that takes 200ms std::this_thread::sleep_for(std::chrono::milliseconds(100)); } - + std::unique_ptr Encrypt(span, std::map) override { // Simulate slow operation that takes 150ms std::this_thread::sleep_for(std::chrono::milliseconds(100)); return nullptr; } - + std::unique_ptr Decrypt(span, std::map) override { // Simulate slow operation that takes 120ms std::this_thread::sleep_for(std::chrono::milliseconds(100)); return nullptr; } }; - + // Create an executor with very short timeouts auto slow_agent = std::make_unique(); auto timeout_executor = std::make_unique(std::move(slow_agent), @@ -409,11 +415,11 @@ TEST_F(DBPAExecutorTest, TimeoutExceptionThrownOnSlowOperation) { } catch (...) { FAIL() << "Expected DBPAExecutorTimeoutException, but got different exception type"; } - + // Test encrypt timeout - should throw DBPAExecutorTimeoutException std::vector data = {1, 2, 3, 4, 5}; span data_span(data); - + try { timeout_executor->Encrypt(data_span, {}); FAIL() << "Expected DBPAExecutorTimeoutException to be thrown"; @@ -426,7 +432,7 @@ TEST_F(DBPAExecutorTest, TimeoutExceptionThrownOnSlowOperation) { } catch (...) { FAIL() << "Expected DBPAExecutorTimeoutException, but got different exception type"; } - + // Test decrypt timeout - should throw DBPAExecutorTimeoutException try { timeout_executor->Decrypt(data_span, {}); @@ -447,7 +453,7 @@ TEST_F(DBPAExecutorTest, OriginalExceptionsArePreserved) { // Configure mock agent to throw exceptions mock_agent_ptr_->should_throw_on_init_ = true; mock_agent_ptr_->throw_message_ = "Mock init error"; - + // Test that init exception is preserved try { executor_->init("test_column", {}, "test_context", "test_key_id", @@ -458,15 +464,15 @@ TEST_F(DBPAExecutorTest, OriginalExceptionsArePreserved) { } catch (...) { FAIL() << "Unexpected exception type"; } - + // Reset and test encrypt exception mock_agent_ptr_->should_throw_on_init_ = false; mock_agent_ptr_->should_throw_on_encrypt_ = true; mock_agent_ptr_->throw_message_ = "Mock encrypt error"; - + std::vector data = {1, 2, 3, 4, 5}; span data_span(data); - + try { executor_->Encrypt(data_span, {}); FAIL() << "Expected std::runtime_error to be thrown"; @@ -475,12 +481,12 @@ TEST_F(DBPAExecutorTest, OriginalExceptionsArePreserved) { } catch (...) { FAIL() << "Unexpected exception type"; } - + // Reset and test decrypt exception mock_agent_ptr_->should_throw_on_encrypt_ = false; mock_agent_ptr_->should_throw_on_decrypt_ = true; mock_agent_ptr_->throw_message_ = "Mock decrypt error"; - + try { executor_->Decrypt(data_span, {}); FAIL() << "Expected std::runtime_error to be thrown"; diff --git a/cpp/src/parquet/encryption/external/dbpa_library_wrapper_test.cc b/cpp/src/parquet/encryption/external/dbpa_library_wrapper_test.cc index dd436415da44..55336057d734 100644 --- a/cpp/src/parquet/encryption/external/dbpa_library_wrapper_test.cc +++ b/cpp/src/parquet/encryption/external/dbpa_library_wrapper_test.cc @@ -37,15 +37,17 @@ using dbps::external::EncryptionResult; using dbps::external::DecryptionResult; using dbps::external::Type; using dbps::external::CompressionCodec; - + // Simple implementation of EncryptionResult for testing class TestEncryptionResult : public EncryptionResult { public: TestEncryptionResult(std::vector data, bool success = true, std::string error_msg = "", - std::map error_fields = {}) + std::map error_fields = {}, + std::optional> metadata = std::nullopt) : ciphertext_data_(std::move(data)), success_(success), - error_message_(std::move(error_msg)), error_fields_(std::move(error_fields)) {} + error_message_(std::move(error_msg)), error_fields_(std::move(error_fields)), + metadata_(std::move(metadata)) {} span ciphertext() const override { return span(ciphertext_data_.data(), ciphertext_data_.size()); @@ -53,6 +55,9 @@ class TestEncryptionResult : public EncryptionResult { std::size_t size() const override { return ciphertext_data_.size(); } bool success() const override { return success_; } + const std::optional> encryption_metadata() const override { + return metadata_; + } const std::string& error_message() const override { return error_message_; } const std::map& error_fields() const override { return error_fields_; } @@ -61,6 +66,7 @@ class TestEncryptionResult : public EncryptionResult { bool success_; std::string error_message_; std::map error_fields_; + std::optional> metadata_; }; // Simple implementation of DecryptionResult for testing @@ -92,12 +98,12 @@ class TestDecryptionResult : public DecryptionResult { class DestructionOrderTracker { public: DestructionOrderTracker() : sequence_counter_(0) {} - + // Record an event with a sequence number void RecordEvent(const std::string& event_name) { events_.emplace_back(event_name, ++sequence_counter_); } - + // Get the sequence number for a specific event int GetEventSequence(const std::string& event_name) const { for (const auto& event : events_) { @@ -107,30 +113,30 @@ class DestructionOrderTracker { } return -1; // Event not found } - + // Verify that first_event occurred before second_event bool VerifyOrder(const std::string& first_event, const std::string& second_event) const { int first_seq = GetEventSequence(first_event); int second_seq = GetEventSequence(second_event); - + if (first_seq == -1 || second_seq == -1) { return false; // One or both events not recorded } - + return first_seq < second_seq; } - + // Get all recorded events in order const std::vector>& GetEvents() const { return events_; } - + // Clear all recorded events void Clear() { events_.clear(); sequence_counter_ = 0; } - + // Check if an event was recorded bool WasEventRecorded(const std::string& event_name) const { return GetEventSequence(event_name) != -1; @@ -153,7 +159,7 @@ class MockCompanionDBPA { decrypt_count_(0), init_count_(0), order_tracker_(order_tracker ? order_tracker : std::make_shared()) {} - + // Test helper methods bool WasEncryptCalled() const { return encrypt_called_; } bool WasDecryptCalled() const { return decrypt_called_; } @@ -166,6 +172,20 @@ class MockCompanionDBPA { const std::vector& GetDecryptCiphertext() const { return decrypt_ciphertext_; } size_t GetEncryptCiphertextSize() const { return encrypt_ciphertext_size_; } std::shared_ptr GetOrderTracker() const { return order_tracker_; } + const std::map& GetLastEncryptEncodingAttrs() const { + return last_encrypt_encoding_attrs_; + } + const std::map& GetLastDecryptEncodingAttrs() const { + return last_decrypt_encoding_attrs_; + } + void SetNextEncryptResultMetadata(std::optional> metadata) { + next_encrypt_result_metadata_ = std::move(metadata); + } + std::optional> ConsumeNextEncryptResultMetadata() { + auto tmp = std::move(next_encrypt_result_metadata_); + next_encrypt_result_metadata_.reset(); + return tmp; + } // Init tracking methods const std::string& GetInitColumnName() const { return init_column_name_; } @@ -193,6 +213,12 @@ class MockCompanionDBPA { void SetEncryptPlaintext(const std::vector& plaintext) { encrypt_plaintext_ = plaintext; } void SetDecryptCiphertext(const std::vector& ciphertext) { decrypt_ciphertext_ = ciphertext; } void SetEncryptCiphertextSize(size_t size) { encrypt_ciphertext_size_ = size; } + void SetLastEncryptEncodingAttrs(std::map attrs) { + last_encrypt_encoding_attrs_ = std::move(attrs); + } + void SetLastDecryptEncodingAttrs(std::map attrs) { + last_decrypt_encoding_attrs_ = std::move(attrs); + } // Init parameter tracking void SetInitParameters( @@ -226,7 +252,7 @@ class MockCompanionDBPA { std::vector decrypt_ciphertext_; size_t encrypt_ciphertext_size_; std::shared_ptr order_tracker_; - + // Init parameters std::string init_column_name_; std::map init_connection_config_; @@ -238,6 +264,7 @@ class MockCompanionDBPA { std::optional> init_column_encryption_metadata_; std::map last_encrypt_encoding_attrs_; std::map last_decrypt_encoding_attrs_; + std::optional> next_encrypt_result_metadata_; }; //MockCompanionDBPA // Companion object to track shared library handle management operations @@ -248,18 +275,18 @@ class SharedLibHandleManagementCompanion { handle_close_count_(0), last_closed_handle_(nullptr), order_tracker_(order_tracker ? order_tracker : std::make_shared()) {} - + // Test helper methods bool WasHandleCloseCalled() const { return handle_close_called_; } int GetHandleCloseCount() const { return handle_close_count_; } void* GetLastClosedHandle() const { return last_closed_handle_; } std::shared_ptr GetOrderTracker() const { return order_tracker_; } - + // State update methods void SetHandleCloseCalled(bool called) { handle_close_called_ = called; } void IncrementHandleCloseCount() { handle_close_count_++; } void SetLastClosedHandle(void* handle) { last_closed_handle_ = handle; } - + // Create a closure that captures this companion object // and returns a function that can be used to close the shared library handle std::function CreateHandleClosingFunction() { @@ -283,11 +310,11 @@ class MockDataBatchProtectionAgent : public DataBatchProtectionAgentInterface { public: explicit MockDataBatchProtectionAgent(std::shared_ptr companion = nullptr) : companion_(companion ? companion : std::make_shared()) {} - + ~MockDataBatchProtectionAgent() override { companion_->SetDestructorCalled(true); } - + void init( std::string column_name, std::map connection_config, @@ -312,23 +339,28 @@ class MockDataBatchProtectionAgent : public DataBatchProtectionAgentInterface { std::unique_ptr Encrypt( span plaintext, - std::map) override { + std::map encoding_attributes) override { companion_->SetEncryptCalled(true); companion_->IncrementEncryptCount(); companion_->SetEncryptPlaintext(std::vector(plaintext.begin(), plaintext.end())); companion_->SetEncryptCiphertextSize(plaintext.size()); + companion_->SetLastEncryptEncodingAttrs(std::move(encoding_attributes)); // Create a simple mock encryption result std::vector ciphertext_data(plaintext.begin(), plaintext.end()); - return std::make_unique(std::move(ciphertext_data)); + auto result_metadata = companion_->ConsumeNextEncryptResultMetadata(); + return std::make_unique(std::move(ciphertext_data), true, "", + std::map{}, + std::move(result_metadata)); } std::unique_ptr Decrypt( span ciphertext, - std::map) override { + std::map encoding_attributes) override { companion_->SetDecryptCalled(true); companion_->IncrementDecryptCount(); companion_->SetDecryptCiphertext(std::vector(ciphertext.begin(), ciphertext.end())); + companion_->SetLastDecryptEncodingAttrs(std::move(encoding_attributes)); // Create a simple mock decryption result std::vector plaintext_data(ciphertext.begin(), ciphertext.end()); @@ -349,14 +381,14 @@ class DBPALibraryWrapperTest : public ::testing::Test { // Create test data test_plaintext_ = "Hello, World!"; test_ciphertext_.resize(test_plaintext_.size()); - + // Create shared destruction order tracker destruction_order_tracker_ = std::make_shared(); - + // Create companion objects with shared order tracker mock_companion_ = std::make_shared(destruction_order_tracker_); handle_companion_ = std::make_shared(destruction_order_tracker_); - + // Create mock agent mock_agent_ = std::make_unique(mock_companion_); mock_agent_ptr_ = mock_agent_.get(); @@ -376,7 +408,7 @@ class DBPALibraryWrapperTest : public ::testing::Test { std::unique_ptr CreateWrapperWithAgent( std::unique_ptr agent) { void* dummy_handle = reinterpret_cast(0x12345678); - + // Use the existing handle companion from the test fixture return std::make_unique( std::move(agent), dummy_handle, handle_companion_->CreateHandleClosingFunction()); @@ -386,7 +418,7 @@ class DBPALibraryWrapperTest : public ::testing::Test { std::unique_ptr CreateWrapperWithCustomClosing( std::function handle_closing_fn) { void* dummy_handle = reinterpret_cast(0x12345678); - + return std::make_unique( std::move(mock_agent_), dummy_handle, handle_closing_fn); } @@ -407,7 +439,7 @@ class DBPALibraryWrapperTest : public ::testing::Test { TEST_F(DBPALibraryWrapperTest, ConstructorValidParameters) { auto mock_agent = std::make_unique(); void* dummy_handle = reinterpret_cast(0x12345678); - + EXPECT_NO_THROW({ DBPALibraryWrapper wrapper(std::move(mock_agent), dummy_handle, handle_companion_->CreateHandleClosingFunction()); }); @@ -416,7 +448,7 @@ TEST_F(DBPALibraryWrapperTest, ConstructorValidParameters) { TEST_F(DBPALibraryWrapperTest, ConstructorValidParametersWithDefaultClosing) { auto mock_agent = std::make_unique(); void* dummy_handle = reinterpret_cast(0x12345678); - + EXPECT_NO_THROW({ DBPALibraryWrapper wrapper(std::move(mock_agent), dummy_handle, handle_companion_->CreateHandleClosingFunction()); }); @@ -424,7 +456,7 @@ TEST_F(DBPALibraryWrapperTest, ConstructorValidParametersWithDefaultClosing) { TEST_F(DBPALibraryWrapperTest, ConstructorNullAgent) { void* dummy_handle = reinterpret_cast(0x12345678); - + // Test with custom function EXPECT_THROW({ DBPALibraryWrapper wrapper(nullptr, dummy_handle, handle_companion_->CreateHandleClosingFunction()); @@ -447,16 +479,16 @@ TEST_F(DBPALibraryWrapperTest, ConstructorNullLibraryHandle) { TEST_F(DBPALibraryWrapperTest, HandleClosingFunctionCalled) { void* dummy_handle = reinterpret_cast(0x12345678); - + // Create wrapper in a scope to trigger destructor { auto wrapper = CreateWrapper(); - + // Verify handle closing hasn't been called yet EXPECT_FALSE(handle_companion_->WasHandleCloseCalled()); EXPECT_EQ(handle_companion_->GetHandleCloseCount(), 0); } - + // After wrapper destruction, handle closing should have been called EXPECT_TRUE(handle_companion_->WasHandleCloseCalled()); EXPECT_EQ(handle_companion_->GetHandleCloseCount(), 1); @@ -466,24 +498,24 @@ TEST_F(DBPALibraryWrapperTest, HandleClosingFunctionCalled) { TEST_F(DBPALibraryWrapperTest, CustomHandleClosingFunction) { bool custom_function_called = false; void* custom_last_handle = nullptr; - + auto custom_closing_fn = [&custom_function_called, &custom_last_handle](void* handle) { custom_function_called = true; custom_last_handle = handle; }; - + void* dummy_handle = reinterpret_cast(0x87654321); - + // Create wrapper with custom closing function { auto mock_agent = std::make_unique(); DBPALibraryWrapper wrapper(std::move(mock_agent), dummy_handle, custom_closing_fn); } - + // Verify custom function was called EXPECT_TRUE(custom_function_called); EXPECT_EQ(custom_last_handle, dummy_handle); - + // Verify our handle companion wasn't called EXPECT_FALSE(handle_companion_->WasHandleCloseCalled()); EXPECT_EQ(handle_companion_->GetHandleCloseCount(), 0); @@ -495,7 +527,7 @@ TEST_F(DBPALibraryWrapperTest, CustomHandleClosingFunction) { TEST_F(DBPALibraryWrapperTest, InitDelegation) { auto wrapper = CreateWrapper(); - + // Test data for init parameters std::string column_name = "test_column"; std::map connection_config = { @@ -508,14 +540,14 @@ TEST_F(DBPALibraryWrapperTest, InitDelegation) { Type::type data_type = Type::INT32; CompressionCodec::type compression_type = CompressionCodec::SNAPPY; std::optional> column_encryption_metadata = std::map{{"metaKey", "metaValue"}}; - + // Call init through wrapper wrapper->init(column_name, connection_config, app_context, column_key_id, data_type, std::nullopt, compression_type, column_encryption_metadata); - + // Verify the mock agent was called EXPECT_TRUE(mock_companion_->WasInitCalled()); EXPECT_EQ(mock_companion_->GetInitCount(), 1); - + // Verify the correct parameters were passed to the mock EXPECT_EQ(mock_companion_->GetInitColumnName(), column_name); EXPECT_EQ(mock_companion_->GetInitConnectionConfig(), connection_config); @@ -547,7 +579,7 @@ TEST_F(DBPALibraryWrapperTest, InitDelegationWithDatatypeLength) { TEST_F(DBPALibraryWrapperTest, InitDelegationWithEmptyParameters) { auto wrapper = CreateWrapper(); - + // Test init with empty parameters std::string empty_column_name = ""; std::map empty_connection_config = {}; @@ -555,14 +587,14 @@ TEST_F(DBPALibraryWrapperTest, InitDelegationWithEmptyParameters) { std::string empty_column_key_id = ""; Type::type data_type = Type::BYTE_ARRAY; CompressionCodec::type compression_type = CompressionCodec::UNCOMPRESSED; - + // Call init through wrapper wrapper->init(empty_column_name, empty_connection_config, empty_app_context, empty_column_key_id, data_type, std::nullopt, compression_type, std::nullopt); - + // Verify the mock agent was called EXPECT_TRUE(mock_companion_->WasInitCalled()); EXPECT_EQ(mock_companion_->GetInitCount(), 1); - + // Verify the empty parameters were passed correctly EXPECT_EQ(mock_companion_->GetInitColumnName(), empty_column_name); EXPECT_EQ(mock_companion_->GetInitConnectionConfig(), empty_connection_config); @@ -575,28 +607,28 @@ TEST_F(DBPALibraryWrapperTest, InitDelegationWithEmptyParameters) { TEST_F(DBPALibraryWrapperTest, EncryptDelegation) { auto wrapper = CreateWrapper(); - + // Convert test data to spans span plaintext_span( reinterpret_cast(test_plaintext_.data()), test_plaintext_.size()); span ciphertext_span(test_ciphertext_.data(), test_ciphertext_.size()); - + // Call encrypt through wrapper auto result = wrapper->Encrypt(plaintext_span, {}); - + // Verify the mock agent was called EXPECT_TRUE(mock_companion_->WasEncryptCalled()); EXPECT_EQ(mock_companion_->GetEncryptCount(), 1); - + // Verify the correct plaintext was passed to the mock auto mock_plaintext = mock_companion_->GetEncryptPlaintext(); std::string mock_plaintext_str(mock_plaintext.begin(), mock_plaintext.end()); EXPECT_EQ(mock_plaintext_str, test_plaintext_); - + // Verify the correct ciphertext size was passed EXPECT_EQ(mock_companion_->GetEncryptCiphertextSize(), test_ciphertext_.size()); - + // Verify result is not null EXPECT_NE(result, nullptr); } @@ -605,6 +637,9 @@ TEST_F(DBPALibraryWrapperTest, EncryptDecryptEncodingAttributesDelegation) { auto wrapper = CreateWrapper(); // Encrypt + std::map expected_result_metadata = {{"encryption_algorithm_version", "1"}, + {"test_kid", "kid_123"}}; + mock_companion_->SetNextEncryptResultMetadata(expected_result_metadata); std::map enc_attrs = {{"format", "plain"}, {"scale", "0"}}; span plaintext_span( reinterpret_cast(test_plaintext_.data()), @@ -612,54 +647,58 @@ TEST_F(DBPALibraryWrapperTest, EncryptDecryptEncodingAttributesDelegation) { auto enc_result = wrapper->Encrypt(plaintext_span, enc_attrs); EXPECT_NE(enc_result, nullptr); + EXPECT_EQ(mock_companion_->GetLastEncryptEncodingAttrs(), enc_attrs); + ASSERT_TRUE(enc_result->encryption_metadata().has_value()); + EXPECT_EQ(enc_result->encryption_metadata().value(), expected_result_metadata); // Decrypt auto ct_span = enc_result->ciphertext(); std::map dec_attrs = {{"format", "plain"}}; auto dec_result = wrapper->Decrypt(ct_span, dec_attrs); EXPECT_NE(dec_result, nullptr); + EXPECT_EQ(mock_companion_->GetLastDecryptEncodingAttrs(), dec_attrs); } TEST_F(DBPALibraryWrapperTest, DecryptDelegation) { auto wrapper = CreateWrapper(); - + // Convert test data to spans span ciphertext_span( reinterpret_cast(test_plaintext_.data()), test_plaintext_.size()); - + // Call decrypt through wrapper auto result = wrapper->Decrypt(ciphertext_span, {}); - + // Verify the mock agent was called EXPECT_TRUE(mock_companion_->WasDecryptCalled()); EXPECT_EQ(mock_companion_->GetDecryptCount(), 1); - + // Verify the correct ciphertext was passed to the mock auto mock_ciphertext = mock_companion_->GetDecryptCiphertext(); std::string mock_ciphertext_str(mock_ciphertext.begin(), mock_ciphertext.end()); EXPECT_EQ(mock_ciphertext_str, test_plaintext_); - + // Verify result is not null EXPECT_NE(result, nullptr); } TEST_F(DBPALibraryWrapperTest, MultipleEncryptDelegations) { auto wrapper = CreateWrapper(); - + // Perform multiple encrypt operations for (int i = 0; i < 5; ++i) { std::string plaintext = "Test " + std::to_string(i); std::vector ciphertext(plaintext.size()); - + span plaintext_span( reinterpret_cast(plaintext.data()), plaintext.size()); - + auto result = wrapper->Encrypt(plaintext_span, {}); EXPECT_NE(result, nullptr); } - + // Verify the mock agent was called the correct number of times EXPECT_TRUE(mock_companion_->WasEncryptCalled()); EXPECT_EQ(mock_companion_->GetEncryptCount(), 5); @@ -667,19 +706,19 @@ TEST_F(DBPALibraryWrapperTest, MultipleEncryptDelegations) { TEST_F(DBPALibraryWrapperTest, MultipleDecryptDelegations) { auto wrapper = CreateWrapper(); - + // Perform multiple decrypt operations for (int i = 0; i < 3; ++i) { std::string ciphertext = "Test " + std::to_string(i); - + span ciphertext_span( reinterpret_cast(ciphertext.data()), ciphertext.size()); - + auto result = wrapper->Decrypt(ciphertext_span, {}); EXPECT_NE(result, nullptr); } - + // Verify the mock agent was called the correct number of times EXPECT_TRUE(mock_companion_->WasDecryptCalled()); EXPECT_EQ(mock_companion_->GetDecryptCount(), 3); @@ -687,26 +726,26 @@ TEST_F(DBPALibraryWrapperTest, MultipleDecryptDelegations) { TEST_F(DBPALibraryWrapperTest, MixedOperationsDelegation) { auto wrapper = CreateWrapper(); - + // Perform mixed encrypt and decrypt operations std::vector test_data = {"Hello", "World", "Test", "Data"}; int call_count = test_data.size(); - + for (const auto& data : test_data) { // Encrypt span plaintext_span( reinterpret_cast(data.data()), data.size()); - + auto encrypt_result = wrapper->Encrypt(plaintext_span, {}); EXPECT_NE(encrypt_result, nullptr); - + // Decrypt using the ciphertext from the encryption result auto ciphertext_span = encrypt_result->ciphertext(); auto decrypt_result = wrapper->Decrypt(ciphertext_span, {}); EXPECT_NE(decrypt_result, nullptr); } - + // Verify both operations were called EXPECT_TRUE(mock_companion_->WasEncryptCalled()); EXPECT_TRUE(mock_companion_->WasDecryptCalled()); @@ -716,7 +755,7 @@ TEST_F(DBPALibraryWrapperTest, MixedOperationsDelegation) { TEST_F(DBPALibraryWrapperTest, InitWithEncryptDecryptOperations) { auto wrapper = CreateWrapper(); - + // First, initialize the wrapper std::string column_name = "test_column"; std::map connection_config = { @@ -727,32 +766,32 @@ TEST_F(DBPALibraryWrapperTest, InitWithEncryptDecryptOperations) { std::string column_key_id = "test_key"; Type::type data_type = Type::INT32; CompressionCodec::type compression_type = CompressionCodec::SNAPPY; - + wrapper->init(column_name, connection_config, app_context, column_key_id, data_type, std::nullopt, compression_type, std::nullopt); - + // Verify init was called EXPECT_TRUE(mock_companion_->WasInitCalled()); EXPECT_EQ(mock_companion_->GetInitCount(), 1); - + // Then perform encrypt/decrypt operations std::string test_data = "Test data after init"; span plaintext_span( reinterpret_cast(test_data.data()), test_data.size()); - + auto encrypt_result = wrapper->Encrypt(plaintext_span, {}); EXPECT_NE(encrypt_result, nullptr); - + auto ciphertext_span = encrypt_result->ciphertext(); auto decrypt_result = wrapper->Decrypt(ciphertext_span, {}); EXPECT_NE(decrypt_result, nullptr); - + // Verify all operations were called EXPECT_TRUE(mock_companion_->WasEncryptCalled()); EXPECT_TRUE(mock_companion_->WasDecryptCalled()); EXPECT_EQ(mock_companion_->GetEncryptCount(), 1); EXPECT_EQ(mock_companion_->GetDecryptCount(), 1); - + // Verify init parameters are still accessible EXPECT_EQ(mock_companion_->GetInitColumnName(), column_name); EXPECT_EQ(mock_companion_->GetInitConnectionConfig(), connection_config); @@ -765,16 +804,16 @@ TEST_F(DBPALibraryWrapperTest, InitWithEncryptDecryptOperations) { TEST_F(DBPALibraryWrapperTest, DelegationWithEmptyData) { auto wrapper = CreateWrapper(); - + // Test encryption with empty data std::vector empty_plaintext; - + span plaintext_span(empty_plaintext); - + auto encrypt_result = wrapper->Encrypt(plaintext_span, {}); EXPECT_NE(encrypt_result, nullptr); EXPECT_TRUE(mock_companion_->WasEncryptCalled()); - + // Test decryption with empty data auto ciphertext_span = encrypt_result->ciphertext(); auto decrypt_result = wrapper->Decrypt(ciphertext_span, {}); @@ -784,21 +823,21 @@ TEST_F(DBPALibraryWrapperTest, DelegationWithEmptyData) { TEST_F(DBPALibraryWrapperTest, DelegationWithNullData) { auto wrapper = CreateWrapper(); - + // Test encryption with null data pointers but valid spans // This tests that the wrapper properly delegates even with null data span null_plaintext_span(nullptr, size_t{0}); - + auto encrypt_result = wrapper->Encrypt(null_plaintext_span, {}); EXPECT_NE(encrypt_result, nullptr); EXPECT_TRUE(mock_companion_->WasEncryptCalled()); - + // Test decryption with null data pointer but valid span span null_decrypt_span(nullptr, size_t{0}); auto decrypt_result = wrapper->Decrypt(null_decrypt_span, {}); EXPECT_NE(decrypt_result, nullptr); EXPECT_TRUE(mock_companion_->WasDecryptCalled()); - + // Verify the mock agent received the correct data (empty vectors) auto mock_plaintext = mock_companion_->GetEncryptPlaintext(); auto mock_ciphertext = mock_companion_->GetDecryptCiphertext(); @@ -812,24 +851,24 @@ TEST_F(DBPALibraryWrapperTest, DelegationWithNullData) { TEST_F(DBPALibraryWrapperTest, DestructorBasicBehavior) { void* dummy_handle = reinterpret_cast(0x12345678); - + // Create wrapper in a scope to test destructor { auto wrapper = CreateWrapper(); - + // Perform some operations to ensure the wrapper is used std::vector plaintext = {1, 2, 3, 4, 5}; - + span plaintext_span(plaintext.data(), plaintext.size()); - + auto result = wrapper->Encrypt(plaintext_span, {}); EXPECT_NE(result, nullptr); - + // Verify handle closing hasn't been called yet EXPECT_FALSE(handle_companion_->WasHandleCloseCalled()); EXPECT_EQ(handle_companion_->GetHandleCloseCount(), 0); } - + // At this point, the wrapper should have been destroyed and handle closed EXPECT_TRUE(handle_companion_->WasHandleCloseCalled()); EXPECT_EQ(handle_companion_->GetHandleCloseCount(), 1); @@ -838,28 +877,28 @@ TEST_F(DBPALibraryWrapperTest, DestructorBasicBehavior) { TEST_F(DBPALibraryWrapperTest, DestructorWithMultipleOperations) { void* dummy_handle = reinterpret_cast(0x12345678); - + // Create wrapper in a scope to test destructor { auto wrapper = CreateWrapper(); - + // Perform multiple operations for (int i = 0; i < 10; ++i) { std::string plaintext = "Test " + std::to_string(i); std::vector ciphertext(plaintext.size()); - + span plaintext_span( reinterpret_cast(plaintext.data()), plaintext.size()); - + auto encrypt_result = wrapper->Encrypt(plaintext_span, {}); EXPECT_NE(encrypt_result, nullptr); - + auto ciphertext_span = encrypt_result->ciphertext(); auto decrypt_result = wrapper->Decrypt(ciphertext_span, {}); EXPECT_NE(decrypt_result, nullptr); } - + // Verify operations completed but handle not closed yet EXPECT_FALSE(handle_companion_->WasHandleCloseCalled()); } @@ -873,38 +912,38 @@ TEST_F(DBPALibraryWrapperTest, DestructorWithMultipleOperations) { TEST_F(DBPALibraryWrapperTest, DestructorOrderVerification) { // Clear any previous events from the shared order tracker destruction_order_tracker_->Clear(); - + // Create a custom mock agent that tracks destruction order auto custom_companion = std::make_shared(destruction_order_tracker_); auto custom_agent = std::make_unique(custom_companion); - + void* dummy_handle = reinterpret_cast(0x12345678); - + // Create wrapper in a scope { auto wrapper = CreateWrapperWithAgent(std::move(custom_agent)); - + // Perform some operations std::vector plaintext = {1, 2, 3}; - + span plaintext_span(plaintext.data(), plaintext.size()); - + auto result = wrapper->Encrypt(plaintext_span, {}); EXPECT_NE(result, nullptr); - + // Verify neither destructor nor handle closing has been called yet EXPECT_FALSE(custom_companion->WasDestructorCalled()); EXPECT_FALSE(handle_companion_->WasHandleCloseCalled()); EXPECT_FALSE(destruction_order_tracker_->WasEventRecorded("handle_close")); EXPECT_FALSE(destruction_order_tracker_->WasEventRecorded("agent_destructor")); } - + // Verify both the custom agent was destroyed and handle was closed EXPECT_TRUE(custom_companion->WasDestructorCalled()); EXPECT_TRUE(handle_companion_->WasHandleCloseCalled()); EXPECT_EQ(handle_companion_->GetHandleCloseCount(), 1); EXPECT_EQ(handle_companion_->GetLastClosedHandle(), dummy_handle); - + // Verify the order of destruction: handle_close should be called BEFORE agent_destructor EXPECT_TRUE(destruction_order_tracker_->WasEventRecorded("agent_destructor")); EXPECT_TRUE(destruction_order_tracker_->WasEventRecorded("handle_close")); @@ -914,27 +953,27 @@ TEST_F(DBPALibraryWrapperTest, DestructorOrderVerification) { TEST_F(DBPALibraryWrapperTest, DestructionOrderTrackerFunctionality) { // Test the destruction order tracker functionality independently auto tracker = std::make_shared(); - + // Record events in a specific order tracker->RecordEvent("first"); tracker->RecordEvent("second"); tracker->RecordEvent("third"); - + // Verify order tracking EXPECT_TRUE(tracker->VerifyOrder("first", "second")); EXPECT_TRUE(tracker->VerifyOrder("second", "third")); EXPECT_TRUE(tracker->VerifyOrder("first", "third")); - + // Verify reverse order is false EXPECT_FALSE(tracker->VerifyOrder("second", "first")); EXPECT_FALSE(tracker->VerifyOrder("third", "second")); EXPECT_FALSE(tracker->VerifyOrder("third", "first")); - + // Verify sequence numbers EXPECT_EQ(tracker->GetEventSequence("first"), 1); EXPECT_EQ(tracker->GetEventSequence("second"), 2); EXPECT_EQ(tracker->GetEventSequence("third"), 3); - + // Verify event recording EXPECT_TRUE(tracker->WasEventRecorded("first")); EXPECT_TRUE(tracker->WasEventRecorded("second")); @@ -948,11 +987,11 @@ TEST_F(DBPALibraryWrapperTest, DestructionOrderTrackerFunctionality) { TEST_F(DBPALibraryWrapperTest, InterfaceCompliancePolymorphic) { auto wrapper = CreateWrapper(); - + // Verify the wrapper can be used polymorphically DataBatchProtectionAgentInterface* interface_ptr = wrapper.get(); EXPECT_NE(interface_ptr, nullptr); - + // Test polymorphic init call std::string column_name = "polymorphic_column"; std::map connection_config = {{"test", "value"}}; @@ -960,9 +999,9 @@ TEST_F(DBPALibraryWrapperTest, InterfaceCompliancePolymorphic) { std::string column_key_id = "polymorphic_key"; Type::type data_type = Type::INT64; CompressionCodec::type compression_type = CompressionCodec::GZIP; - + interface_ptr->init(column_name, connection_config, app_context, column_key_id, data_type, std::nullopt, compression_type, std::nullopt); - + // Verify init was called through the interface EXPECT_TRUE(mock_companion_->WasInitCalled()); EXPECT_EQ(mock_companion_->GetInitCount(), 1); @@ -970,19 +1009,19 @@ TEST_F(DBPALibraryWrapperTest, InterfaceCompliancePolymorphic) { EXPECT_EQ(mock_companion_->GetInitDataType(), data_type); EXPECT_EQ(mock_companion_->GetInitCompressionType(), compression_type); EXPECT_FALSE(mock_companion_->GetInitColumnEncryptionMetadata().has_value()); - + // Test polymorphic encrypt/decrypt calls std::vector plaintext = {1, 2, 3}; - + span plaintext_span(plaintext.data(), plaintext.size()); - + auto encrypt_result = interface_ptr->Encrypt(plaintext_span, {}); EXPECT_NE(encrypt_result, nullptr); - + auto ciphertext_span = encrypt_result->ciphertext(); auto decrypt_result = interface_ptr->Decrypt(ciphertext_span, {}); EXPECT_NE(decrypt_result, nullptr); - + // Verify the mock agent was called through the interface EXPECT_TRUE(mock_companion_->WasEncryptCalled()); EXPECT_TRUE(mock_companion_->WasDecryptCalled()); @@ -994,40 +1033,40 @@ TEST_F(DBPALibraryWrapperTest, InterfaceCompliancePolymorphic) { TEST_F(DBPALibraryWrapperTest, EdgeCaseZeroSizeSpans) { auto wrapper = CreateWrapper(); - + // Test with zero-size spans std::vector empty_data; - + span empty_plaintext_span(empty_data); - + auto encrypt_result = wrapper->Encrypt(empty_plaintext_span, {}); EXPECT_NE(encrypt_result, nullptr); - + auto ciphertext_span = encrypt_result->ciphertext(); auto decrypt_result = wrapper->Decrypt(ciphertext_span, {}); EXPECT_NE(decrypt_result, nullptr); - + EXPECT_TRUE(mock_companion_->WasEncryptCalled()); EXPECT_TRUE(mock_companion_->WasDecryptCalled()); } TEST_F(DBPALibraryWrapperTest, EdgeCaseSingleByteData) { auto wrapper = CreateWrapper(); - + // Test with single byte data std::vector single_byte = {0x42}; - + span plaintext_span(single_byte.data(), single_byte.size()); - + auto encrypt_result = wrapper->Encrypt(plaintext_span, {}); EXPECT_NE(encrypt_result, nullptr); - + auto ciphertext_span = encrypt_result->ciphertext(); auto decrypt_result = wrapper->Decrypt(ciphertext_span, {}); EXPECT_NE(decrypt_result, nullptr); - + EXPECT_TRUE(mock_companion_->WasEncryptCalled()); EXPECT_TRUE(mock_companion_->WasDecryptCalled()); } -} // namespace parquet::encryption::external::test +} // namespace parquet::encryption::external::test \ No newline at end of file From 56463c16282c912f5a25f91a931fa06d667accd8 Mon Sep 17 00:00:00 2001 From: Marco Arguedas Date: Wed, 17 Dec 2025 19:28:22 -0600 Subject: [PATCH 04/10] Updated the type of 'key_size' in Get*AesEncryptor() to make the compiler happy --- cpp/src/parquet/encryption/aes_encryption.cc | 11 +++++++++-- cpp/src/parquet/encryption/aes_encryption.h | 4 ++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/cpp/src/parquet/encryption/aes_encryption.cc b/cpp/src/parquet/encryption/aes_encryption.cc index 387a558e3874..2fe6e71832a4 100644 --- a/cpp/src/parquet/encryption/aes_encryption.cc +++ b/cpp/src/parquet/encryption/aes_encryption.cc @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -334,7 +335,10 @@ uint64_t AesEncryptorFactory::MakeCacheKey( } AesEncryptor* AesEncryptorFactory::GetMetaAesEncryptor( - ParquetCipher::type alg_id, int32_t key_size) { + ParquetCipher::type alg_id, size_t key_size) { + if (key_size > static_cast(std::numeric_limits::max())) { + throw ParquetException("Invalid key length: exceeds int32_t max"); + } auto key_len = static_cast(key_size); // Create the cache key using the algorithm id, key length, and metadata flag // to avoid collisions for encryptors with the same key length. @@ -350,7 +354,10 @@ AesEncryptor* AesEncryptorFactory::GetMetaAesEncryptor( } AesEncryptor* AesEncryptorFactory::GetDataAesEncryptor( - ParquetCipher::type alg_id, int32_t key_size) { + ParquetCipher::type alg_id, size_t key_size) { + if (key_size > static_cast(std::numeric_limits::max())) { + throw ParquetException("Invalid key length: exceeds int32_t max"); + } auto key_len = static_cast(key_size); // Create the cache key using the algorithm id, key length, and metadata flag // to avoid collisions for encryptors with the same key length. diff --git a/cpp/src/parquet/encryption/aes_encryption.h b/cpp/src/parquet/encryption/aes_encryption.h index b54553939741..e741a89ee791 100644 --- a/cpp/src/parquet/encryption/aes_encryption.h +++ b/cpp/src/parquet/encryption/aes_encryption.h @@ -122,8 +122,8 @@ class PARQUET_EXPORT AesEncryptor : public AesCryptoContext, public EncryptorInt // store the encryptors for the different key lengths. class AesEncryptorFactory { public: - AesEncryptor* GetMetaAesEncryptor(ParquetCipher::type alg_id, int32_t key_size); - AesEncryptor* GetDataAesEncryptor(ParquetCipher::type alg_id, int32_t key_size); + AesEncryptor* GetMetaAesEncryptor(ParquetCipher::type alg_id, size_t key_size); + AesEncryptor* GetDataAesEncryptor(ParquetCipher::type alg_id, size_t key_size); private: /// Build a cache key including algorithm id, key length, and metadata flag. From 13b21c966c0bbc888b84bc4193e280235d7ab4e6 Mon Sep 17 00:00:00 2001 From: Marco Arguedas Date: Wed, 17 Dec 2025 19:59:34 -0600 Subject: [PATCH 05/10] Adding size checks to make the compiler happy (external_dbpa_encryption.cc) --- .../encryption/external_dbpa_encryption.cc | 19 ++++++++++++++++--- .../external_dbpa_encryption_test.cc | 19 ++++++++++++++----- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/cpp/src/parquet/encryption/external_dbpa_encryption.cc b/cpp/src/parquet/encryption/external_dbpa_encryption.cc index 35f26a201bc6..7ab5b3b0c5f3 100644 --- a/cpp/src/parquet/encryption/external_dbpa_encryption.cc +++ b/cpp/src/parquet/encryption/external_dbpa_encryption.cc @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -354,7 +355,11 @@ int32_t ExternalDBPAEncryptorAdapter::InvokeExternalEncrypt( ARROW_LOG(DEBUG) << " result size: " << result->size() << " bytes"; ARROW_LOG(DEBUG) << " result ciphertext size: " << result->ciphertext().size() << " bytes"; - int32_t ciphertext_size = result->ciphertext().size(); + const auto ciphertext_size64 = result->ciphertext().size(); + if (ciphertext_size64 > static_cast(std::numeric_limits::max())) { + throw ParquetException("Ciphertext size exceeds int32_t max"); + } + const int32_t ciphertext_size = static_cast(ciphertext_size64); auto status = ciphertext->Resize(ciphertext_size, false); if (!status.ok()) { ARROW_LOG(ERROR) << "Ciphertext buffer resize failed: " << status.ToString(); @@ -595,7 +600,11 @@ int32_t ExternalDBPADecryptorAdapter::InvokeExternalDecrypt( ARROW_LOG(DEBUG) << " result size: " << result->size() << " bytes"; ARROW_LOG(DEBUG) << " result plaintext size: " << result->plaintext().size() << " bytes"; - int32_t plaintext_size = result->plaintext().size(); + const auto plaintext_size64 = result->plaintext().size(); + if (plaintext_size64 > static_cast(std::numeric_limits::max())) { + throw ParquetException("Plaintext size exceeds int32_t max"); + } + const int32_t plaintext_size = static_cast(plaintext_size64); auto status = plaintext->Resize(plaintext_size, false); if (!status.ok()) { ARROW_LOG(ERROR) << "Plaintext buffer resize failed: " << status.ToString(); @@ -606,7 +615,11 @@ int32_t ExternalDBPADecryptorAdapter::InvokeExternalDecrypt( std::memcpy(plaintext->mutable_data(), result->plaintext().data(), plaintext_size); ARROW_LOG(DEBUG) << "Decryption completed successfully"; - return result->size(); + const auto total_size64 = result->size(); + if (total_size64 > static_cast(std::numeric_limits::max())) { + throw ParquetException("Result size exceeds int32_t max"); + } + return static_cast(total_size64); } std::unique_ptr ExternalDBPADecryptorAdapterFactory::GetDecryptor( diff --git a/cpp/src/parquet/encryption/external_dbpa_encryption_test.cc b/cpp/src/parquet/encryption/external_dbpa_encryption_test.cc index 81e7307fe9c7..6bf1c3fb72c7 100644 --- a/cpp/src/parquet/encryption/external_dbpa_encryption_test.cc +++ b/cpp/src/parquet/encryption/external_dbpa_encryption_test.cc @@ -16,6 +16,7 @@ // under the License. #include +#include #include #include #include @@ -94,7 +95,9 @@ class ExternalDBPAEncryptorAdapterTest : public ::testing::Test { encryptor->UpdateEncodingProperties(builder.Build()); - int32_t expected_ciphertext_length = plaintext.size(); + ASSERT_LE(plaintext.size(), + static_cast(std::numeric_limits::max())); + int32_t expected_ciphertext_length = static_cast(plaintext.size()); std::shared_ptr ciphertext_buffer = AllocateBuffer( ::arrow::default_memory_pool(), expected_ciphertext_length); @@ -119,7 +122,9 @@ class ExternalDBPAEncryptorAdapterTest : public ::testing::Test { decryptor->UpdateEncodingProperties(builder.Build()); - int32_t expected_plaintext_length = ciphertext_str.size(); + ASSERT_LE(ciphertext_str.size(), + static_cast(std::numeric_limits::max())); + int32_t expected_plaintext_length = static_cast(ciphertext_str.size()); std::shared_ptr plaintext_buffer = AllocateBuffer( ::arrow::default_memory_pool(), expected_plaintext_length); int32_t decryption_length = decryptor->DecryptWithManagedBuffer( @@ -555,8 +560,10 @@ TEST_F(ExternalDBPAEncryptorAdapterTest, EncryptCallShouldFail) { std::unique_ptr encryptor = CreateEncryptor( algorithm, column_name, key_id, data_type, compression_type, encoding_type); ASSERT_FALSE(encryptor->CanCalculateCiphertextLength()); + ASSERT_LE(plaintext.size(), + static_cast(std::numeric_limits::max())); EXPECT_THROW( - (void) encryptor->CiphertextLength(plaintext.size()), + (void) encryptor->CiphertextLength(static_cast(plaintext.size())), ParquetException); EXPECT_THROW( encryptor->Encrypt( @@ -577,11 +584,13 @@ TEST_F(ExternalDBPAEncryptorAdapterTest, DecryptCallShouldFail) { std::unique_ptr decryptor = CreateDecryptor( algorithm, column_name, key_id, data_type, compression_type, encoding_type); ASSERT_FALSE(decryptor->CanCalculateLengths()); + ASSERT_LE(ciphertext.size(), + static_cast(std::numeric_limits::max())); EXPECT_THROW( - (void) decryptor->CiphertextLength(ciphertext.size()), + (void) decryptor->CiphertextLength(static_cast(ciphertext.size())), ParquetException); EXPECT_THROW( - (void) decryptor->PlaintextLength(ciphertext.size()), + (void) decryptor->PlaintextLength(static_cast(ciphertext.size())), ParquetException); EXPECT_THROW( decryptor->Decrypt( From 37968f7872f9dc6cc7e763c125336b17ac4be3fb Mon Sep 17 00:00:00 2001 From: Marco Arguedas Date: Wed, 17 Dec 2025 20:45:50 -0600 Subject: [PATCH 06/10] Updating dbpa_library_wrapper_test.cc to avoid type-shortening errors from the compiler --- .../parquet/encryption/external/dbpa_library_wrapper_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/parquet/encryption/external/dbpa_library_wrapper_test.cc b/cpp/src/parquet/encryption/external/dbpa_library_wrapper_test.cc index 55336057d734..2404ed7dab64 100644 --- a/cpp/src/parquet/encryption/external/dbpa_library_wrapper_test.cc +++ b/cpp/src/parquet/encryption/external/dbpa_library_wrapper_test.cc @@ -729,7 +729,7 @@ TEST_F(DBPALibraryWrapperTest, MixedOperationsDelegation) { // Perform mixed encrypt and decrypt operations std::vector test_data = {"Hello", "World", "Test", "Data"}; - int call_count = test_data.size(); + auto call_count = static_cast(test_data.size()); for (const auto& data : test_data) { // Encrypt From dc342ce0c66540fce60d3d53ce9a8dcbb6df1a11 Mon Sep 17 00:00:00 2001 From: Marco Arguedas Date: Wed, 17 Dec 2025 20:48:57 -0600 Subject: [PATCH 07/10] Preventing type-shortening errors from the compiler --- cpp/src/parquet/encryption/external/dbpa_test_agent.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/parquet/encryption/external/dbpa_test_agent.cc b/cpp/src/parquet/encryption/external/dbpa_test_agent.cc index 785648a9beb1..a14f8ab198a2 100644 --- a/cpp/src/parquet/encryption/external/dbpa_test_agent.cc +++ b/cpp/src/parquet/encryption/external/dbpa_test_agent.cc @@ -49,7 +49,7 @@ class TestEncryptionResult : public EncryptionResult { return span(ciphertext_data_.data(), ciphertext_data_.size()); } - std::size_t size() const override { return static_cast(ciphertext_data_.size()); } + std::size_t size() const override { return ciphertext_data_.size(); } bool success() const override { return success_; } const std::string& error_message() const override { return error_message_; } const std::map& error_fields() const override { return error_fields_; } From 7375bb57e3623bfbd4cfdba77fccbe89f547dfcf Mon Sep 17 00:00:00 2001 From: Marco Arguedas Date: Wed, 17 Dec 2025 21:32:56 -0600 Subject: [PATCH 08/10] Removing external_dbpa_encryption_integ_test.cc --- cpp/src/parquet/encryption/CMakeLists.txt | 6 - .../external_dbpa_encryption_integ_test.cc | 486 ------------------ 2 files changed, 492 deletions(-) delete mode 100644 cpp/src/parquet/encryption/external_dbpa_encryption_integ_test.cc diff --git a/cpp/src/parquet/encryption/CMakeLists.txt b/cpp/src/parquet/encryption/CMakeLists.txt index 2dbea2c4a674..d211d778c91e 100644 --- a/cpp/src/parquet/encryption/CMakeLists.txt +++ b/cpp/src/parquet/encryption/CMakeLists.txt @@ -53,10 +53,4 @@ if(ARROW_TESTING) add_parquet_test(dbpa-executor-test SOURCES external/dbpa_executor_test.cc LABELS "parquet-tests" "encryption-tests") - - # Integration-like tests for External DBPA with various parquet settings - add_parquet_test(external-dbpa-encryption-integ_test - SOURCES external_dbpa_encryption_integ_test.cc - external/test_utils.cc - LABELS "parquet-tests" "encryption-tests") endif() diff --git a/cpp/src/parquet/encryption/external_dbpa_encryption_integ_test.cc b/cpp/src/parquet/encryption/external_dbpa_encryption_integ_test.cc deleted file mode 100644 index f558554c5e27..000000000000 --- a/cpp/src/parquet/encryption/external_dbpa_encryption_integ_test.cc +++ /dev/null @@ -1,486 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "arrow/io/memory.h" -#include "arrow/result.h" -#include "arrow/status.h" -#include "arrow/testing/gtest_compat.h" - -#include "parquet/column_reader.h" -#include "parquet/column_writer.h" -#include "parquet/encryption/encryption.h" -#include "parquet/encryption/external/test_utils.h" -#include "parquet/encryption/test_encryption_util.h" -#include "parquet/file_reader.h" -#include "parquet/file_writer.h" -#include "parquet/properties.h" -#include "parquet/schema.h" -#include "parquet/types.h" -#include "parquet/arrow/writer.h" -#include "parquet/arrow/reader.h" -#include "arrow/array/builder_primitive.h" -#include "arrow/array/builder_binary.h" -#include "arrow/table.h" -#include "arrow/util/compression.h" -#include "arrow/util/secure_string.h" - -using ::arrow::io::BufferReader; -using ::arrow::io::BufferOutputStream; -using parquet::Compression; -using parquet::Encoding; -using parquet::ParquetDataPageVersion; -using parquet::ParquetVersion; -using parquet::ReaderProperties; -using parquet::Repetition; -using parquet::Type; -using parquet::WriterProperties; -using parquet::schema::GroupNode; -using parquet::schema::PrimitiveNode; - -//Integration test for External DBPA encryption and decryption. -// main functionality is written in DoRoundtrip() and BuildParams() -namespace parquet::encryption::test { - -namespace { - -struct TestParams { - Type::type physical_type; - bool dictionary_on; - ParquetDataPageVersion dpv; - Compression::type compression; - std::optional encoding; - std::optional data_type_length; -}; - -std::string TestParamsToString(const TestParams& p) { - std::string type = parquet::TypeToString(p.physical_type); - std::string dict = p.dictionary_on ? "dict_on" : "dict_off"; - std::string ver = (p.dpv == ParquetDataPageVersion::V1) ? "v1" : "v2"; - std::string comp = ::arrow::util::Codec::GetCodecAsString(p.compression); - std::string enc = p.encoding.has_value() ? parquet::EncodingToString(p.encoding.value()) - : std::string("default"); - std::string len = (p.physical_type == Type::FIXED_LEN_BYTE_ARRAY && p.data_type_length.has_value()) - ? (std::string("_len") + std::to_string(*p.data_type_length)) - : std::string(""); - return type + "_" + dict + "_" + ver + "_" + comp + "_" + enc + len; -} - -// In here, we build the test matrix. We combine parameters for these dimensions and values. -// - physical type: INT32, BYTE_ARRAY, BOOLEAN, INT64, FLOAT, DOUBLE, FIXED_LEN_BYTE_ARRAY -// - dictionary page enabled: false -// - page version: V1 and V2 -// - compression: UNCOMPRESSED and GZIP -// - encoding: PLAIN -std::vector BuildParams() { - - // Cover an expanded set including FIXED_LEN_BYTE_ARRAY lengths; others can be added later. - std::vector types = { - Type::INT32, - Type::BYTE_ARRAY, - Type::BOOLEAN, - Type::INT64, - Type::FLOAT, - Type::DOUBLE, - Type::FIXED_LEN_BYTE_ARRAY}; - std::vector is_dictionary_page_enabled = {false}; // add true to cover dict paths - std::vector page_versions = {ParquetDataPageVersion::V1, ParquetDataPageVersion::V2}; - std::vector compressions = {Compression::UNCOMPRESSED, Compression::GZIP}; - std::vector flba_lengths = {8, 16}; - - std::vector all_test_params; - - for (auto type : types) { - for (bool dict_on : is_dictionary_page_enabled) { - for (auto page_version : page_versions) { - for (auto compression : compressions) { - if (type == Type::FIXED_LEN_BYTE_ARRAY) { - for (auto len : flba_lengths) { - if (dict_on) { - all_test_params.push_back({type, true, page_version, compression, std::nullopt, len}); - } else { - all_test_params.push_back({type, false, page_version, compression, Encoding::PLAIN, len}); - } - } - } - else if (dict_on) { - all_test_params.push_back({type, true, page_version, compression, std::nullopt, std::nullopt}); - } - else { - if (type == Type::INT32) { - all_test_params.push_back({type, false, page_version, compression, Encoding::PLAIN, std::nullopt}); - all_test_params.push_back({type, false, page_version, compression, Encoding::DELTA_BINARY_PACKED, std::nullopt}); - } - else if (type == Type::INT64) { - all_test_params.push_back({type, false, page_version, compression, Encoding::PLAIN, std::nullopt}); - all_test_params.push_back({type, false, page_version, compression, Encoding::DELTA_BINARY_PACKED, std::nullopt}); - } - else if (type == Type::BYTE_ARRAY) { - all_test_params.push_back({type, false, page_version, compression, Encoding::PLAIN, std::nullopt}); - all_test_params.push_back({type, false, page_version, compression, Encoding::DELTA_LENGTH_BYTE_ARRAY, std::nullopt}); - all_test_params.push_back({type, false, page_version, compression, Encoding::DELTA_BYTE_ARRAY, std::nullopt}); - } - else if (type == Type::BOOLEAN) { - all_test_params.push_back({type, false, page_version, compression, Encoding::RLE, std::nullopt}); - } - else { - all_test_params.push_back({type, false, page_version, compression, std::nullopt, std::nullopt}); - } - } - } - } - } - } - return all_test_params; -} - -// This is use to name/identify each of the test cases. -std::string ParamName(const testing::TestParamInfo& info) { - return TestParamsToString(info.param); -} - -class ExternalDbpaIntegrationTest : public ::testing::TestWithParam { - protected: - void SetUp() override { - - // Default library path, can be overridden by environment variable DBPA_LIBRARY_PATH - library_path_ = "libdbpsRemoteAgent.so"; - if (const char* lib_env = std::getenv("DBPA_LIBRARY_PATH")) { - if (*lib_env != '\0') { - library_path_ = std::string(lib_env); - } - } - - // Build shared connection_config for both encryption and decryption - std::map algo_config; - algo_config["agent_library_path"] = library_path_; - // If using a remote agent, attach the connection config file path - { - std::string library_path_lower = library_path_; - std::transform(library_path_lower.begin(), library_path_lower.end(), - library_path_lower.begin(), ::tolower); - const bool is_remote_agent = library_path_lower.find("remote") != std::string::npos; - if (is_remote_agent) { - std::string config_file_name = "test_connection_config_file.json"; - if (const char* cfg_env = std::getenv("DBPA_CONFIG_FILE_NAME")) { - if (*cfg_env != '\0') { - config_file_name = std::string(cfg_env); - } - } - - if (!std::filesystem::exists(config_file_name)) { - FAIL() << "Connection config [" << config_file_name << "] file not found"; - } - algo_config["connection_config_file_path"] = config_file_name; - } - } - - connection_config_[parquet::ParquetCipher::EXTERNAL_DBPA_V1] = std::move(algo_config); - - // Default number of rows for test input tables - num_rows_ = 10; - } - - // Build a single-column Arrow table according to the given params. - // Only a subset of types is supported for now. - std::shared_ptr<::arrow::Table> MakeInputTable(Type::type physical_type, - bool dictionary_on, - const std::optional& data_type_length) { - if (physical_type == Type::INT32) { - return MakeInt32Table(dictionary_on); - } - else if (physical_type == Type::BYTE_ARRAY) { - return MakeByteArrayTable(dictionary_on); - } - else if (physical_type == Type::BOOLEAN) { - return MakeBooleanTable(dictionary_on); - } - else if (physical_type == Type::INT64) { - return MakeInt64Table(dictionary_on); - } - else if (physical_type == Type::FLOAT) { - return MakeFloatTable(dictionary_on); - } - else if (physical_type == Type::DOUBLE) { - return MakeDoubleTable(dictionary_on); - } - else if (physical_type == Type::FIXED_LEN_BYTE_ARRAY) { - if (!data_type_length.has_value()) { - return nullptr; - } - return MakeFixedSizeBinaryTable(dictionary_on, *data_type_length); - } - else { - return nullptr; - } - } - - std::shared_ptr<::arrow::Table> MakeInt32Table(bool dictionary_on) { - ::arrow::Int32Builder builder; - for (int64_t i = 0; i < num_rows_; ++i) { - int32_t value = dictionary_on ? static_cast(i % 16) - : static_cast(i); - auto st = builder.Append(value); - EXPECT_TRUE(st.ok()) << st.ToString(); - } - std::shared_ptr<::arrow::Array> array; - { - auto st = builder.Finish(&array); - EXPECT_TRUE(st.ok()) << st.ToString(); - } - return ::arrow::Table::Make( - ::arrow::schema({::arrow::field("col", ::arrow::int32())}), {array}); - } - - std::shared_ptr<::arrow::Table> MakeByteArrayTable(bool dictionary_on) { - ::arrow::StringBuilder builder; - for (int64_t i = 0; i < num_rows_; ++i) { - std::string s = dictionary_on ? std::string("k") + std::to_string(i % 8) - : std::string("val_") + std::to_string(i); - auto st = builder.Append(s); - EXPECT_TRUE(st.ok()) << st.ToString(); - } - std::shared_ptr<::arrow::Array> array; - { - auto st = builder.Finish(&array); - EXPECT_TRUE(st.ok()) << st.ToString(); - } - return ::arrow::Table::Make( - ::arrow::schema({::arrow::field("col", ::arrow::utf8())}), {array}); - } - - std::shared_ptr<::arrow::Table> MakeBooleanTable(bool dictionary_on) { - ::arrow::BooleanBuilder builder; - for (int64_t i = 0; i < num_rows_; ++i) { - bool value = dictionary_on ? (i % 2 == 0) : (i % 3 == 0); - auto st = builder.Append(value); - EXPECT_TRUE(st.ok()) << st.ToString(); - } - std::shared_ptr<::arrow::Array> array; - { - auto st = builder.Finish(&array); - EXPECT_TRUE(st.ok()) << st.ToString(); - } - return ::arrow::Table::Make( - ::arrow::schema({::arrow::field("col", ::arrow::boolean())}), {array}); - } - - std::shared_ptr<::arrow::Table> MakeInt64Table(bool dictionary_on) { - ::arrow::Int64Builder builder; - for (int64_t i = 0; i < num_rows_; ++i) { - int64_t value = dictionary_on ? static_cast(i % 16) : static_cast(i); - auto st = builder.Append(value); - EXPECT_TRUE(st.ok()) << st.ToString(); - } - std::shared_ptr<::arrow::Array> array; - { - auto st = builder.Finish(&array); - EXPECT_TRUE(st.ok()) << st.ToString(); - } - return ::arrow::Table::Make( - ::arrow::schema({::arrow::field("col", ::arrow::int64())}), {array}); - } - - std::shared_ptr<::arrow::Table> MakeFloatTable(bool dictionary_on) { - ::arrow::FloatBuilder builder; - for (int64_t i = 0; i < num_rows_; ++i) { - float value = dictionary_on ? static_cast(i % 16) : static_cast(i); - auto st = builder.Append(value); - EXPECT_TRUE(st.ok()) << st.ToString(); - } - std::shared_ptr<::arrow::Array> array; - { - auto st = builder.Finish(&array); - EXPECT_TRUE(st.ok()) << st.ToString(); - } - return ::arrow::Table::Make( - ::arrow::schema({::arrow::field("col", ::arrow::float32())}), {array}); - } - - std::shared_ptr<::arrow::Table> MakeDoubleTable(bool dictionary_on) { - ::arrow::DoubleBuilder builder; - for (int64_t i = 0; i < num_rows_; ++i) { - double value = dictionary_on ? static_cast(i % 16) : static_cast(i); - auto st = builder.Append(value); - EXPECT_TRUE(st.ok()) << st.ToString(); - } - std::shared_ptr<::arrow::Array> array; - { - auto st = builder.Finish(&array); - EXPECT_TRUE(st.ok()) << st.ToString(); - } - return ::arrow::Table::Make( - ::arrow::schema({::arrow::field("col", ::arrow::float64())}), {array}); - } - - std::shared_ptr<::arrow::Table> MakeFixedSizeBinaryTable(bool dictionary_on, int32_t byte_width) { - auto type = ::arrow::fixed_size_binary(byte_width); - ::arrow::FixedSizeBinaryBuilder builder(type); - for (int64_t i = 0; i < num_rows_; ++i) { - std::string s; - if (dictionary_on) { - int64_t k = i % 8; - s = std::string(byte_width, static_cast('A' + (k % 26))); - } else { - s.resize(byte_width); - for (int32_t j = 0; j < byte_width; ++j) { - s[j] = static_cast('a' + ((i + j) % 26)); - } - } - auto st = builder.Append(reinterpret_cast(s.data())); - EXPECT_TRUE(st.ok()) << st.ToString(); - } - std::shared_ptr<::arrow::Array> array; - { - auto st = builder.Finish(&array); - EXPECT_TRUE(st.ok()) << st.ToString(); - } - return ::arrow::Table::Make( - ::arrow::schema({::arrow::field("col", ::arrow::fixed_size_binary(byte_width))}), {array}); - } - - // Build ExternalFileEncryptionProperties using EXTERNAL_DBPA_V1 for the column. - std::shared_ptr MakeExternalEncryptionProperties( - const std::string& column_path) { - std::map> cols; - parquet::ColumnEncryptionProperties::Builder col_builder(column_path); - col_builder.key(kColumnEncryptionKey2)->key_id("kc2"); - col_builder.parquet_cipher(parquet::ParquetCipher::EXTERNAL_DBPA_V1); - cols[column_path] = col_builder.build(); - - parquet::ExternalFileEncryptionProperties::Builder feb(kFooterEncryptionKey); - feb.footer_key_metadata("kf") - ->encrypted_columns(cols) - ->algorithm(parquet::ParquetCipher::AES_GCM_V1) - ->app_context(app_context_) - // Use shared connection_config built in SetUp() - ->connection_config(connection_config_); - return feb.build_external(); - } - - std::shared_ptr MakeExternalDecryptionProperties() { - auto kr = std::make_shared(); - kr->PutKey("kf", kFooterEncryptionKey); - kr->PutKey("kc2", kColumnEncryptionKey2); - parquet::ExternalFileDecryptionProperties::Builder fdb; - fdb.key_retriever(kr) - ->app_context(app_context_) - // Use shared connection_config built in SetUp() - ->connection_config(connection_config_); - return fdb.build_external(); - } - - void DoRoundtrip(const TestParams& p) { - - // Build Arrow table - std::shared_ptr<::arrow::Table> input_table = MakeInputTable( - p.physical_type, p.dictionary_on, p.data_type_length); - if (!input_table) { - GTEST_SKIP() << "Type not covered in this test variant"; - } - - // Writer properties - WriterProperties::Builder wpb; - wpb.version(ParquetVersion::PARQUET_2_6) - ->data_page_version(p.dpv) - ->data_pagesize(16 * 1024) - ->compression(p.compression); - // Configure dictionary usage at the writer level: - // - enable_dictionary(): allow dictionary encoding; combined with low-cardinality data - // above, this yields a dictionary page. - // - disable_dictionary(): force non-dictionary encoding; if an explicit Encoding was - // provided in the test params, set it for column "col" to exercise different - // non-dictionary encodings. - if (p.dictionary_on) { - wpb.enable_dictionary(); - wpb.dictionary_pagesize_limit(8 * 1024); - } - else { - wpb.disable_dictionary(); - if (p.encoding.has_value()) { - wpb.encoding("col", p.encoding.value()); - } - } - auto enc_props = MakeExternalEncryptionProperties("col"); - wpb.encryption(enc_props); - auto writer_props = wpb.build(); - - // Write using high-level parquet::arrow API - auto sink_res = BufferOutputStream::Create(); - ASSERT_TRUE(sink_res.ok()) << sink_res.status().ToString(); - auto sink = *sink_res; - { - auto st = parquet::arrow::WriteTable(*input_table, ::arrow::default_memory_pool(), sink, - /*chunk_size=*/num_rows_, writer_props); - ASSERT_TRUE(st.ok()) << st.ToString(); - } - auto buffer_res = sink->Finish(); - ASSERT_TRUE(buffer_res.ok()) << buffer_res.status().ToString(); - auto buffer = *buffer_res; - - // Read back using parquet::arrow API - parquet::ReaderProperties rp = parquet::default_reader_properties(); - rp.file_decryption_properties(MakeExternalDecryptionProperties()); - parquet::arrow::FileReaderBuilder frb; - { - auto st = frb.Open(std::make_shared(buffer), rp); - ASSERT_TRUE(st.ok()) << st.ToString(); - } - std::unique_ptr fr; - { - auto st = frb.Build(&fr); - ASSERT_TRUE(st.ok()) << st.ToString(); - } - std::shared_ptr<::arrow::Table> output_table; - { - auto st = fr->ReadTable(&output_table); - ASSERT_TRUE(st.ok()) << st.ToString(); - } - - // Assert equality - ASSERT_TRUE(output_table->Equals(*input_table)); - } - - std::string app_context_ = - "{\"user_id\": \"abc123\", \"location\": {\"lat\": 9.7489, \"lon\": -83.7534}}"; - std::string library_path_; - std::map> connection_config_; - //::arrow::util::SecureString kFooterEncryptionKey_(kFooterEncryptionKey); - //::arrow::util::SecureString kColumnEncryptionKey2_(kColumnEncryptionKey2); - int64_t num_rows_; -}; - -TEST_P(ExternalDbpaIntegrationTest, Roundtrip) { DoRoundtrip(GetParam()); } - -INSTANTIATE_TEST_SUITE_P(DBPAIntegration, - ExternalDbpaIntegrationTest, - ::testing::ValuesIn(BuildParams()), ParamName); -} // namespace - -} // namespace parquet::encryption::test From 537de52e405144c8f24dcc8b7521b1fc3bc4323e Mon Sep 17 00:00:00 2001 From: Marco Arguedas Date: Thu, 18 Dec 2025 17:11:45 -0600 Subject: [PATCH 09/10] Ensuring that memcpy() is never called with an array of size 0. Added a regression test --- .../parquet/encryption/external_dbpa_encryption.cc | 8 ++++++-- .../encryption/external_dbpa_encryption_test.cc | 13 +++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/cpp/src/parquet/encryption/external_dbpa_encryption.cc b/cpp/src/parquet/encryption/external_dbpa_encryption.cc index 7ab5b3b0c5f3..bf35d649efd4 100644 --- a/cpp/src/parquet/encryption/external_dbpa_encryption.cc +++ b/cpp/src/parquet/encryption/external_dbpa_encryption.cc @@ -367,7 +367,9 @@ int32_t ExternalDBPAEncryptorAdapter::InvokeExternalEncrypt( } ARROW_LOG(DEBUG) << "Copying result to ciphertext buffer..."; - std::memcpy(ciphertext->mutable_data(), result->ciphertext().data(), ciphertext_size); + if (ciphertext_size > 0) { + std::memcpy(ciphertext->mutable_data(), result->ciphertext().data(), ciphertext_size); + } ARROW_LOG(DEBUG) << "Encryption completed successfully"; // Accumulate any column_encryption_metadata returned by the result per module type @@ -612,7 +614,9 @@ int32_t ExternalDBPADecryptorAdapter::InvokeExternalDecrypt( } ARROW_LOG(DEBUG) << "Copying result to plaintext buffer..."; - std::memcpy(plaintext->mutable_data(), result->plaintext().data(), plaintext_size); + if (plaintext_size > 0) { + std::memcpy(plaintext->mutable_data(), result->plaintext().data(), plaintext_size); + } ARROW_LOG(DEBUG) << "Decryption completed successfully"; const auto total_size64 = result->size(); diff --git a/cpp/src/parquet/encryption/external_dbpa_encryption_test.cc b/cpp/src/parquet/encryption/external_dbpa_encryption_test.cc index 6bf1c3fb72c7..7bdda314b69b 100644 --- a/cpp/src/parquet/encryption/external_dbpa_encryption_test.cc +++ b/cpp/src/parquet/encryption/external_dbpa_encryption_test.cc @@ -158,6 +158,19 @@ TEST_F(ExternalDBPAEncryptorAdapterTest, RoundtripEncryptionSucceeds) { algorithm, column_name, key_id, data_type, compression_type, encoding_type, plaintext); } +TEST_F(ExternalDBPAEncryptorAdapterTest, RoundtripEncryption_EmptyPlaintextDoesNotCrash) { + ParquetCipher::type algorithm = ParquetCipher::EXTERNAL_DBPA_V1; + std::string column_name = "employee_name"; + std::string key_id = "employee_name_key"; + Type::type data_type = Type::BYTE_ARRAY; + Compression::type compression_type = Compression::UNCOMPRESSED; + Encoding::type encoding_type = Encoding::PLAIN; + std::string plaintext = ""; + + RoundtripEncryption( + algorithm, column_name, key_id, data_type, compression_type, encoding_type, plaintext); +} + TEST_F(ExternalDBPAEncryptorAdapterTest, GetKeyValueMetadataReturnsNullWhenEmpty) { ParquetCipher::type algorithm = ParquetCipher::EXTERNAL_DBPA_V1; std::string column_name = "employee_name"; From b63ada7c9f72cff237188dd7f16bf4be24f17429 Mon Sep 17 00:00:00 2001 From: Marco Arguedas Date: Thu, 18 Dec 2025 18:38:26 -0600 Subject: [PATCH 10/10] In types_test.cc removing a (somewhat redundant) test which caused the sanitizer to be unhappy --- cpp/src/parquet/types_test.cc | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cpp/src/parquet/types_test.cc b/cpp/src/parquet/types_test.cc index 47ff34d4633f..c1f24a30993a 100644 --- a/cpp/src/parquet/types_test.cc +++ b/cpp/src/parquet/types_test.cc @@ -218,11 +218,6 @@ TEST(TestIsParquetCipherSupported, SupportedCiphers) { ASSERT_TRUE(IsParquetCipherSupported(ParquetCipher::EXTERNAL_DBPA_V1)); } -TEST(TestIsParquetCipherSupported, UnsupportedCiphers) { - ParquetCipher::type unsupported_cipher = static_cast(100); - ASSERT_FALSE(IsParquetCipherSupported(unsupported_cipher)); -} - #if !(defined(_WIN32) || defined(__CYGWIN__)) # pragma GCC diagnostic pop #elif _MSC_VER