diff --git a/cpp/src/parquet/CMakeLists.txt b/cpp/src/parquet/CMakeLists.txt index 49c26ba606d5..382c70d4be8f 100644 --- a/cpp/src/parquet/CMakeLists.txt +++ b/cpp/src/parquet/CMakeLists.txt @@ -242,7 +242,8 @@ if(PARQUET_REQUIRE_ENCRYPTION) set(PARQUET_SRCS ${PARQUET_SRCS} encryption/encryption_internal.cc encryption/openssl_internal.cc encryption/external/loadable_encryptor_utils.cc - encryption/external/dbpa_library_wrapper.cc) + encryption/external/dbpa_library_wrapper.cc + encryption/external/dbpa_utils.cc) # Encryption key management set(PARQUET_SRCS ${PARQUET_SRCS} diff --git a/cpp/src/parquet/encryption/encryption_internal.cc b/cpp/src/parquet/encryption/encryption_internal.cc index 30eb8bb7474f..e0070e8ad4e2 100644 --- a/cpp/src/parquet/encryption/encryption_internal.cc +++ b/cpp/src/parquet/encryption/encryption_internal.cc @@ -32,12 +32,25 @@ #include "parquet/encryption/openssl_internal.h" #include "parquet/exception.h" +#include "parquet/encryption/external/loadable_encryptor_utils.h" +#include "parquet/encryption/external/dbpa_utils.h" +#include "parquet/types.h" using ::arrow::util::span; using parquet::ParquetException; +using parquet::encryption::external::LoadableEncryptorUtils; +using parquet::encryption::external::dbpa_utils; + +using dbps::external::EncryptionResult; +using dbps::external::DecryptionResult; +using dbps::external::DataBatchProtectionAgentInterface; namespace parquet::encryption { +// Forward declarations for utility functions +std::string HackTypeToString(Type::type t); +std::string CompressName(Compression::type codec); + #define ENCRYPT_INIT(CTX, ALG) \ if (1 != EVP_EncryptInit_ex(CTX, ALG, nullptr, nullptr, nullptr)) { \ throw ParquetException("Couldn't init ALG encryption"); \ @@ -586,31 +599,87 @@ void RandBytes(unsigned char* buf, size_t num) { void EnsureBackendInitialized() { openssl::EnsureInitialized(); } -ExternalEncryptorImpl::ExternalEncryptorImpl(ParquetCipher::type alg_id, int32_t key_len, - std::string column_name, Type::type data_type, - Compression::type compression_type, - Encoding::type encoding, std::string ext_column_key, - std::string user_id, std::string app_context, - bool metadata, bool write_length) - : column_name_(column_name), data_type_(data_type), compression_type_(compression_type), - encoding_(encoding), ext_column_key_(ext_column_key), user_id_(user_id), - app_context_(app_context), aes_encryptor_{std::unique_ptr( - new AesEncryptorImpl(alg_id, key_len, metadata, write_length))} { - std::cout << "Created ExternalEncryptorImpl" << std::endl; +ExternalEncryptorImpl::ExternalEncryptorImpl( + std::unique_ptr agent_instance) + : agent_instance_(std::move(agent_instance)) { + std::cout << "[DEBUG] ExternalEncryptorImpl constructor called" << std::endl; } std::unique_ptr ExternalEncryptorImpl::Make( ParquetCipher::type alg_id, int32_t key_len, std::string column_name, Type::type data_type, Compression::type compression_type, Encoding::type encoding, std::string ext_column_key, std::string user_id, std::string app_context, bool metadata, bool write_length) { - return std::make_unique(alg_id, key_len, column_name, data_type, - compression_type, encoding, ext_column_key, user_id, app_context, metadata, write_length); + + std::cout << "[DEBUG] ExternalEncryptorImpl::Make called with parameters:" << std::endl; + std::cout << " alg_id: " << alg_id << std::endl; + std::cout << " key_len: " << key_len << std::endl; + std::cout << " column_name: " << column_name << std::endl; + std::cout << " data_type: " << HackTypeToString(data_type) << std::endl; + std::cout << " compression_type: " << CompressName(compression_type) << std::endl; + std::cout << " encoding: " << encoding << std::endl; + std::cout << " ext_column_key: " << ext_column_key << std::endl; + std::cout << " user_id: " << user_id << std::endl; + std::cout << " app_context: " << app_context << std::endl; + std::cout << " metadata: " << (metadata ? "true" : "false") << std::endl; + std::cout << " write_length: " << (write_length ? "true" : "false") << std::endl; + + //TODO: this should be a config parameter + std::string dbpa_library_path = "libDBPATestAgent.so"; + std::cout << "[DEBUG] Loading DBPA agent from: " << dbpa_library_path << std::endl; + auto dbpa_agent = LoadableEncryptorUtils::LoadFromLibrary(dbpa_library_path); + if (!dbpa_agent) { + std::cout << "[ERROR] Failed to create instance of DataBatchProtectionAgentInterface" << std::endl; + throw ParquetException("Failed to create instance of DataBatchProtectionAgentInterface"); + } + std::cout << "[DEBUG] Successfully loaded DBPA agent" << std::endl; + + std::cout << "[DEBUG] Initializing DBPA agent..." << std::endl; + dbpa_agent->init( + /*column_name*/ column_name, + /*connection_config*/ std::map{}, + /*app_context*/ app_context, + /*column_key_id*/ ext_column_key, + /*data_type*/ dbpa_utils::ParquetTypeToExternal(data_type), + /*compression_type*/ dbpa_utils::ArrowCompressionToExternal(compression_type) + ); + std::cout << "[DEBUG] DBPA agent initialized successfully" << std::endl; + + auto result = std::make_unique(std::move(dbpa_agent)); + std::cout << "[DEBUG] ExternalEncryptorImpl created successfully" << std::endl; + return result; } int32_t ExternalEncryptorImpl::Encrypt(span plaintext, span key, span aad, span ciphertext) { - ConstructExternalCall(plaintext); - return aes_encryptor_->Encrypt(plaintext, key, aad, ciphertext); + + std::cout << "[DEBUG] ExternalEncryptorImpl::Encrypt called" << std::endl; + std::cout << " plaintext size: " << plaintext.size() << " bytes" << std::endl; + std::cout << " key size: " << key.size() << " bytes" << std::endl; + std::cout << " aad size: " << aad.size() << " bytes" << std::endl; + std::cout << " ciphertext buffer size: " << ciphertext.size() << " bytes" << std::endl; + + std::cout << "[DEBUG] Calling agent_instance_->Encrypt..." << std::endl; + std::unique_ptr result = agent_instance_->Encrypt(plaintext); + + if (!result->success()) { + std::cout << "[ERROR] Encryption failed: " << result->error_message() << std::endl; + throw ParquetException(result->error_message()); + } + std::cout << "[DEBUG] Encryption successful" << std::endl; + std::cout << " result size: " << result->size() << " bytes" << std::endl; + std::cout << " result ciphertext size: " << result->ciphertext().size() << " bytes" << std::endl; + + if (ciphertext.size() < result->ciphertext().size()) { + std::cout << "[ERROR] Ciphertext buffer too small. Need " << result->ciphertext().size() + << " bytes, have " << ciphertext.size() << " bytes" << std::endl; + throw ParquetException("Ciphertext buffer too small for encrypted result"); + } + + std::cout << "[DEBUG] Copying result to ciphertext buffer..." << std::endl; + std::copy(result->ciphertext().begin(), result->ciphertext().end(), ciphertext.begin()); + std::cout << "[DEBUG] Encryption completed successfully" << std::endl; + + return result->size(); } int32_t ExternalEncryptorImpl::SignedFooterEncrypt(span footer, @@ -618,11 +687,50 @@ int32_t ExternalEncryptorImpl::SignedFooterEncrypt(span footer, span aad, span nonce, span encrypted_footer) { - return aes_encryptor_->SignedFooterEncrypt(footer, key, aad, nonce, encrypted_footer); + std::cout << "[DEBUG] ExternalEncryptorImpl::SignedFooterEncrypt called" << std::endl; + std::cout << " footer size: " << footer.size() << " bytes" << std::endl; + std::cout << " key size: " << key.size() << " bytes" << std::endl; + std::cout << " aad size: " << aad.size() << " bytes" << std::endl; + std::cout << " nonce size: " << nonce.size() << " bytes" << std::endl; + std::cout << " encrypted_footer buffer size: " << encrypted_footer.size() << " bytes" << std::endl; + + std::cout << "[DEBUG] Calling agent_instance_->Encrypt for footer..." << std::endl; + std::unique_ptr result = agent_instance_->Encrypt(footer); + if (!result->success()) { + std::cout << "[ERROR] Footer encryption failed: " << result->error_message() << std::endl; + throw ParquetException(result->error_message()); + } + std::cout << "[DEBUG] Footer encryption successful" << std::endl; + std::cout << " result size: " << result->size() << " bytes" << std::endl; + + if (encrypted_footer.size() < result->ciphertext().size()) { + std::cout << "[ERROR] Encrypted footer buffer too small. Need " << result->ciphertext().size() + << " bytes, have " << encrypted_footer.size() << " bytes" << std::endl; + throw ParquetException("Encrypted footer buffer too small for encrypted result"); + } + + std::cout << "[DEBUG] Copying footer result to encrypted_footer buffer..." << std::endl; + std::copy(result->ciphertext().begin(), result->ciphertext().end(), encrypted_footer.begin()); + std::cout << "[DEBUG] Footer encryption completed successfully" << std::endl; + + return result->size(); } int32_t ExternalEncryptorImpl::CiphertextLength(int64_t plaintext_len) const { - return aes_encryptor_->CiphertextLength(plaintext_len); + std::cout << "[DEBUG] ExternalEncryptorImpl::CiphertextLength called with plaintext_len: " << plaintext_len << std::endl; + //TODO + // This is not production code. We know that the one DPBA Agent we have uses XOR encryption. + // Therefore it's safe to assume that the ciphertext length is the same as the plaintext length. + // This is not true for all DPBA Agents. + if (plaintext_len < 0) { + std::cout << "[ERROR] Negative plaintext length: " << plaintext_len << std::endl; + std::stringstream ss; + ss << "Negative plaintext length " << plaintext_len; + throw ParquetException(ss.str()); + } + int32_t result = static_cast(plaintext_len); + std::cout << "[DEBUG] CiphertextLength returning: " << result << std::endl; + return result; } std::string HackTypeToString(Type::type t) { @@ -680,99 +788,170 @@ void ExternalEncryptorImpl::ConstructExternalCall(span plaintext) std::cout << "Payload:" << std::endl; - std::cout << "{" << std::endl; - std::cout << " \"columnReference\": {" << std::endl; - std::cout << " \"column\": \"" << column_name_ << "\"" << std::endl; - std::cout << " }," << std::endl; - - std::cout << " \"dataBatch\": {" << std::endl; - std::cout << " \"compressed value (hex)\": \""; - PrintSpan(plaintext); - std::cout << " \"datatype\": \"" << HackTypeToString(data_type_) << "\"," << std::endl; - std::cout << " \"serialization\": {" << std::endl; - std::cout << " \"compression\": \"" << CompressName(compression_type_) << "\"," << std::endl; - std::cout << " \"format\": \"" << MapArrowEncodingToExternalFormat(encoding_) << "\"," - << std::endl; - std::cout << " \"encoding\": \"" << MapArrowEncodingToExternalEncoding(encoding_) << "\"," - << std::endl; - std::cout << " }," << std::endl; - std::cout << " }," << std::endl; - - std::cout << " \"encryption\": {" << std::endl; - std::cout << " " << ext_column_key_ << std::endl; - std::cout << " }," << std::endl; - - std::cout << " \"access\": {" << std::endl; - std::cout << " \"userId\": \"" << user_id_ << "\"," << std::endl; - std::cout << " }," << std::endl; - - std::cout << " \"application_context\": {" << std::endl; - std::cout << app_context_ << "\"," << std::endl; - std::cout << " }," << std::endl; - - std::cout << "}" << std::endl; - - std::cout << "!*!*!*!*!*!* END ExternalEncryptorImpl:ConstructExternalCall!*!*!*!*!*!*\n\n" << std::endl; } -ExternalDecryptorImpl::ExternalDecryptorImpl(ParquetCipher::type alg_id, int32_t key_len, - bool metadata, bool contains_length) - : aes_decryptor_{std::unique_ptr( - new AesDecryptorImpl(alg_id, key_len, metadata, contains_length))} { - std::cout << "Created ExternalDecryptorImpl" << std::endl; +ExternalDecryptorImpl::ExternalDecryptorImpl( + std::unique_ptr agent_instance) + : agent_instance_(std::move(agent_instance)) { + std::cout << "[DEBUG] ExternalDecryptorImpl constructor called" << std::endl; } -std::unique_ptr ExternalDecryptorImpl::Make(ParquetCipher::type alg_id, - int32_t key_len, bool metadata){ - return std::make_unique(alg_id, key_len, metadata); +std::unique_ptr ExternalDecryptorImpl::Make( + ParquetCipher::type alg_id, int32_t key_len, std::string column_name, Type::type data_type, + Compression::type compression_type, Encoding::type encoding, std::string ext_column_key, + std::string user_id, std::string app_context, bool metadata, bool contains_length) { + + std::cout << "[DEBUG] ExternalDecryptorImpl::Make (full) called with parameters:" << std::endl; + std::cout << " alg_id: " << alg_id << std::endl; + std::cout << " key_len: " << key_len << std::endl; + std::cout << " column_name: " << column_name << std::endl; + std::cout << " data_type: " << HackTypeToString(data_type) << std::endl; + std::cout << " compression_type: " << CompressName(compression_type) << std::endl; + std::cout << " encoding: " << encoding << std::endl; + std::cout << " ext_column_key: " << ext_column_key << std::endl; + std::cout << " user_id: " << user_id << std::endl; + std::cout << " app_context: " << app_context << std::endl; + std::cout << " metadata: " << (metadata ? "true" : "false") << std::endl; + std::cout << " contains_length: " << (contains_length ? "true" : "false") << std::endl; + + //TODO: this should be a config parameter + std::string dbpa_library_path = "libDBPATestAgent.so"; + std::cout << "[DEBUG] Loading DBPA agent from: " << dbpa_library_path << std::endl; + auto dbpa_agent = LoadableEncryptorUtils::LoadFromLibrary(dbpa_library_path); + if (!dbpa_agent) { + std::cout << "[ERROR] Failed to create instance of DataBatchProtectionAgentInterface" << std::endl; + throw ParquetException("Failed to create instance of DataBatchProtectionAgentInterface"); + } + std::cout << "[DEBUG] Successfully loaded DBPA agent" << std::endl; + + std::cout << "[DEBUG] Initializing DBPA agent..." << std::endl; + dbpa_agent->init( + /*column_name*/ column_name, + /*connection_config*/ std::map{}, + /*app_context*/ app_context, + /*column_key_id*/ ext_column_key, + /*data_type*/ dbpa_utils::ParquetTypeToExternal(data_type), + /*compression_type*/ dbpa_utils::ArrowCompressionToExternal(compression_type) + ); + std::cout << "[DEBUG] DBPA agent initialized successfully" << std::endl; + + auto result = std::make_unique(std::move(dbpa_agent)); + std::cout << "[DEBUG] ExternalDecryptorImpl created successfully" << std::endl; + return result; +} + +std::unique_ptr ExternalDecryptorImpl::Make(ParquetCipher::type alg_id, int32_t key_len, bool metadata) { + std::cout << "[DEBUG] ExternalDecryptorImpl::Make (simple) called with parameters:" << std::endl; + std::cout << " alg_id: " << alg_id << std::endl; + std::cout << " key_len: " << key_len << std::endl; + std::cout << " metadata: " << (metadata ? "true" : "false") << std::endl; + + //TODO: this should be a config parameter + std::string dbpa_library_path = "dbpa_agent.so"; + std::cout << "[DEBUG] Loading DBPA agent from: " << dbpa_library_path << std::endl; + auto dbpa_agent = LoadableEncryptorUtils::LoadFromLibrary(dbpa_library_path); + if (!dbpa_agent) { + std::cout << "[ERROR] Failed to create instance of DataBatchProtectionAgentInterface" << std::endl; + throw ParquetException("Failed to create instance of DataBatchProtectionAgentInterface"); + } + std::cout << "[DEBUG] Successfully loaded DBPA agent" << std::endl; + + // For simple cases (like footer decryption), we use default values + std::cout << "[DEBUG] Initializing DBPA agent with default values..." << std::endl; + dbpa_agent->init( + /*column_name*/ "footer", // Default column name for footer + /*connection_config*/ std::map{}, + /*app_context*/ "default", // Default app context + /*column_key_id*/ "default", // Default column key + /*data_type*/ dbps::external::Type::BYTE_ARRAY, // Default to byte array for footer + /*compression_type*/ dbps::external::CompressionCodec::UNCOMPRESSED // Default to uncompressed + ); + std::cout << "[DEBUG] DBPA agent initialized successfully with default values" << std::endl; + + auto result = std::make_unique(std::move(dbpa_agent)); + std::cout << "[DEBUG] ExternalDecryptorImpl created successfully" << std::endl; + return result; } int32_t ExternalDecryptorImpl::Decrypt(span ciphertext, span key, span aad, span plaintext) { - std::cout << "ExternalDecryptorImpl::Decrypt called" << std::endl; - std::cout << "ciphertext size: " << ciphertext.size() << std::endl; - std::cout << "key size: " << key.size() << std::endl; - std::cout << "aad size: " << aad.size() << std::endl; - std::cout << "plaintext size: " << plaintext.size() << std::endl; - ConstructExternalCall(); + std::cout << "[DEBUG] ExternalDecryptorImpl::Decrypt called" << std::endl; + std::cout << " ciphertext size: " << ciphertext.size() << " bytes" << std::endl; + std::cout << " key size: " << key.size() << " bytes" << std::endl; + std::cout << " aad size: " << aad.size() << " bytes" << std::endl; + std::cout << " plaintext buffer size: " << plaintext.size() << " bytes" << std::endl; - std::cout << "About to call aes_decryptor_->Decrypt()" << std::endl; - std::cout << "About to call aes_decryptor_->Decrypt()" << std::endl; - std::cout << "ciphertext first few bytes: "; - for (int i = 0; i < std::min(8, (int)ciphertext.size()); i++) { - std::cout << std::hex << std::setw(2) << std::setfill('0') << (int)ciphertext[i] << " "; + std::cout << "[DEBUG] Calling agent_instance_->Decrypt..." << std::endl; + std::unique_ptr result = agent_instance_->Decrypt(ciphertext); + + if (!result->success()) { + std::cout << "[ERROR] Decryption failed: " << result->error_message() << std::endl; + throw ParquetException(result->error_message()); } - std::cout << std::dec << std::endl; + std::cout << "[DEBUG] Decryption successful" << std::endl; + std::cout << " result size: " << result->size() << " bytes" << std::endl; + std::cout << " result plaintext size: " << result->plaintext().size() << " bytes" << std::endl; - std::cout << "key first few bytes: "; - for (int i = 0; i < std::min(8, (int)key.size()); i++) { - std::cout << std::hex << std::setw(2) << std::setfill('0') << (int)key[i] << " "; + if (plaintext.size() < result->plaintext().size()) { + std::cout << "[ERROR] Plaintext buffer too small. Need " << result->plaintext().size() + << " bytes, have " << plaintext.size() << " bytes" << std::endl; + throw ParquetException("Plaintext buffer too small for decrypted result"); } - std::cout << std::dec << std::endl; + + std::cout << "[DEBUG] Copying result to plaintext buffer..." << std::endl; + std::copy(result->plaintext().begin(), result->plaintext().end(), plaintext.begin()); + std::cout << "[DEBUG] Decryption completed successfully" << std::endl; - std::cout << "aad first few bytes: "; - for (int i = 0; i < std::min(8, (int)aad.size()); i++) { - std::cout << std::hex << std::setw(2) << std::setfill('0') << (int)aad[i] << " "; - } - std::cout << std::dec << std::endl; - int32_t result = aes_decryptor_->Decrypt(ciphertext, key, aad, plaintext); - std::cout << "aes_decryptor_->Decrypt() returned: " << result << std::endl; - return result; + return result->size(); } int32_t ExternalDecryptorImpl::PlaintextLength(int32_t ciphertext_len) const { - return aes_decryptor_->PlaintextLength(ciphertext_len); + std::cout << "[DEBUG] ExternalDecryptorImpl::PlaintextLength called with ciphertext_len: " << ciphertext_len << std::endl; + //TODO + // This is not production code. We know that the one DPBA Agent we have uses XOR encryption. + // Therefore it's safe to assume that the plaintext length is the same as the ciphertext length. + // This is not true for all DPBA Agents. + if (ciphertext_len < 0) { + std::cout << "[ERROR] Negative ciphertext length: " << ciphertext_len << std::endl; + std::stringstream ss; + ss << "Negative ciphertext length " << ciphertext_len; + throw ParquetException(ss.str()); + } + int32_t result = ciphertext_len; + std::cout << "[DEBUG] PlaintextLength returning: " << result << std::endl; + return result; } int32_t ExternalDecryptorImpl::CiphertextLength(int32_t plaintext_len) const { - return aes_decryptor_->CiphertextLength(plaintext_len); + std::cout << "[DEBUG] ExternalDecryptorImpl::CiphertextLength called with plaintext_len: " << plaintext_len << std::endl; + //TODO + // This is not production code. We know that the one DPBA Agent we have uses XOR encryption. + // Therefore it's safe to assume that the ciphertext length is the same as the plaintext length. + // This is not true for all DPBA Agents. + if (plaintext_len < 0) { + std::cout << "[ERROR] Negative plaintext length: " << plaintext_len << std::endl; + std::stringstream ss; + ss << "Negative plaintext length " << plaintext_len; + throw ParquetException(ss.str()); + } + int32_t result = plaintext_len; + std::cout << "[DEBUG] CiphertextLength returning: " << result << std::endl; + return result; } -void ExternalDecryptorImpl::ConstructExternalCall() { - std::cout << "Here I would call the external decryption service. Hold for params." << std::endl; +void ExternalDecryptorImpl::ConstructExternalCall(span ciphertext) { + std::cout << "\n\n!*!*!*!*!*!* START ExternalDecryptorImpl:ConstructExternalCall!*!*!*!*!*!*" + << std::endl; + + std::cout << "Calling ExternalDecryptorService." << std::endl; + + std::cout << "Payload:" << std::endl; + + std::cout << "!*!*!*!*!*!* END ExternalDecryptorImpl:ConstructExternalCall!*!*!*!*!*!*\n\n" + << std::endl; } #undef ENCRYPT_INIT diff --git a/cpp/src/parquet/encryption/encryption_internal.h b/cpp/src/parquet/encryption/encryption_internal.h index d970d297fc14..3bdedf16cdff 100644 --- a/cpp/src/parquet/encryption/encryption_internal.h +++ b/cpp/src/parquet/encryption/encryption_internal.h @@ -26,6 +26,7 @@ #include "parquet/encryption/openssl_internal.h" #include "parquet/properties.h" #include "parquet/types.h" +#include "parquet/encryption/external/dbpa_interface.h" using parquet::ParquetCipher; using ::arrow::util::span; @@ -168,12 +169,7 @@ class PARQUET_EXPORT AesEncryptorImpl : public AesCryptoContext, public Encrypto class PARQUET_EXPORT ExternalEncryptorImpl : public EncryptorInterface { public: - explicit ExternalEncryptorImpl(ParquetCipher::type alg_id, int32_t key_len, - std::string column_name, Type::type data_type, - Compression::type compression_type, Encoding::type encoding_, - std::string ext_column_key, std::string user_id, - std::string app_context, - bool metadata, bool write_length); + explicit ExternalEncryptorImpl(std::unique_ptr agent_instance); static std::unique_ptr Make( ParquetCipher::type alg_id, int32_t key_len, std::string column_name, Type::type data_type, @@ -192,17 +188,9 @@ class PARQUET_EXPORT ExternalEncryptorImpl : public EncryptorInterface { private: void ConstructExternalCall(span plaintext); - std::string column_name_; - Type::type data_type_; - Compression::type compression_type_; - Encoding::type encoding_; - std::string ext_column_key_; - std::string user_id_; - std::string app_context_; - std::unique_ptr aes_encryptor_; + std::unique_ptr agent_instance_; }; - class PARQUET_EXPORT DecryptorInterface { public: virtual ~DecryptorInterface() = default; @@ -273,11 +261,15 @@ class PARQUET_EXPORT AesDecryptorImpl : public AesCryptoContext, public Decrypto class PARQUET_EXPORT ExternalDecryptorImpl : public DecryptorInterface { public: - explicit ExternalDecryptorImpl(ParquetCipher::type alg_id, int32_t key_len, bool metadata, - bool contains_length = true); + explicit ExternalDecryptorImpl(std::unique_ptr agent_instance); - static std::unique_ptr Make(ParquetCipher::type alg_id, int32_t key_len, - bool metadata); + static std::unique_ptr Make( + ParquetCipher::type alg_id, int32_t key_len, std::string column_name, Type::type data_type, + Compression::type compression_type, Encoding::type encoding, std::string ext_column_key, + std::string user_id, std::string app_context, bool metadata, bool contains_length = true); + + // Simple version for cases where we don't have all metadata (e.g., footer decryption) + static std::unique_ptr Make(ParquetCipher::type alg_id, int32_t key_len, bool metadata); int32_t Decrypt(span ciphertext, span key, span aad, span plaintext) override; @@ -287,8 +279,9 @@ class PARQUET_EXPORT ExternalDecryptorImpl : public DecryptorInterface { [[nodiscard]] int32_t CiphertextLength(int32_t plaintext_len) const override; private: - void ConstructExternalCall(); - std::unique_ptr aes_decryptor_; + void ConstructExternalCall(span ciphertext); + + std::unique_ptr agent_instance_; }; std::string CreateModuleAad(const std::string& file_aad, int8_t module_type, diff --git a/cpp/src/parquet/encryption/external/dbpa_utils.cc b/cpp/src/parquet/encryption/external/dbpa_utils.cc new file mode 100644 index 000000000000..6a5e003d8623 --- /dev/null +++ b/cpp/src/parquet/encryption/external/dbpa_utils.cc @@ -0,0 +1,115 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "parquet/encryption/external/dbpa_utils.h" + +#include + +namespace parquet::encryption::external { + +dbps::external::Type::type dbpa_utils::ParquetTypeToExternal(parquet::Type::type parquet_type) { + // The enums have identical values, so we can do a direct cast + switch (parquet_type) { + case parquet::Type::BOOLEAN: + return dbps::external::Type::BOOLEAN; + case parquet::Type::INT32: + return dbps::external::Type::INT32; + case parquet::Type::INT64: + return dbps::external::Type::INT64; + case parquet::Type::INT96: + return dbps::external::Type::INT96; + case parquet::Type::FLOAT: + return dbps::external::Type::FLOAT; + case parquet::Type::DOUBLE: + return dbps::external::Type::DOUBLE; + case parquet::Type::BYTE_ARRAY: + return dbps::external::Type::BYTE_ARRAY; + case parquet::Type::FIXED_LEN_BYTE_ARRAY: + return dbps::external::Type::FIXED_LEN_BYTE_ARRAY; + case parquet::Type::UNDEFINED: + default: + throw std::invalid_argument("Invalid parquet::Type value"); + } +} + +parquet::Type::type dbpa_utils::ExternalTypeToParquet(dbps::external::Type::type external_type) { + // The enums have identical values, so we can do a direct cast + switch (external_type) { + case dbps::external::Type::BOOLEAN: + return parquet::Type::BOOLEAN; + case dbps::external::Type::INT32: + return parquet::Type::INT32; + case dbps::external::Type::INT64: + return parquet::Type::INT64; + case dbps::external::Type::INT96: + return parquet::Type::INT96; + case dbps::external::Type::FLOAT: + return parquet::Type::FLOAT; + case dbps::external::Type::DOUBLE: + return parquet::Type::DOUBLE; + case dbps::external::Type::BYTE_ARRAY: + return parquet::Type::BYTE_ARRAY; + case dbps::external::Type::FIXED_LEN_BYTE_ARRAY: + return parquet::Type::FIXED_LEN_BYTE_ARRAY; + default: + throw std::invalid_argument("Invalid dbps::external::Type value"); + } +} + +dbps::external::CompressionCodec::type dbpa_utils::ArrowCompressionToExternal(::arrow::Compression::type arrow_compression) { + switch (arrow_compression) { + case ::arrow::Compression::UNCOMPRESSED: + return dbps::external::CompressionCodec::UNCOMPRESSED; + case ::arrow::Compression::SNAPPY: + return dbps::external::CompressionCodec::SNAPPY; + case ::arrow::Compression::GZIP: + return dbps::external::CompressionCodec::GZIP; + case ::arrow::Compression::LZO: + return dbps::external::CompressionCodec::LZO; + case ::arrow::Compression::BROTLI: + return dbps::external::CompressionCodec::BROTLI; + case ::arrow::Compression::LZ4: + return dbps::external::CompressionCodec::LZ4; + case ::arrow::Compression::ZSTD: + return dbps::external::CompressionCodec::ZSTD; + default: + throw std::invalid_argument("Invalid arrow::Compression value"); + } +} + +::arrow::Compression::type dbpa_utils::ExternalCompressionToArrow(dbps::external::CompressionCodec::type external_compression) { + switch (external_compression) { + case dbps::external::CompressionCodec::UNCOMPRESSED: + return ::arrow::Compression::UNCOMPRESSED; + case dbps::external::CompressionCodec::SNAPPY: + return ::arrow::Compression::SNAPPY; + case dbps::external::CompressionCodec::GZIP: + return ::arrow::Compression::GZIP; + case dbps::external::CompressionCodec::LZO: + return ::arrow::Compression::LZO; + case dbps::external::CompressionCodec::BROTLI: + return ::arrow::Compression::BROTLI; + case dbps::external::CompressionCodec::LZ4: + return ::arrow::Compression::LZ4; + case dbps::external::CompressionCodec::ZSTD: + return ::arrow::Compression::ZSTD; + default: + throw std::invalid_argument("Invalid dbps::external::CompressionCodec value"); + } +} + +} // namespace parquet::encryption::external diff --git a/cpp/src/parquet/encryption/external/dbpa_utils.h b/cpp/src/parquet/encryption/external/dbpa_utils.h new file mode 100644 index 000000000000..9b2f1baa27df --- /dev/null +++ b/cpp/src/parquet/encryption/external/dbpa_utils.h @@ -0,0 +1,55 @@ +#pragma once + +#include + +#include "parquet/encryption/external/dbpa_interface.h" +#include "parquet/types.h" +#include "arrow/type_fwd.h" // For arrow::Compression + +namespace parquet::encryption::external { + +/** + * Utility class for translating between Parquet/Arrow enums and dbps::external enums. + * + * This class provides methods to convert between: + * - parquet::Type and dbps::external::Type + * - arrow::Compression and dbps::external::CompressionCodec + */ +class dbpa_utils { +public: + /** + * Convert parquet::Type to dbps::external::Type + * + * @param parquet_type The parquet type to convert + * @return The corresponding dbps::external::Type + */ + static dbps::external::Type::type ParquetTypeToExternal(parquet::Type::type parquet_type); + + /** + * Convert dbps::external::Type to parquet::Type + * + * @param external_type The dbps::external type to convert + * @return The corresponding parquet::Type + */ + static parquet::Type::type ExternalTypeToParquet(dbps::external::Type::type external_type); + + /** + * Convert arrow::Compression to dbps::external::CompressionCodec + * + * @param arrow_compression The Arrow compression type to convert + * @return The corresponding dbps::external::CompressionCodec + * @throws std::invalid_argument if the Arrow compression type cannot be mapped + */ + static dbps::external::CompressionCodec::type ArrowCompressionToExternal(::arrow::Compression::type arrow_compression); + + /** + * Convert dbps::external::CompressionCodec to arrow::Compression + * + * @param external_compression The dbps::external compression type to convert + * @return The corresponding arrow::Compression + * @throws std::invalid_argument if the external compression type cannot be mapped + */ + static ::arrow::Compression::type ExternalCompressionToArrow(dbps::external::CompressionCodec::type external_compression); +}; + +} // namespace parquet::encryption::external diff --git a/cpp/src/parquet/encryption/external/loadable_encryptor_utils.cc b/cpp/src/parquet/encryption/external/loadable_encryptor_utils.cc index 54477825d03d..3684b133f470 100644 --- a/cpp/src/parquet/encryption/external/loadable_encryptor_utils.cc +++ b/cpp/src/parquet/encryption/external/loadable_encryptor_utils.cc @@ -20,6 +20,7 @@ namespace parquet::encryption::external { // This needs to match the return type of the create_new_instance function in the shared library. typedef DataBatchProtectionAgentInterface* (*create_encryptor_t)(); +//TODO: this should be private std::unique_ptr CreateInstance(void* library_handle) { auto symbol_result = arrow::internal::GetSymbol(library_handle, "create_new_instance"); if (!symbol_result.ok()) { @@ -47,7 +48,6 @@ std::unique_ptr CreateInstance(void* library_ return instance_ptr; } // CreateInstance() - std::unique_ptr LoadableEncryptorUtils::LoadFromLibrary(const std::string& library_path) { //TODO: remove this. std::cout << "Inside LoadableEncryptorUtils::LoadFromLibrary" << std::endl;