diff --git a/ci/docker/ubuntu-22.04-cpp.dockerfile b/ci/docker/ubuntu-22.04-cpp.dockerfile index 88a27efe335d..6cc8b4e11ff6 100644 --- a/ci/docker/ubuntu-22.04-cpp.dockerfile +++ b/ci/docker/ubuntu-22.04-cpp.dockerfile @@ -72,6 +72,7 @@ RUN apt-get update -y -q && \ gdb \ git \ libbenchmark-dev \ + libboost-date-time-dev \ libboost-filesystem-dev \ libboost-system-dev \ libbrotli-dev \ diff --git a/ci/docker/ubuntu-24.04-cpp.dockerfile b/ci/docker/ubuntu-24.04-cpp.dockerfile index 0347d452d7bf..126e1d852479 100644 --- a/ci/docker/ubuntu-24.04-cpp.dockerfile +++ b/ci/docker/ubuntu-24.04-cpp.dockerfile @@ -73,6 +73,7 @@ RUN apt-get update -y -q && \ gdb \ git \ libbenchmark-dev \ + libboost-date-time-dev \ libboost-filesystem-dev \ libboost-system-dev \ libbrotli-dev \ diff --git a/cpp/src/arrow/dataset/file_parquet_encryption_test.cc b/cpp/src/arrow/dataset/file_parquet_encryption_test.cc index 91d813530d43..7c95d459e9dc 100644 --- a/cpp/src/arrow/dataset/file_parquet_encryption_test.cc +++ b/cpp/src/arrow/dataset/file_parquet_encryption_test.cc @@ -38,7 +38,7 @@ #include "arrow/util/thread_pool.h" #include "parquet/arrow/reader.h" #include "parquet/encryption/crypto_factory.h" -#include "parquet/encryption/encryption_internal.h" +#include "parquet/encryption/encryption_utils.h" #include "parquet/encryption/kms_client.h" #include "parquet/encryption/test_in_memory_kms.h" diff --git a/cpp/src/arrow/util/io_util.cc b/cpp/src/arrow/util/io_util.cc index 50f3bd9a15e0..ed1772b5583d 100644 --- a/cpp/src/arrow/util/io_util.cc +++ b/cpp/src/arrow/util/io_util.cc @@ -2296,4 +2296,24 @@ Result GetSymbol(void* handle, const char* name) { #endif } +Status CloseDynamicLibrary(void* handle) { + if (handle == nullptr) { + return Status::Invalid("Attempting to close null library handle"); + } +#ifdef _WIN32 + if (FreeLibrary(reinterpret_cast(handle))) { + return Status::OK(); + } + // win32 api doc: "If the function fails, the return value is zero." + return IOErrorFromWinError(GetLastError(), "FreeLibrary() failed"); +#else + if (dlclose(handle) == 0) { + return Status::OK(); + } + // dlclose(3) man page: "On success, dlclose() returns 0; on error, it returns a nonzero value." + auto* error = dlerror(); + return Status::IOError("dlclose() failed: ", error ? error : "unknown error"); +#endif +} + } // namespace arrow::internal diff --git a/cpp/src/arrow/util/io_util.h b/cpp/src/arrow/util/io_util.h index e9f218b5205b..b0309bec50db 100644 --- a/cpp/src/arrow/util/io_util.h +++ b/cpp/src/arrow/util/io_util.h @@ -449,6 +449,13 @@ ARROW_EXPORT Result LoadDynamicLibrary(const char* path); /// returned; instead an error will be raised. ARROW_EXPORT Result GetSymbol(void* handle, const char* name); +/// \brief Close a dynamic library +/// +/// This wraps dlclose() except on Windows, where FreeLibrary() is called. +/// +/// \return Status::OK() if the library was closed successfully, otherwise an error is returned. +ARROW_EXPORT Status CloseDynamicLibrary(void* handle); + template Result GetSymbolAs(void* handle, const char* name) { ARROW_ASSIGN_OR_RAISE(void* sym, GetSymbol(handle, name)); diff --git a/cpp/src/generated/parquet_types.cpp b/cpp/src/generated/parquet_types.cpp index 0ee973f2a2d6..d13dd2a98be3 100644 --- a/cpp/src/generated/parquet_types.cpp +++ b/cpp/src/generated/parquet_types.cpp @@ -1,5 +1,5 @@ /** - * Autogenerated by Thrift Compiler (0.21.0) + * Autogenerated by Thrift Compiler (0.22.0) * * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated @@ -4204,6 +4204,320 @@ void ColumnMetaData::printTo(std::ostream& out) const { } +AesGcmV1::~AesGcmV1() noexcept { +} + +AesGcmV1::AesGcmV1() noexcept + : aad_prefix(), + aad_file_unique(), + supply_aad_prefix(0) { +} + +void AesGcmV1::__set_aad_prefix(const std::string& val) { + this->aad_prefix = val; +__isset.aad_prefix = true; +} + +void AesGcmV1::__set_aad_file_unique(const std::string& val) { + this->aad_file_unique = val; +__isset.aad_file_unique = true; +} + +void AesGcmV1::__set_supply_aad_prefix(const bool val) { + this->supply_aad_prefix = val; +__isset.supply_aad_prefix = true; +} +std::ostream& operator<<(std::ostream& out, const AesGcmV1& obj) +{ + obj.printTo(out); + return out; +} + + +void swap(AesGcmV1 &a, AesGcmV1 &b) { + using ::std::swap; + swap(a.aad_prefix, b.aad_prefix); + swap(a.aad_file_unique, b.aad_file_unique); + swap(a.supply_aad_prefix, b.supply_aad_prefix); + swap(a.__isset, b.__isset); +} + +bool AesGcmV1::operator==(const AesGcmV1 & rhs) const +{ + if (__isset.aad_prefix != rhs.__isset.aad_prefix) + return false; + else if (__isset.aad_prefix && !(aad_prefix == rhs.aad_prefix)) + return false; + if (__isset.aad_file_unique != rhs.__isset.aad_file_unique) + return false; + else if (__isset.aad_file_unique && !(aad_file_unique == rhs.aad_file_unique)) + return false; + if (__isset.supply_aad_prefix != rhs.__isset.supply_aad_prefix) + return false; + else if (__isset.supply_aad_prefix && !(supply_aad_prefix == rhs.supply_aad_prefix)) + return false; + return true; +} + +AesGcmV1::AesGcmV1(const AesGcmV1& other229) { + aad_prefix = other229.aad_prefix; + aad_file_unique = other229.aad_file_unique; + supply_aad_prefix = other229.supply_aad_prefix; + __isset = other229.__isset; +} +AesGcmV1::AesGcmV1(AesGcmV1&& other230) noexcept { + aad_prefix = std::move(other230.aad_prefix); + aad_file_unique = std::move(other230.aad_file_unique); + supply_aad_prefix = other230.supply_aad_prefix; + __isset = other230.__isset; +} +AesGcmV1& AesGcmV1::operator=(const AesGcmV1& other231) { + aad_prefix = other231.aad_prefix; + aad_file_unique = other231.aad_file_unique; + supply_aad_prefix = other231.supply_aad_prefix; + __isset = other231.__isset; + return *this; +} +AesGcmV1& AesGcmV1::operator=(AesGcmV1&& other232) noexcept { + aad_prefix = std::move(other232.aad_prefix); + aad_file_unique = std::move(other232.aad_file_unique); + supply_aad_prefix = other232.supply_aad_prefix; + __isset = other232.__isset; + return *this; +} +void AesGcmV1::printTo(std::ostream& out) const { + using ::apache::thrift::to_string; + out << "AesGcmV1("; + out << "aad_prefix="; (__isset.aad_prefix ? (out << to_string(aad_prefix)) : (out << "")); + out << ", " << "aad_file_unique="; (__isset.aad_file_unique ? (out << to_string(aad_file_unique)) : (out << "")); + out << ", " << "supply_aad_prefix="; (__isset.supply_aad_prefix ? (out << to_string(supply_aad_prefix)) : (out << "")); + out << ")"; +} + + +AesGcmCtrV1::~AesGcmCtrV1() noexcept { +} + +AesGcmCtrV1::AesGcmCtrV1() noexcept + : aad_prefix(), + aad_file_unique(), + supply_aad_prefix(0) { +} + +void AesGcmCtrV1::__set_aad_prefix(const std::string& val) { + this->aad_prefix = val; +__isset.aad_prefix = true; +} + +void AesGcmCtrV1::__set_aad_file_unique(const std::string& val) { + this->aad_file_unique = val; +__isset.aad_file_unique = true; +} + +void AesGcmCtrV1::__set_supply_aad_prefix(const bool val) { + this->supply_aad_prefix = val; +__isset.supply_aad_prefix = true; +} +std::ostream& operator<<(std::ostream& out, const AesGcmCtrV1& obj) +{ + obj.printTo(out); + return out; +} + + +void swap(AesGcmCtrV1 &a, AesGcmCtrV1 &b) { + using ::std::swap; + swap(a.aad_prefix, b.aad_prefix); + swap(a.aad_file_unique, b.aad_file_unique); + swap(a.supply_aad_prefix, b.supply_aad_prefix); + swap(a.__isset, b.__isset); +} + +bool AesGcmCtrV1::operator==(const AesGcmCtrV1 & rhs) const +{ + if (__isset.aad_prefix != rhs.__isset.aad_prefix) + return false; + else if (__isset.aad_prefix && !(aad_prefix == rhs.aad_prefix)) + return false; + if (__isset.aad_file_unique != rhs.__isset.aad_file_unique) + return false; + else if (__isset.aad_file_unique && !(aad_file_unique == rhs.aad_file_unique)) + return false; + if (__isset.supply_aad_prefix != rhs.__isset.supply_aad_prefix) + return false; + else if (__isset.supply_aad_prefix && !(supply_aad_prefix == rhs.supply_aad_prefix)) + return false; + return true; +} + +AesGcmCtrV1::AesGcmCtrV1(const AesGcmCtrV1& other233) { + aad_prefix = other233.aad_prefix; + aad_file_unique = other233.aad_file_unique; + supply_aad_prefix = other233.supply_aad_prefix; + __isset = other233.__isset; +} +AesGcmCtrV1::AesGcmCtrV1(AesGcmCtrV1&& other234) noexcept { + aad_prefix = std::move(other234.aad_prefix); + aad_file_unique = std::move(other234.aad_file_unique); + supply_aad_prefix = other234.supply_aad_prefix; + __isset = other234.__isset; +} +AesGcmCtrV1& AesGcmCtrV1::operator=(const AesGcmCtrV1& other235) { + aad_prefix = other235.aad_prefix; + aad_file_unique = other235.aad_file_unique; + supply_aad_prefix = other235.supply_aad_prefix; + __isset = other235.__isset; + return *this; +} +AesGcmCtrV1& AesGcmCtrV1::operator=(AesGcmCtrV1&& other236) noexcept { + aad_prefix = std::move(other236.aad_prefix); + aad_file_unique = std::move(other236.aad_file_unique); + supply_aad_prefix = other236.supply_aad_prefix; + __isset = other236.__isset; + return *this; +} +void AesGcmCtrV1::printTo(std::ostream& out) const { + using ::apache::thrift::to_string; + out << "AesGcmCtrV1("; + out << "aad_prefix="; (__isset.aad_prefix ? (out << to_string(aad_prefix)) : (out << "")); + out << ", " << "aad_file_unique="; (__isset.aad_file_unique ? (out << to_string(aad_file_unique)) : (out << "")); + out << ", " << "supply_aad_prefix="; (__isset.supply_aad_prefix ? (out << to_string(supply_aad_prefix)) : (out << "")); + out << ")"; +} + + +ExternalDBPAV1::~ExternalDBPAV1() noexcept { +} + +ExternalDBPAV1::ExternalDBPAV1() noexcept { +} +std::ostream& operator<<(std::ostream& out, const ExternalDBPAV1& obj) +{ + obj.printTo(out); + return out; +} + + +void swap(ExternalDBPAV1 &a, ExternalDBPAV1 &b) { + using ::std::swap; + (void) a; + (void) b; +} + +bool ExternalDBPAV1::operator==(const ExternalDBPAV1 & /* rhs */) const +{ + return true; +} + +ExternalDBPAV1::ExternalDBPAV1(const ExternalDBPAV1& other237) noexcept { + (void) other237; +} +ExternalDBPAV1::ExternalDBPAV1(ExternalDBPAV1&& other238) noexcept { + (void) other238; +} +ExternalDBPAV1& ExternalDBPAV1::operator=(const ExternalDBPAV1& other239) noexcept { + (void) other239; + return *this; +} +ExternalDBPAV1& ExternalDBPAV1::operator=(ExternalDBPAV1&& other240) noexcept { + (void) other240; + return *this; +} +void ExternalDBPAV1::printTo(std::ostream& out) const { + using ::apache::thrift::to_string; + out << "ExternalDBPAV1("; + out << ")"; +} + + +EncryptionAlgorithm::~EncryptionAlgorithm() noexcept { +} + +EncryptionAlgorithm::EncryptionAlgorithm() noexcept { +} + +void EncryptionAlgorithm::__set_AES_GCM_V1(const AesGcmV1& val) { + this->AES_GCM_V1 = val; +__isset.AES_GCM_V1 = true; +} + +void EncryptionAlgorithm::__set_AES_GCM_CTR_V1(const AesGcmCtrV1& val) { + this->AES_GCM_CTR_V1 = val; +__isset.AES_GCM_CTR_V1 = true; +} + +void EncryptionAlgorithm::__set_EXTERNAL_DBPA_V1(const ExternalDBPAV1& val) { + this->EXTERNAL_DBPA_V1 = val; +__isset.EXTERNAL_DBPA_V1 = true; +} +std::ostream& operator<<(std::ostream& out, const EncryptionAlgorithm& obj) +{ + obj.printTo(out); + return out; +} + + +void swap(EncryptionAlgorithm &a, EncryptionAlgorithm &b) { + using ::std::swap; + swap(a.AES_GCM_V1, b.AES_GCM_V1); + swap(a.AES_GCM_CTR_V1, b.AES_GCM_CTR_V1); + swap(a.EXTERNAL_DBPA_V1, b.EXTERNAL_DBPA_V1); + swap(a.__isset, b.__isset); +} + +bool EncryptionAlgorithm::operator==(const EncryptionAlgorithm & rhs) const +{ + if (__isset.AES_GCM_V1 != rhs.__isset.AES_GCM_V1) + return false; + else if (__isset.AES_GCM_V1 && !(AES_GCM_V1 == rhs.AES_GCM_V1)) + return false; + if (__isset.AES_GCM_CTR_V1 != rhs.__isset.AES_GCM_CTR_V1) + return false; + else if (__isset.AES_GCM_CTR_V1 && !(AES_GCM_CTR_V1 == rhs.AES_GCM_CTR_V1)) + return false; + if (__isset.EXTERNAL_DBPA_V1 != rhs.__isset.EXTERNAL_DBPA_V1) + return false; + else if (__isset.EXTERNAL_DBPA_V1 && !(EXTERNAL_DBPA_V1 == rhs.EXTERNAL_DBPA_V1)) + return false; + return true; +} + +EncryptionAlgorithm::EncryptionAlgorithm(const EncryptionAlgorithm& other241) { + AES_GCM_V1 = other241.AES_GCM_V1; + AES_GCM_CTR_V1 = other241.AES_GCM_CTR_V1; + EXTERNAL_DBPA_V1 = other241.EXTERNAL_DBPA_V1; + __isset = other241.__isset; +} +EncryptionAlgorithm::EncryptionAlgorithm(EncryptionAlgorithm&& other242) noexcept { + AES_GCM_V1 = std::move(other242.AES_GCM_V1); + AES_GCM_CTR_V1 = std::move(other242.AES_GCM_CTR_V1); + EXTERNAL_DBPA_V1 = std::move(other242.EXTERNAL_DBPA_V1); + __isset = other242.__isset; +} +EncryptionAlgorithm& EncryptionAlgorithm::operator=(const EncryptionAlgorithm& other243) { + AES_GCM_V1 = other243.AES_GCM_V1; + AES_GCM_CTR_V1 = other243.AES_GCM_CTR_V1; + EXTERNAL_DBPA_V1 = other243.EXTERNAL_DBPA_V1; + __isset = other243.__isset; + return *this; +} +EncryptionAlgorithm& EncryptionAlgorithm::operator=(EncryptionAlgorithm&& other244) noexcept { + AES_GCM_V1 = std::move(other244.AES_GCM_V1); + AES_GCM_CTR_V1 = std::move(other244.AES_GCM_CTR_V1); + EXTERNAL_DBPA_V1 = std::move(other244.EXTERNAL_DBPA_V1); + __isset = other244.__isset; + return *this; +} +void EncryptionAlgorithm::printTo(std::ostream& out) const { + using ::apache::thrift::to_string; + out << "EncryptionAlgorithm("; + out << "AES_GCM_V1="; (__isset.AES_GCM_V1 ? (out << to_string(AES_GCM_V1)) : (out << "")); + out << ", " << "AES_GCM_CTR_V1="; (__isset.AES_GCM_CTR_V1 ? (out << to_string(AES_GCM_CTR_V1)) : (out << "")); + out << ", " << "EXTERNAL_DBPA_V1="; (__isset.EXTERNAL_DBPA_V1 ? (out << to_string(EXTERNAL_DBPA_V1)) : (out << "")); + out << ")"; +} + + EncryptionWithFooterKey::~EncryptionWithFooterKey() noexcept { } @@ -4227,18 +4541,18 @@ bool EncryptionWithFooterKey::operator==(const EncryptionWithFooterKey & /* rhs return true; } -EncryptionWithFooterKey::EncryptionWithFooterKey(const EncryptionWithFooterKey& other229) noexcept { - (void) other229; +EncryptionWithFooterKey::EncryptionWithFooterKey(const EncryptionWithFooterKey& other245) noexcept { + (void) other245; } -EncryptionWithFooterKey::EncryptionWithFooterKey(EncryptionWithFooterKey&& other230) noexcept { - (void) other230; +EncryptionWithFooterKey::EncryptionWithFooterKey(EncryptionWithFooterKey&& other246) noexcept { + (void) other246; } -EncryptionWithFooterKey& EncryptionWithFooterKey::operator=(const EncryptionWithFooterKey& other231) noexcept { - (void) other231; +EncryptionWithFooterKey& EncryptionWithFooterKey::operator=(const EncryptionWithFooterKey& other247) noexcept { + (void) other247; return *this; } -EncryptionWithFooterKey& EncryptionWithFooterKey::operator=(EncryptionWithFooterKey&& other232) noexcept { - (void) other232; +EncryptionWithFooterKey& EncryptionWithFooterKey::operator=(EncryptionWithFooterKey&& other248) noexcept { + (void) other248; return *this; } void EncryptionWithFooterKey::printTo(std::ostream& out) const { @@ -4263,6 +4577,11 @@ void EncryptionWithColumnKey::__set_key_metadata(const std::string& val) { this->key_metadata = val; __isset.key_metadata = true; } + +void EncryptionWithColumnKey::__set_encryption_algorithm(const EncryptionAlgorithm& val) { + this->encryption_algorithm = val; +__isset.encryption_algorithm = true; +} std::ostream& operator<<(std::ostream& out, const EncryptionWithColumnKey& obj) { obj.printTo(out); @@ -4274,6 +4593,7 @@ void swap(EncryptionWithColumnKey &a, EncryptionWithColumnKey &b) { using ::std::swap; swap(a.path_in_schema, b.path_in_schema); swap(a.key_metadata, b.key_metadata); + swap(a.encryption_algorithm, b.encryption_algorithm); swap(a.__isset, b.__isset); } @@ -4285,29 +4605,37 @@ bool EncryptionWithColumnKey::operator==(const EncryptionWithColumnKey & rhs) co return false; else if (__isset.key_metadata && !(key_metadata == rhs.key_metadata)) return false; + if (__isset.encryption_algorithm != rhs.__isset.encryption_algorithm) + return false; + else if (__isset.encryption_algorithm && !(encryption_algorithm == rhs.encryption_algorithm)) + return false; return true; } -EncryptionWithColumnKey::EncryptionWithColumnKey(const EncryptionWithColumnKey& other239) { - path_in_schema = other239.path_in_schema; - key_metadata = other239.key_metadata; - __isset = other239.__isset; +EncryptionWithColumnKey::EncryptionWithColumnKey(const EncryptionWithColumnKey& other255) { + path_in_schema = other255.path_in_schema; + key_metadata = other255.key_metadata; + encryption_algorithm = other255.encryption_algorithm; + __isset = other255.__isset; } -EncryptionWithColumnKey::EncryptionWithColumnKey(EncryptionWithColumnKey&& other240) noexcept { - path_in_schema = std::move(other240.path_in_schema); - key_metadata = std::move(other240.key_metadata); - __isset = other240.__isset; +EncryptionWithColumnKey::EncryptionWithColumnKey(EncryptionWithColumnKey&& other256) noexcept { + path_in_schema = std::move(other256.path_in_schema); + key_metadata = std::move(other256.key_metadata); + encryption_algorithm = std::move(other256.encryption_algorithm); + __isset = other256.__isset; } -EncryptionWithColumnKey& EncryptionWithColumnKey::operator=(const EncryptionWithColumnKey& other241) { - path_in_schema = other241.path_in_schema; - key_metadata = other241.key_metadata; - __isset = other241.__isset; +EncryptionWithColumnKey& EncryptionWithColumnKey::operator=(const EncryptionWithColumnKey& other257) { + path_in_schema = other257.path_in_schema; + key_metadata = other257.key_metadata; + encryption_algorithm = other257.encryption_algorithm; + __isset = other257.__isset; return *this; } -EncryptionWithColumnKey& EncryptionWithColumnKey::operator=(EncryptionWithColumnKey&& other242) noexcept { - path_in_schema = std::move(other242.path_in_schema); - key_metadata = std::move(other242.key_metadata); - __isset = other242.__isset; +EncryptionWithColumnKey& EncryptionWithColumnKey::operator=(EncryptionWithColumnKey&& other258) noexcept { + path_in_schema = std::move(other258.path_in_schema); + key_metadata = std::move(other258.key_metadata); + encryption_algorithm = std::move(other258.encryption_algorithm); + __isset = other258.__isset; return *this; } void EncryptionWithColumnKey::printTo(std::ostream& out) const { @@ -4315,6 +4643,7 @@ void EncryptionWithColumnKey::printTo(std::ostream& out) const { out << "EncryptionWithColumnKey("; out << "path_in_schema=" << to_string(path_in_schema); out << ", " << "key_metadata="; (__isset.key_metadata ? (out << to_string(key_metadata)) : (out << "")); + out << ", " << "encryption_algorithm="; (__isset.encryption_algorithm ? (out << to_string(encryption_algorithm)) : (out << "")); out << ")"; } @@ -4361,26 +4690,26 @@ bool ColumnCryptoMetaData::operator==(const ColumnCryptoMetaData & rhs) const return true; } -ColumnCryptoMetaData::ColumnCryptoMetaData(const ColumnCryptoMetaData& other243) { - ENCRYPTION_WITH_FOOTER_KEY = other243.ENCRYPTION_WITH_FOOTER_KEY; - ENCRYPTION_WITH_COLUMN_KEY = other243.ENCRYPTION_WITH_COLUMN_KEY; - __isset = other243.__isset; +ColumnCryptoMetaData::ColumnCryptoMetaData(const ColumnCryptoMetaData& other259) { + ENCRYPTION_WITH_FOOTER_KEY = other259.ENCRYPTION_WITH_FOOTER_KEY; + ENCRYPTION_WITH_COLUMN_KEY = other259.ENCRYPTION_WITH_COLUMN_KEY; + __isset = other259.__isset; } -ColumnCryptoMetaData::ColumnCryptoMetaData(ColumnCryptoMetaData&& other244) noexcept { - ENCRYPTION_WITH_FOOTER_KEY = std::move(other244.ENCRYPTION_WITH_FOOTER_KEY); - ENCRYPTION_WITH_COLUMN_KEY = std::move(other244.ENCRYPTION_WITH_COLUMN_KEY); - __isset = other244.__isset; +ColumnCryptoMetaData::ColumnCryptoMetaData(ColumnCryptoMetaData&& other260) noexcept { + ENCRYPTION_WITH_FOOTER_KEY = std::move(other260.ENCRYPTION_WITH_FOOTER_KEY); + ENCRYPTION_WITH_COLUMN_KEY = std::move(other260.ENCRYPTION_WITH_COLUMN_KEY); + __isset = other260.__isset; } -ColumnCryptoMetaData& ColumnCryptoMetaData::operator=(const ColumnCryptoMetaData& other245) { - ENCRYPTION_WITH_FOOTER_KEY = other245.ENCRYPTION_WITH_FOOTER_KEY; - ENCRYPTION_WITH_COLUMN_KEY = other245.ENCRYPTION_WITH_COLUMN_KEY; - __isset = other245.__isset; +ColumnCryptoMetaData& ColumnCryptoMetaData::operator=(const ColumnCryptoMetaData& other261) { + ENCRYPTION_WITH_FOOTER_KEY = other261.ENCRYPTION_WITH_FOOTER_KEY; + ENCRYPTION_WITH_COLUMN_KEY = other261.ENCRYPTION_WITH_COLUMN_KEY; + __isset = other261.__isset; return *this; } -ColumnCryptoMetaData& ColumnCryptoMetaData::operator=(ColumnCryptoMetaData&& other246) noexcept { - ENCRYPTION_WITH_FOOTER_KEY = std::move(other246.ENCRYPTION_WITH_FOOTER_KEY); - ENCRYPTION_WITH_COLUMN_KEY = std::move(other246.ENCRYPTION_WITH_COLUMN_KEY); - __isset = other246.__isset; +ColumnCryptoMetaData& ColumnCryptoMetaData::operator=(ColumnCryptoMetaData&& other262) noexcept { + ENCRYPTION_WITH_FOOTER_KEY = std::move(other262.ENCRYPTION_WITH_FOOTER_KEY); + ENCRYPTION_WITH_COLUMN_KEY = std::move(other262.ENCRYPTION_WITH_COLUMN_KEY); + __isset = other262.__isset; return *this; } void ColumnCryptoMetaData::printTo(std::ostream& out) const { @@ -4508,54 +4837,54 @@ bool ColumnChunk::operator==(const ColumnChunk & rhs) const return true; } -ColumnChunk::ColumnChunk(const ColumnChunk& other247) { - file_path = other247.file_path; - file_offset = other247.file_offset; - meta_data = other247.meta_data; - offset_index_offset = other247.offset_index_offset; - offset_index_length = other247.offset_index_length; - column_index_offset = other247.column_index_offset; - column_index_length = other247.column_index_length; - crypto_metadata = other247.crypto_metadata; - encrypted_column_metadata = other247.encrypted_column_metadata; - __isset = other247.__isset; -} -ColumnChunk::ColumnChunk(ColumnChunk&& other248) noexcept { - file_path = std::move(other248.file_path); - file_offset = other248.file_offset; - meta_data = std::move(other248.meta_data); - offset_index_offset = other248.offset_index_offset; - offset_index_length = other248.offset_index_length; - column_index_offset = other248.column_index_offset; - column_index_length = other248.column_index_length; - crypto_metadata = std::move(other248.crypto_metadata); - encrypted_column_metadata = std::move(other248.encrypted_column_metadata); - __isset = other248.__isset; -} -ColumnChunk& ColumnChunk::operator=(const ColumnChunk& other249) { - file_path = other249.file_path; - file_offset = other249.file_offset; - meta_data = other249.meta_data; - offset_index_offset = other249.offset_index_offset; - offset_index_length = other249.offset_index_length; - column_index_offset = other249.column_index_offset; - column_index_length = other249.column_index_length; - crypto_metadata = other249.crypto_metadata; - encrypted_column_metadata = other249.encrypted_column_metadata; - __isset = other249.__isset; +ColumnChunk::ColumnChunk(const ColumnChunk& other263) { + file_path = other263.file_path; + file_offset = other263.file_offset; + meta_data = other263.meta_data; + offset_index_offset = other263.offset_index_offset; + offset_index_length = other263.offset_index_length; + column_index_offset = other263.column_index_offset; + column_index_length = other263.column_index_length; + crypto_metadata = other263.crypto_metadata; + encrypted_column_metadata = other263.encrypted_column_metadata; + __isset = other263.__isset; +} +ColumnChunk::ColumnChunk(ColumnChunk&& other264) noexcept { + file_path = std::move(other264.file_path); + file_offset = other264.file_offset; + meta_data = std::move(other264.meta_data); + offset_index_offset = other264.offset_index_offset; + offset_index_length = other264.offset_index_length; + column_index_offset = other264.column_index_offset; + column_index_length = other264.column_index_length; + crypto_metadata = std::move(other264.crypto_metadata); + encrypted_column_metadata = std::move(other264.encrypted_column_metadata); + __isset = other264.__isset; +} +ColumnChunk& ColumnChunk::operator=(const ColumnChunk& other265) { + file_path = other265.file_path; + file_offset = other265.file_offset; + meta_data = other265.meta_data; + offset_index_offset = other265.offset_index_offset; + offset_index_length = other265.offset_index_length; + column_index_offset = other265.column_index_offset; + column_index_length = other265.column_index_length; + crypto_metadata = other265.crypto_metadata; + encrypted_column_metadata = other265.encrypted_column_metadata; + __isset = other265.__isset; return *this; } -ColumnChunk& ColumnChunk::operator=(ColumnChunk&& other250) noexcept { - file_path = std::move(other250.file_path); - file_offset = other250.file_offset; - meta_data = std::move(other250.meta_data); - offset_index_offset = other250.offset_index_offset; - offset_index_length = other250.offset_index_length; - column_index_offset = other250.column_index_offset; - column_index_length = other250.column_index_length; - crypto_metadata = std::move(other250.crypto_metadata); - encrypted_column_metadata = std::move(other250.encrypted_column_metadata); - __isset = other250.__isset; +ColumnChunk& ColumnChunk::operator=(ColumnChunk&& other266) noexcept { + file_path = std::move(other266.file_path); + file_offset = other266.file_offset; + meta_data = std::move(other266.meta_data); + offset_index_offset = other266.offset_index_offset; + offset_index_length = other266.offset_index_length; + column_index_offset = other266.column_index_offset; + column_index_length = other266.column_index_length; + crypto_metadata = std::move(other266.crypto_metadata); + encrypted_column_metadata = std::move(other266.encrypted_column_metadata); + __isset = other266.__isset; return *this; } void ColumnChunk::printTo(std::ostream& out) const { @@ -4662,46 +4991,46 @@ bool RowGroup::operator==(const RowGroup & rhs) const return true; } -RowGroup::RowGroup(const RowGroup& other263) { - columns = other263.columns; - total_byte_size = other263.total_byte_size; - num_rows = other263.num_rows; - sorting_columns = other263.sorting_columns; - file_offset = other263.file_offset; - total_compressed_size = other263.total_compressed_size; - ordinal = other263.ordinal; - __isset = other263.__isset; -} -RowGroup::RowGroup(RowGroup&& other264) noexcept { - columns = std::move(other264.columns); - total_byte_size = other264.total_byte_size; - num_rows = other264.num_rows; - sorting_columns = std::move(other264.sorting_columns); - file_offset = other264.file_offset; - total_compressed_size = other264.total_compressed_size; - ordinal = other264.ordinal; - __isset = other264.__isset; -} -RowGroup& RowGroup::operator=(const RowGroup& other265) { - columns = other265.columns; - total_byte_size = other265.total_byte_size; - num_rows = other265.num_rows; - sorting_columns = other265.sorting_columns; - file_offset = other265.file_offset; - total_compressed_size = other265.total_compressed_size; - ordinal = other265.ordinal; - __isset = other265.__isset; +RowGroup::RowGroup(const RowGroup& other279) { + columns = other279.columns; + total_byte_size = other279.total_byte_size; + num_rows = other279.num_rows; + sorting_columns = other279.sorting_columns; + file_offset = other279.file_offset; + total_compressed_size = other279.total_compressed_size; + ordinal = other279.ordinal; + __isset = other279.__isset; +} +RowGroup::RowGroup(RowGroup&& other280) noexcept { + columns = std::move(other280.columns); + total_byte_size = other280.total_byte_size; + num_rows = other280.num_rows; + sorting_columns = std::move(other280.sorting_columns); + file_offset = other280.file_offset; + total_compressed_size = other280.total_compressed_size; + ordinal = other280.ordinal; + __isset = other280.__isset; +} +RowGroup& RowGroup::operator=(const RowGroup& other281) { + columns = other281.columns; + total_byte_size = other281.total_byte_size; + num_rows = other281.num_rows; + sorting_columns = other281.sorting_columns; + file_offset = other281.file_offset; + total_compressed_size = other281.total_compressed_size; + ordinal = other281.ordinal; + __isset = other281.__isset; return *this; } -RowGroup& RowGroup::operator=(RowGroup&& other266) noexcept { - columns = std::move(other266.columns); - total_byte_size = other266.total_byte_size; - num_rows = other266.num_rows; - sorting_columns = std::move(other266.sorting_columns); - file_offset = other266.file_offset; - total_compressed_size = other266.total_compressed_size; - ordinal = other266.ordinal; - __isset = other266.__isset; +RowGroup& RowGroup::operator=(RowGroup&& other282) noexcept { + columns = std::move(other282.columns); + total_byte_size = other282.total_byte_size; + num_rows = other282.num_rows; + sorting_columns = std::move(other282.sorting_columns); + file_offset = other282.file_offset; + total_compressed_size = other282.total_compressed_size; + ordinal = other282.ordinal; + __isset = other282.__isset; return *this; } void RowGroup::printTo(std::ostream& out) const { @@ -4741,18 +5070,18 @@ bool TypeDefinedOrder::operator==(const TypeDefinedOrder & /* rhs */) const return true; } -TypeDefinedOrder::TypeDefinedOrder(const TypeDefinedOrder& other267) noexcept { - (void) other267; +TypeDefinedOrder::TypeDefinedOrder(const TypeDefinedOrder& other283) noexcept { + (void) other283; } -TypeDefinedOrder::TypeDefinedOrder(TypeDefinedOrder&& other268) noexcept { - (void) other268; +TypeDefinedOrder::TypeDefinedOrder(TypeDefinedOrder&& other284) noexcept { + (void) other284; } -TypeDefinedOrder& TypeDefinedOrder::operator=(const TypeDefinedOrder& other269) noexcept { - (void) other269; +TypeDefinedOrder& TypeDefinedOrder::operator=(const TypeDefinedOrder& other285) noexcept { + (void) other285; return *this; } -TypeDefinedOrder& TypeDefinedOrder::operator=(TypeDefinedOrder&& other270) noexcept { - (void) other270; +TypeDefinedOrder& TypeDefinedOrder::operator=(TypeDefinedOrder&& other286) noexcept { + (void) other286; return *this; } void TypeDefinedOrder::printTo(std::ostream& out) const { @@ -4794,22 +5123,22 @@ bool ColumnOrder::operator==(const ColumnOrder & rhs) const return true; } -ColumnOrder::ColumnOrder(const ColumnOrder& other271) noexcept { - TYPE_ORDER = other271.TYPE_ORDER; - __isset = other271.__isset; +ColumnOrder::ColumnOrder(const ColumnOrder& other287) noexcept { + TYPE_ORDER = other287.TYPE_ORDER; + __isset = other287.__isset; } -ColumnOrder::ColumnOrder(ColumnOrder&& other272) noexcept { - TYPE_ORDER = std::move(other272.TYPE_ORDER); - __isset = other272.__isset; +ColumnOrder::ColumnOrder(ColumnOrder&& other288) noexcept { + TYPE_ORDER = std::move(other288.TYPE_ORDER); + __isset = other288.__isset; } -ColumnOrder& ColumnOrder::operator=(const ColumnOrder& other273) noexcept { - TYPE_ORDER = other273.TYPE_ORDER; - __isset = other273.__isset; +ColumnOrder& ColumnOrder::operator=(const ColumnOrder& other289) noexcept { + TYPE_ORDER = other289.TYPE_ORDER; + __isset = other289.__isset; return *this; } -ColumnOrder& ColumnOrder::operator=(ColumnOrder&& other274) noexcept { - TYPE_ORDER = std::move(other274.TYPE_ORDER); - __isset = other274.__isset; +ColumnOrder& ColumnOrder::operator=(ColumnOrder&& other290) noexcept { + TYPE_ORDER = std::move(other290.TYPE_ORDER); + __isset = other290.__isset; return *this; } void ColumnOrder::printTo(std::ostream& out) const { @@ -4865,26 +5194,26 @@ bool PageLocation::operator==(const PageLocation & rhs) const return true; } -PageLocation::PageLocation(const PageLocation& other275) noexcept { - offset = other275.offset; - compressed_page_size = other275.compressed_page_size; - first_row_index = other275.first_row_index; +PageLocation::PageLocation(const PageLocation& other291) noexcept { + offset = other291.offset; + compressed_page_size = other291.compressed_page_size; + first_row_index = other291.first_row_index; } -PageLocation::PageLocation(PageLocation&& other276) noexcept { - offset = other276.offset; - compressed_page_size = other276.compressed_page_size; - first_row_index = other276.first_row_index; +PageLocation::PageLocation(PageLocation&& other292) noexcept { + offset = other292.offset; + compressed_page_size = other292.compressed_page_size; + first_row_index = other292.first_row_index; } -PageLocation& PageLocation::operator=(const PageLocation& other277) noexcept { - offset = other277.offset; - compressed_page_size = other277.compressed_page_size; - first_row_index = other277.first_row_index; +PageLocation& PageLocation::operator=(const PageLocation& other293) noexcept { + offset = other293.offset; + compressed_page_size = other293.compressed_page_size; + first_row_index = other293.first_row_index; return *this; } -PageLocation& PageLocation::operator=(PageLocation&& other278) noexcept { - offset = other278.offset; - compressed_page_size = other278.compressed_page_size; - first_row_index = other278.first_row_index; +PageLocation& PageLocation::operator=(PageLocation&& other294) noexcept { + offset = other294.offset; + compressed_page_size = other294.compressed_page_size; + first_row_index = other294.first_row_index; return *this; } void PageLocation::printTo(std::ostream& out) const { @@ -4936,26 +5265,26 @@ bool OffsetIndex::operator==(const OffsetIndex & rhs) const return true; } -OffsetIndex::OffsetIndex(const OffsetIndex& other291) { - page_locations = other291.page_locations; - unencoded_byte_array_data_bytes = other291.unencoded_byte_array_data_bytes; - __isset = other291.__isset; +OffsetIndex::OffsetIndex(const OffsetIndex& other307) { + page_locations = other307.page_locations; + unencoded_byte_array_data_bytes = other307.unencoded_byte_array_data_bytes; + __isset = other307.__isset; } -OffsetIndex::OffsetIndex(OffsetIndex&& other292) noexcept { - page_locations = std::move(other292.page_locations); - unencoded_byte_array_data_bytes = std::move(other292.unencoded_byte_array_data_bytes); - __isset = other292.__isset; +OffsetIndex::OffsetIndex(OffsetIndex&& other308) noexcept { + page_locations = std::move(other308.page_locations); + unencoded_byte_array_data_bytes = std::move(other308.unencoded_byte_array_data_bytes); + __isset = other308.__isset; } -OffsetIndex& OffsetIndex::operator=(const OffsetIndex& other293) { - page_locations = other293.page_locations; - unencoded_byte_array_data_bytes = other293.unencoded_byte_array_data_bytes; - __isset = other293.__isset; +OffsetIndex& OffsetIndex::operator=(const OffsetIndex& other309) { + page_locations = other309.page_locations; + unencoded_byte_array_data_bytes = other309.unencoded_byte_array_data_bytes; + __isset = other309.__isset; return *this; } -OffsetIndex& OffsetIndex::operator=(OffsetIndex&& other294) noexcept { - page_locations = std::move(other294.page_locations); - unencoded_byte_array_data_bytes = std::move(other294.unencoded_byte_array_data_bytes); - __isset = other294.__isset; +OffsetIndex& OffsetIndex::operator=(OffsetIndex&& other310) noexcept { + page_locations = std::move(other310.page_locations); + unencoded_byte_array_data_bytes = std::move(other310.unencoded_byte_array_data_bytes); + __isset = other310.__isset; return *this; } void OffsetIndex::printTo(std::ostream& out) const { @@ -5048,46 +5377,46 @@ bool ColumnIndex::operator==(const ColumnIndex & rhs) const return true; } -ColumnIndex::ColumnIndex(const ColumnIndex& other332) { - null_pages = other332.null_pages; - min_values = other332.min_values; - max_values = other332.max_values; - boundary_order = other332.boundary_order; - null_counts = other332.null_counts; - repetition_level_histograms = other332.repetition_level_histograms; - definition_level_histograms = other332.definition_level_histograms; - __isset = other332.__isset; -} -ColumnIndex::ColumnIndex(ColumnIndex&& other333) noexcept { - null_pages = std::move(other333.null_pages); - min_values = std::move(other333.min_values); - max_values = std::move(other333.max_values); - boundary_order = other333.boundary_order; - null_counts = std::move(other333.null_counts); - repetition_level_histograms = std::move(other333.repetition_level_histograms); - definition_level_histograms = std::move(other333.definition_level_histograms); - __isset = other333.__isset; -} -ColumnIndex& ColumnIndex::operator=(const ColumnIndex& other334) { - null_pages = other334.null_pages; - min_values = other334.min_values; - max_values = other334.max_values; - boundary_order = other334.boundary_order; - null_counts = other334.null_counts; - repetition_level_histograms = other334.repetition_level_histograms; - definition_level_histograms = other334.definition_level_histograms; - __isset = other334.__isset; +ColumnIndex::ColumnIndex(const ColumnIndex& other348) { + null_pages = other348.null_pages; + min_values = other348.min_values; + max_values = other348.max_values; + boundary_order = other348.boundary_order; + null_counts = other348.null_counts; + repetition_level_histograms = other348.repetition_level_histograms; + definition_level_histograms = other348.definition_level_histograms; + __isset = other348.__isset; +} +ColumnIndex::ColumnIndex(ColumnIndex&& other349) noexcept { + null_pages = std::move(other349.null_pages); + min_values = std::move(other349.min_values); + max_values = std::move(other349.max_values); + boundary_order = other349.boundary_order; + null_counts = std::move(other349.null_counts); + repetition_level_histograms = std::move(other349.repetition_level_histograms); + definition_level_histograms = std::move(other349.definition_level_histograms); + __isset = other349.__isset; +} +ColumnIndex& ColumnIndex::operator=(const ColumnIndex& other350) { + null_pages = other350.null_pages; + min_values = other350.min_values; + max_values = other350.max_values; + boundary_order = other350.boundary_order; + null_counts = other350.null_counts; + repetition_level_histograms = other350.repetition_level_histograms; + definition_level_histograms = other350.definition_level_histograms; + __isset = other350.__isset; return *this; } -ColumnIndex& ColumnIndex::operator=(ColumnIndex&& other335) noexcept { - null_pages = std::move(other335.null_pages); - min_values = std::move(other335.min_values); - max_values = std::move(other335.max_values); - boundary_order = other335.boundary_order; - null_counts = std::move(other335.null_counts); - repetition_level_histograms = std::move(other335.repetition_level_histograms); - definition_level_histograms = std::move(other335.definition_level_histograms); - __isset = other335.__isset; +ColumnIndex& ColumnIndex::operator=(ColumnIndex&& other351) noexcept { + null_pages = std::move(other351.null_pages); + min_values = std::move(other351.min_values); + max_values = std::move(other351.max_values); + boundary_order = other351.boundary_order; + null_counts = std::move(other351.null_counts); + repetition_level_histograms = std::move(other351.repetition_level_histograms); + definition_level_histograms = std::move(other351.definition_level_histograms); + __isset = other351.__isset; return *this; } void ColumnIndex::printTo(std::ostream& out) const { @@ -5104,261 +5433,6 @@ void ColumnIndex::printTo(std::ostream& out) const { } -AesGcmV1::~AesGcmV1() noexcept { -} - -AesGcmV1::AesGcmV1() noexcept - : aad_prefix(), - aad_file_unique(), - supply_aad_prefix(0) { -} - -void AesGcmV1::__set_aad_prefix(const std::string& val) { - this->aad_prefix = val; -__isset.aad_prefix = true; -} - -void AesGcmV1::__set_aad_file_unique(const std::string& val) { - this->aad_file_unique = val; -__isset.aad_file_unique = true; -} - -void AesGcmV1::__set_supply_aad_prefix(const bool val) { - this->supply_aad_prefix = val; -__isset.supply_aad_prefix = true; -} -std::ostream& operator<<(std::ostream& out, const AesGcmV1& obj) -{ - obj.printTo(out); - return out; -} - - -void swap(AesGcmV1 &a, AesGcmV1 &b) { - using ::std::swap; - swap(a.aad_prefix, b.aad_prefix); - swap(a.aad_file_unique, b.aad_file_unique); - swap(a.supply_aad_prefix, b.supply_aad_prefix); - swap(a.__isset, b.__isset); -} - -bool AesGcmV1::operator==(const AesGcmV1 & rhs) const -{ - if (__isset.aad_prefix != rhs.__isset.aad_prefix) - return false; - else if (__isset.aad_prefix && !(aad_prefix == rhs.aad_prefix)) - return false; - if (__isset.aad_file_unique != rhs.__isset.aad_file_unique) - return false; - else if (__isset.aad_file_unique && !(aad_file_unique == rhs.aad_file_unique)) - return false; - if (__isset.supply_aad_prefix != rhs.__isset.supply_aad_prefix) - return false; - else if (__isset.supply_aad_prefix && !(supply_aad_prefix == rhs.supply_aad_prefix)) - return false; - return true; -} - -AesGcmV1::AesGcmV1(const AesGcmV1& other336) { - aad_prefix = other336.aad_prefix; - aad_file_unique = other336.aad_file_unique; - supply_aad_prefix = other336.supply_aad_prefix; - __isset = other336.__isset; -} -AesGcmV1::AesGcmV1(AesGcmV1&& other337) noexcept { - aad_prefix = std::move(other337.aad_prefix); - aad_file_unique = std::move(other337.aad_file_unique); - supply_aad_prefix = other337.supply_aad_prefix; - __isset = other337.__isset; -} -AesGcmV1& AesGcmV1::operator=(const AesGcmV1& other338) { - aad_prefix = other338.aad_prefix; - aad_file_unique = other338.aad_file_unique; - supply_aad_prefix = other338.supply_aad_prefix; - __isset = other338.__isset; - return *this; -} -AesGcmV1& AesGcmV1::operator=(AesGcmV1&& other339) noexcept { - aad_prefix = std::move(other339.aad_prefix); - aad_file_unique = std::move(other339.aad_file_unique); - supply_aad_prefix = other339.supply_aad_prefix; - __isset = other339.__isset; - return *this; -} -void AesGcmV1::printTo(std::ostream& out) const { - using ::apache::thrift::to_string; - out << "AesGcmV1("; - out << "aad_prefix="; (__isset.aad_prefix ? (out << to_string(aad_prefix)) : (out << "")); - out << ", " << "aad_file_unique="; (__isset.aad_file_unique ? (out << to_string(aad_file_unique)) : (out << "")); - out << ", " << "supply_aad_prefix="; (__isset.supply_aad_prefix ? (out << to_string(supply_aad_prefix)) : (out << "")); - out << ")"; -} - - -AesGcmCtrV1::~AesGcmCtrV1() noexcept { -} - -AesGcmCtrV1::AesGcmCtrV1() noexcept - : aad_prefix(), - aad_file_unique(), - supply_aad_prefix(0) { -} - -void AesGcmCtrV1::__set_aad_prefix(const std::string& val) { - this->aad_prefix = val; -__isset.aad_prefix = true; -} - -void AesGcmCtrV1::__set_aad_file_unique(const std::string& val) { - this->aad_file_unique = val; -__isset.aad_file_unique = true; -} - -void AesGcmCtrV1::__set_supply_aad_prefix(const bool val) { - this->supply_aad_prefix = val; -__isset.supply_aad_prefix = true; -} -std::ostream& operator<<(std::ostream& out, const AesGcmCtrV1& obj) -{ - obj.printTo(out); - return out; -} - - -void swap(AesGcmCtrV1 &a, AesGcmCtrV1 &b) { - using ::std::swap; - swap(a.aad_prefix, b.aad_prefix); - swap(a.aad_file_unique, b.aad_file_unique); - swap(a.supply_aad_prefix, b.supply_aad_prefix); - swap(a.__isset, b.__isset); -} - -bool AesGcmCtrV1::operator==(const AesGcmCtrV1 & rhs) const -{ - if (__isset.aad_prefix != rhs.__isset.aad_prefix) - return false; - else if (__isset.aad_prefix && !(aad_prefix == rhs.aad_prefix)) - return false; - if (__isset.aad_file_unique != rhs.__isset.aad_file_unique) - return false; - else if (__isset.aad_file_unique && !(aad_file_unique == rhs.aad_file_unique)) - return false; - if (__isset.supply_aad_prefix != rhs.__isset.supply_aad_prefix) - return false; - else if (__isset.supply_aad_prefix && !(supply_aad_prefix == rhs.supply_aad_prefix)) - return false; - return true; -} - -AesGcmCtrV1::AesGcmCtrV1(const AesGcmCtrV1& other340) { - aad_prefix = other340.aad_prefix; - aad_file_unique = other340.aad_file_unique; - supply_aad_prefix = other340.supply_aad_prefix; - __isset = other340.__isset; -} -AesGcmCtrV1::AesGcmCtrV1(AesGcmCtrV1&& other341) noexcept { - aad_prefix = std::move(other341.aad_prefix); - aad_file_unique = std::move(other341.aad_file_unique); - supply_aad_prefix = other341.supply_aad_prefix; - __isset = other341.__isset; -} -AesGcmCtrV1& AesGcmCtrV1::operator=(const AesGcmCtrV1& other342) { - aad_prefix = other342.aad_prefix; - aad_file_unique = other342.aad_file_unique; - supply_aad_prefix = other342.supply_aad_prefix; - __isset = other342.__isset; - return *this; -} -AesGcmCtrV1& AesGcmCtrV1::operator=(AesGcmCtrV1&& other343) noexcept { - aad_prefix = std::move(other343.aad_prefix); - aad_file_unique = std::move(other343.aad_file_unique); - supply_aad_prefix = other343.supply_aad_prefix; - __isset = other343.__isset; - return *this; -} -void AesGcmCtrV1::printTo(std::ostream& out) const { - using ::apache::thrift::to_string; - out << "AesGcmCtrV1("; - out << "aad_prefix="; (__isset.aad_prefix ? (out << to_string(aad_prefix)) : (out << "")); - out << ", " << "aad_file_unique="; (__isset.aad_file_unique ? (out << to_string(aad_file_unique)) : (out << "")); - out << ", " << "supply_aad_prefix="; (__isset.supply_aad_prefix ? (out << to_string(supply_aad_prefix)) : (out << "")); - out << ")"; -} - - -EncryptionAlgorithm::~EncryptionAlgorithm() noexcept { -} - -EncryptionAlgorithm::EncryptionAlgorithm() noexcept { -} - -void EncryptionAlgorithm::__set_AES_GCM_V1(const AesGcmV1& val) { - this->AES_GCM_V1 = val; -__isset.AES_GCM_V1 = true; -} - -void EncryptionAlgorithm::__set_AES_GCM_CTR_V1(const AesGcmCtrV1& val) { - this->AES_GCM_CTR_V1 = val; -__isset.AES_GCM_CTR_V1 = true; -} -std::ostream& operator<<(std::ostream& out, const EncryptionAlgorithm& obj) -{ - obj.printTo(out); - return out; -} - - -void swap(EncryptionAlgorithm &a, EncryptionAlgorithm &b) { - using ::std::swap; - swap(a.AES_GCM_V1, b.AES_GCM_V1); - swap(a.AES_GCM_CTR_V1, b.AES_GCM_CTR_V1); - swap(a.__isset, b.__isset); -} - -bool EncryptionAlgorithm::operator==(const EncryptionAlgorithm & rhs) const -{ - if (__isset.AES_GCM_V1 != rhs.__isset.AES_GCM_V1) - return false; - else if (__isset.AES_GCM_V1 && !(AES_GCM_V1 == rhs.AES_GCM_V1)) - return false; - if (__isset.AES_GCM_CTR_V1 != rhs.__isset.AES_GCM_CTR_V1) - return false; - else if (__isset.AES_GCM_CTR_V1 && !(AES_GCM_CTR_V1 == rhs.AES_GCM_CTR_V1)) - return false; - return true; -} - -EncryptionAlgorithm::EncryptionAlgorithm(const EncryptionAlgorithm& other344) { - AES_GCM_V1 = other344.AES_GCM_V1; - AES_GCM_CTR_V1 = other344.AES_GCM_CTR_V1; - __isset = other344.__isset; -} -EncryptionAlgorithm::EncryptionAlgorithm(EncryptionAlgorithm&& other345) noexcept { - AES_GCM_V1 = std::move(other345.AES_GCM_V1); - AES_GCM_CTR_V1 = std::move(other345.AES_GCM_CTR_V1); - __isset = other345.__isset; -} -EncryptionAlgorithm& EncryptionAlgorithm::operator=(const EncryptionAlgorithm& other346) { - AES_GCM_V1 = other346.AES_GCM_V1; - AES_GCM_CTR_V1 = other346.AES_GCM_CTR_V1; - __isset = other346.__isset; - return *this; -} -EncryptionAlgorithm& EncryptionAlgorithm::operator=(EncryptionAlgorithm&& other347) noexcept { - AES_GCM_V1 = std::move(other347.AES_GCM_V1); - AES_GCM_CTR_V1 = std::move(other347.AES_GCM_CTR_V1); - __isset = other347.__isset; - return *this; -} -void EncryptionAlgorithm::printTo(std::ostream& out) const { - using ::apache::thrift::to_string; - out << "EncryptionAlgorithm("; - out << "AES_GCM_V1="; (__isset.AES_GCM_V1 ? (out << to_string(AES_GCM_V1)) : (out << "")); - out << ", " << "AES_GCM_CTR_V1="; (__isset.AES_GCM_CTR_V1 ? (out << to_string(AES_GCM_CTR_V1)) : (out << "")); - out << ")"; -} - - FileMetaData::~FileMetaData() noexcept { } @@ -5463,54 +5537,54 @@ bool FileMetaData::operator==(const FileMetaData & rhs) const return true; } -FileMetaData::FileMetaData(const FileMetaData& other372) { - version = other372.version; - schema = other372.schema; - num_rows = other372.num_rows; - row_groups = other372.row_groups; - key_value_metadata = other372.key_value_metadata; - created_by = other372.created_by; - column_orders = other372.column_orders; - encryption_algorithm = other372.encryption_algorithm; - footer_signing_key_metadata = other372.footer_signing_key_metadata; - __isset = other372.__isset; -} -FileMetaData::FileMetaData(FileMetaData&& other373) noexcept { - version = other373.version; - schema = std::move(other373.schema); - num_rows = other373.num_rows; - row_groups = std::move(other373.row_groups); - key_value_metadata = std::move(other373.key_value_metadata); - created_by = std::move(other373.created_by); - column_orders = std::move(other373.column_orders); - encryption_algorithm = std::move(other373.encryption_algorithm); - footer_signing_key_metadata = std::move(other373.footer_signing_key_metadata); - __isset = other373.__isset; -} -FileMetaData& FileMetaData::operator=(const FileMetaData& other374) { - version = other374.version; - schema = other374.schema; - num_rows = other374.num_rows; - row_groups = other374.row_groups; - key_value_metadata = other374.key_value_metadata; - created_by = other374.created_by; - column_orders = other374.column_orders; - encryption_algorithm = other374.encryption_algorithm; - footer_signing_key_metadata = other374.footer_signing_key_metadata; - __isset = other374.__isset; +FileMetaData::FileMetaData(const FileMetaData& other376) { + version = other376.version; + schema = other376.schema; + num_rows = other376.num_rows; + row_groups = other376.row_groups; + key_value_metadata = other376.key_value_metadata; + created_by = other376.created_by; + column_orders = other376.column_orders; + encryption_algorithm = other376.encryption_algorithm; + footer_signing_key_metadata = other376.footer_signing_key_metadata; + __isset = other376.__isset; +} +FileMetaData::FileMetaData(FileMetaData&& other377) noexcept { + version = other377.version; + schema = std::move(other377.schema); + num_rows = other377.num_rows; + row_groups = std::move(other377.row_groups); + key_value_metadata = std::move(other377.key_value_metadata); + created_by = std::move(other377.created_by); + column_orders = std::move(other377.column_orders); + encryption_algorithm = std::move(other377.encryption_algorithm); + footer_signing_key_metadata = std::move(other377.footer_signing_key_metadata); + __isset = other377.__isset; +} +FileMetaData& FileMetaData::operator=(const FileMetaData& other378) { + version = other378.version; + schema = other378.schema; + num_rows = other378.num_rows; + row_groups = other378.row_groups; + key_value_metadata = other378.key_value_metadata; + created_by = other378.created_by; + column_orders = other378.column_orders; + encryption_algorithm = other378.encryption_algorithm; + footer_signing_key_metadata = other378.footer_signing_key_metadata; + __isset = other378.__isset; return *this; } -FileMetaData& FileMetaData::operator=(FileMetaData&& other375) noexcept { - version = other375.version; - schema = std::move(other375.schema); - num_rows = other375.num_rows; - row_groups = std::move(other375.row_groups); - key_value_metadata = std::move(other375.key_value_metadata); - created_by = std::move(other375.created_by); - column_orders = std::move(other375.column_orders); - encryption_algorithm = std::move(other375.encryption_algorithm); - footer_signing_key_metadata = std::move(other375.footer_signing_key_metadata); - __isset = other375.__isset; +FileMetaData& FileMetaData::operator=(FileMetaData&& other379) noexcept { + version = other379.version; + schema = std::move(other379.schema); + num_rows = other379.num_rows; + row_groups = std::move(other379.row_groups); + key_value_metadata = std::move(other379.key_value_metadata); + created_by = std::move(other379.created_by); + column_orders = std::move(other379.column_orders); + encryption_algorithm = std::move(other379.encryption_algorithm); + footer_signing_key_metadata = std::move(other379.footer_signing_key_metadata); + __isset = other379.__isset; return *this; } void FileMetaData::printTo(std::ostream& out) const { @@ -5569,26 +5643,26 @@ bool FileCryptoMetaData::operator==(const FileCryptoMetaData & rhs) const return true; } -FileCryptoMetaData::FileCryptoMetaData(const FileCryptoMetaData& other376) { - encryption_algorithm = other376.encryption_algorithm; - key_metadata = other376.key_metadata; - __isset = other376.__isset; +FileCryptoMetaData::FileCryptoMetaData(const FileCryptoMetaData& other380) { + encryption_algorithm = other380.encryption_algorithm; + key_metadata = other380.key_metadata; + __isset = other380.__isset; } -FileCryptoMetaData::FileCryptoMetaData(FileCryptoMetaData&& other377) noexcept { - encryption_algorithm = std::move(other377.encryption_algorithm); - key_metadata = std::move(other377.key_metadata); - __isset = other377.__isset; +FileCryptoMetaData::FileCryptoMetaData(FileCryptoMetaData&& other381) noexcept { + encryption_algorithm = std::move(other381.encryption_algorithm); + key_metadata = std::move(other381.key_metadata); + __isset = other381.__isset; } -FileCryptoMetaData& FileCryptoMetaData::operator=(const FileCryptoMetaData& other378) { - encryption_algorithm = other378.encryption_algorithm; - key_metadata = other378.key_metadata; - __isset = other378.__isset; +FileCryptoMetaData& FileCryptoMetaData::operator=(const FileCryptoMetaData& other382) { + encryption_algorithm = other382.encryption_algorithm; + key_metadata = other382.key_metadata; + __isset = other382.__isset; return *this; } -FileCryptoMetaData& FileCryptoMetaData::operator=(FileCryptoMetaData&& other379) noexcept { - encryption_algorithm = std::move(other379.encryption_algorithm); - key_metadata = std::move(other379.key_metadata); - __isset = other379.__isset; +FileCryptoMetaData& FileCryptoMetaData::operator=(FileCryptoMetaData&& other383) noexcept { + encryption_algorithm = std::move(other383.encryption_algorithm); + key_metadata = std::move(other383.key_metadata); + __isset = other383.__isset; return *this; } void FileCryptoMetaData::printTo(std::ostream& out) const { diff --git a/cpp/src/generated/parquet_types.h b/cpp/src/generated/parquet_types.h index 1f1e254f5cf2..5917d753e18a 100644 --- a/cpp/src/generated/parquet_types.h +++ b/cpp/src/generated/parquet_types.h @@ -1,5 +1,5 @@ /** - * Autogenerated by Thrift Compiler (0.21.0) + * Autogenerated by Thrift Compiler (0.22.0) * * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated @@ -453,6 +453,14 @@ class PageEncodingStats; class ColumnMetaData; +class AesGcmV1; + +class AesGcmCtrV1; + +class ExternalDBPAV1; + +class EncryptionAlgorithm; + class EncryptionWithFooterKey; class EncryptionWithColumnKey; @@ -473,12 +481,6 @@ class OffsetIndex; class ColumnIndex; -class AesGcmV1; - -class AesGcmCtrV1; - -class EncryptionAlgorithm; - class FileMetaData; class FileCryptoMetaData; @@ -507,7 +509,7 @@ class SizeStatistics { SizeStatistics& operator=(SizeStatistics&&) noexcept; SizeStatistics() noexcept; - virtual ~SizeStatistics() noexcept; + ~SizeStatistics() noexcept; /** * The number of physical bytes stored for BYTE_ARRAY data values assuming * no encoding. This is exclusive of the bytes needed to store the length of @@ -566,7 +568,7 @@ class SizeStatistics { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(SizeStatistics &a, SizeStatistics &b); @@ -594,7 +596,7 @@ class BoundingBox { BoundingBox& operator=(BoundingBox&&) noexcept; BoundingBox() noexcept; - virtual ~BoundingBox() noexcept; + ~BoundingBox() noexcept; double xmin; double xmax; double ymin; @@ -634,7 +636,7 @@ class BoundingBox { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(BoundingBox &a, BoundingBox &b); @@ -659,7 +661,7 @@ class GeospatialStatistics { GeospatialStatistics& operator=(GeospatialStatistics&&) noexcept; GeospatialStatistics() noexcept; - virtual ~GeospatialStatistics() noexcept; + ~GeospatialStatistics() noexcept; /** * A bounding box of geospatial instances */ @@ -687,7 +689,7 @@ class GeospatialStatistics { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(GeospatialStatistics &a, GeospatialStatistics &b); @@ -719,7 +721,7 @@ class Statistics { Statistics& operator=(Statistics&&) noexcept; Statistics() noexcept; - virtual ~Statistics() noexcept; + ~Statistics() noexcept; /** * DEPRECATED: min and max value of the column. Use min_value and max_value. * @@ -801,7 +803,7 @@ class Statistics { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(Statistics &a, Statistics &b); @@ -821,7 +823,7 @@ class StringType { StringType& operator=(StringType&&) noexcept; StringType() noexcept; - virtual ~StringType() noexcept; + ~StringType() noexcept; bool operator == (const StringType & /* rhs */) const; bool operator != (const StringType &rhs) const { @@ -835,7 +837,7 @@ class StringType { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(StringType &a, StringType &b); @@ -852,7 +854,7 @@ class UUIDType { UUIDType& operator=(UUIDType&&) noexcept; UUIDType() noexcept; - virtual ~UUIDType() noexcept; + ~UUIDType() noexcept; bool operator == (const UUIDType & /* rhs */) const; bool operator != (const UUIDType &rhs) const { @@ -866,7 +868,7 @@ class UUIDType { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(UUIDType &a, UUIDType &b); @@ -883,7 +885,7 @@ class MapType { MapType& operator=(MapType&&) noexcept; MapType() noexcept; - virtual ~MapType() noexcept; + ~MapType() noexcept; bool operator == (const MapType & /* rhs */) const; bool operator != (const MapType &rhs) const { @@ -897,7 +899,7 @@ class MapType { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(MapType &a, MapType &b); @@ -914,7 +916,7 @@ class ListType { ListType& operator=(ListType&&) noexcept; ListType() noexcept; - virtual ~ListType() noexcept; + ~ListType() noexcept; bool operator == (const ListType & /* rhs */) const; bool operator != (const ListType &rhs) const { @@ -928,7 +930,7 @@ class ListType { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(ListType &a, ListType &b); @@ -945,7 +947,7 @@ class EnumType { EnumType& operator=(EnumType&&) noexcept; EnumType() noexcept; - virtual ~EnumType() noexcept; + ~EnumType() noexcept; bool operator == (const EnumType & /* rhs */) const; bool operator != (const EnumType &rhs) const { @@ -959,7 +961,7 @@ class EnumType { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(EnumType &a, EnumType &b); @@ -976,7 +978,7 @@ class DateType { DateType& operator=(DateType&&) noexcept; DateType() noexcept; - virtual ~DateType() noexcept; + ~DateType() noexcept; bool operator == (const DateType & /* rhs */) const; bool operator != (const DateType &rhs) const { @@ -990,7 +992,7 @@ class DateType { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(DateType &a, DateType &b); @@ -1007,7 +1009,7 @@ class Float16Type { Float16Type& operator=(Float16Type&&) noexcept; Float16Type() noexcept; - virtual ~Float16Type() noexcept; + ~Float16Type() noexcept; bool operator == (const Float16Type & /* rhs */) const; bool operator != (const Float16Type &rhs) const { @@ -1021,7 +1023,7 @@ class Float16Type { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(Float16Type &a, Float16Type &b); @@ -1045,7 +1047,7 @@ class NullType { NullType& operator=(NullType&&) noexcept; NullType() noexcept; - virtual ~NullType() noexcept; + ~NullType() noexcept; bool operator == (const NullType & /* rhs */) const; bool operator != (const NullType &rhs) const { @@ -1059,7 +1061,7 @@ class NullType { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(NullType &a, NullType &b); @@ -1087,7 +1089,7 @@ class DecimalType { DecimalType& operator=(DecimalType&&) noexcept; DecimalType() noexcept; - virtual ~DecimalType() noexcept; + ~DecimalType() noexcept; int32_t scale; int32_t precision; @@ -1107,7 +1109,7 @@ class DecimalType { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(DecimalType &a, DecimalType &b); @@ -1127,7 +1129,7 @@ class MilliSeconds { MilliSeconds& operator=(MilliSeconds&&) noexcept; MilliSeconds() noexcept; - virtual ~MilliSeconds() noexcept; + ~MilliSeconds() noexcept; bool operator == (const MilliSeconds & /* rhs */) const; bool operator != (const MilliSeconds &rhs) const { @@ -1141,7 +1143,7 @@ class MilliSeconds { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(MilliSeconds &a, MilliSeconds &b); @@ -1158,7 +1160,7 @@ class MicroSeconds { MicroSeconds& operator=(MicroSeconds&&) noexcept; MicroSeconds() noexcept; - virtual ~MicroSeconds() noexcept; + ~MicroSeconds() noexcept; bool operator == (const MicroSeconds & /* rhs */) const; bool operator != (const MicroSeconds &rhs) const { @@ -1172,7 +1174,7 @@ class MicroSeconds { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(MicroSeconds &a, MicroSeconds &b); @@ -1189,7 +1191,7 @@ class NanoSeconds { NanoSeconds& operator=(NanoSeconds&&) noexcept; NanoSeconds() noexcept; - virtual ~NanoSeconds() noexcept; + ~NanoSeconds() noexcept; bool operator == (const NanoSeconds & /* rhs */) const; bool operator != (const NanoSeconds &rhs) const { @@ -1203,7 +1205,7 @@ class NanoSeconds { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(NanoSeconds &a, NanoSeconds &b); @@ -1226,7 +1228,7 @@ class TimeUnit { TimeUnit& operator=(TimeUnit&&) noexcept; TimeUnit() noexcept; - virtual ~TimeUnit() noexcept; + ~TimeUnit() noexcept; MilliSeconds MILLIS; MicroSeconds MICROS; NanoSeconds NANOS; @@ -1251,7 +1253,7 @@ class TimeUnit { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(TimeUnit &a, TimeUnit &b); @@ -1273,7 +1275,7 @@ class TimestampType { TimestampType& operator=(TimestampType&&) noexcept; TimestampType() noexcept; - virtual ~TimestampType() noexcept; + ~TimestampType() noexcept; bool isAdjustedToUTC; TimeUnit unit; @@ -1293,7 +1295,7 @@ class TimestampType { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(TimestampType &a, TimestampType &b); @@ -1315,7 +1317,7 @@ class TimeType { TimeType& operator=(TimeType&&) noexcept; TimeType() noexcept; - virtual ~TimeType() noexcept; + ~TimeType() noexcept; bool isAdjustedToUTC; TimeUnit unit; @@ -1335,7 +1337,7 @@ class TimeType { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(TimeType &a, TimeType &b); @@ -1359,7 +1361,7 @@ class IntType { IntType& operator=(IntType&&) noexcept; IntType() noexcept; - virtual ~IntType() noexcept; + ~IntType() noexcept; int8_t bitWidth; bool isSigned; @@ -1379,7 +1381,7 @@ class IntType { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(IntType &a, IntType &b); @@ -1401,7 +1403,7 @@ class JsonType { JsonType& operator=(JsonType&&) noexcept; JsonType() noexcept; - virtual ~JsonType() noexcept; + ~JsonType() noexcept; bool operator == (const JsonType & /* rhs */) const; bool operator != (const JsonType &rhs) const { @@ -1415,7 +1417,7 @@ class JsonType { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(JsonType &a, JsonType &b); @@ -1437,7 +1439,7 @@ class BsonType { BsonType& operator=(BsonType&&) noexcept; BsonType() noexcept; - virtual ~BsonType() noexcept; + ~BsonType() noexcept; bool operator == (const BsonType & /* rhs */) const; bool operator != (const BsonType &rhs) const { @@ -1451,7 +1453,7 @@ class BsonType { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(BsonType &a, BsonType &b); @@ -1475,7 +1477,7 @@ class VariantType { VariantType& operator=(VariantType&&) noexcept; VariantType() noexcept; - virtual ~VariantType() noexcept; + ~VariantType() noexcept; int8_t specification_version; _VariantType__isset __isset; @@ -1494,7 +1496,7 @@ class VariantType { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(VariantType &a, VariantType &b); @@ -1529,7 +1531,7 @@ class GeometryType { GeometryType& operator=(GeometryType&&) noexcept; GeometryType() noexcept; - virtual ~GeometryType() noexcept; + ~GeometryType() noexcept; std::string crs; _GeometryType__isset __isset; @@ -1548,7 +1550,7 @@ class GeometryType { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(GeometryType &a, GeometryType &b); @@ -1587,7 +1589,7 @@ class GeographyType { GeographyType& operator=(GeographyType&&) noexcept; GeographyType() noexcept; - virtual ~GeographyType() noexcept; + ~GeographyType() noexcept; std::string crs; /** * @@ -1613,7 +1615,7 @@ class GeographyType { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(GeographyType &a, GeographyType &b); @@ -1657,7 +1659,7 @@ class LogicalType { LogicalType& operator=(LogicalType&&) noexcept; LogicalType() noexcept; - virtual ~LogicalType() noexcept; + ~LogicalType() noexcept; StringType STRING; MapType MAP; ListType LIST; @@ -1724,7 +1726,7 @@ class LogicalType { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(LogicalType &a, LogicalType &b); @@ -1759,7 +1761,7 @@ class SchemaElement { SchemaElement& operator=(SchemaElement&&) noexcept; SchemaElement() noexcept; - virtual ~SchemaElement() noexcept; + ~SchemaElement() noexcept; /** * Data type for this field. Not set if the current element is a non-leaf node * @@ -1855,7 +1857,7 @@ class SchemaElement { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(SchemaElement &a, SchemaElement &b); @@ -1879,7 +1881,7 @@ class DataPageHeader { DataPageHeader& operator=(DataPageHeader&&) noexcept; DataPageHeader() noexcept; - virtual ~DataPageHeader() noexcept; + ~DataPageHeader() noexcept; /** * Number of values, including NULLs, in this data page. * @@ -1936,7 +1938,7 @@ class DataPageHeader { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(DataPageHeader &a, DataPageHeader &b); @@ -1953,7 +1955,7 @@ class IndexPageHeader { IndexPageHeader& operator=(IndexPageHeader&&) noexcept; IndexPageHeader() noexcept; - virtual ~IndexPageHeader() noexcept; + ~IndexPageHeader() noexcept; bool operator == (const IndexPageHeader & /* rhs */) const; bool operator != (const IndexPageHeader &rhs) const { @@ -1967,7 +1969,7 @@ class IndexPageHeader { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(IndexPageHeader &a, IndexPageHeader &b); @@ -1994,7 +1996,7 @@ class DictionaryPageHeader { DictionaryPageHeader& operator=(DictionaryPageHeader&&) noexcept; DictionaryPageHeader() noexcept; - virtual ~DictionaryPageHeader() noexcept; + ~DictionaryPageHeader() noexcept; /** * Number of values in the dictionary * */ @@ -2030,7 +2032,7 @@ class DictionaryPageHeader { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(DictionaryPageHeader &a, DictionaryPageHeader &b); @@ -2058,7 +2060,7 @@ class DataPageHeaderV2 { DataPageHeaderV2& operator=(DataPageHeaderV2&&) noexcept; DataPageHeaderV2() noexcept; - virtual ~DataPageHeaderV2() noexcept; + ~DataPageHeaderV2() noexcept; /** * Number of values, including NULLs, in this data page. * */ @@ -2132,7 +2134,7 @@ class DataPageHeaderV2 { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(DataPageHeaderV2 &a, DataPageHeaderV2 &b); @@ -2152,7 +2154,7 @@ class SplitBlockAlgorithm { SplitBlockAlgorithm& operator=(SplitBlockAlgorithm&&) noexcept; SplitBlockAlgorithm() noexcept; - virtual ~SplitBlockAlgorithm() noexcept; + ~SplitBlockAlgorithm() noexcept; bool operator == (const SplitBlockAlgorithm & /* rhs */) const; bool operator != (const SplitBlockAlgorithm &rhs) const { @@ -2166,7 +2168,7 @@ class SplitBlockAlgorithm { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(SplitBlockAlgorithm &a, SplitBlockAlgorithm &b); @@ -2190,7 +2192,7 @@ class BloomFilterAlgorithm { BloomFilterAlgorithm& operator=(BloomFilterAlgorithm&&) noexcept; BloomFilterAlgorithm() noexcept; - virtual ~BloomFilterAlgorithm() noexcept; + ~BloomFilterAlgorithm() noexcept; /** * Block-based Bloom filter. * */ @@ -2212,7 +2214,7 @@ class BloomFilterAlgorithm { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(BloomFilterAlgorithm &a, BloomFilterAlgorithm &b); @@ -2234,7 +2236,7 @@ class XxHash { XxHash& operator=(XxHash&&) noexcept; XxHash() noexcept; - virtual ~XxHash() noexcept; + ~XxHash() noexcept; bool operator == (const XxHash & /* rhs */) const; bool operator != (const XxHash &rhs) const { @@ -2248,7 +2250,7 @@ class XxHash { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(XxHash &a, XxHash &b); @@ -2274,7 +2276,7 @@ class BloomFilterHash { BloomFilterHash& operator=(BloomFilterHash&&) noexcept; BloomFilterHash() noexcept; - virtual ~BloomFilterHash() noexcept; + ~BloomFilterHash() noexcept; /** * xxHash Strategy. * */ @@ -2296,7 +2298,7 @@ class BloomFilterHash { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(BloomFilterHash &a, BloomFilterHash &b); @@ -2317,7 +2319,7 @@ class Uncompressed { Uncompressed& operator=(Uncompressed&&) noexcept; Uncompressed() noexcept; - virtual ~Uncompressed() noexcept; + ~Uncompressed() noexcept; bool operator == (const Uncompressed & /* rhs */) const; bool operator != (const Uncompressed &rhs) const { @@ -2331,7 +2333,7 @@ class Uncompressed { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(Uncompressed &a, Uncompressed &b); @@ -2352,7 +2354,7 @@ class BloomFilterCompression { BloomFilterCompression& operator=(BloomFilterCompression&&) noexcept; BloomFilterCompression() noexcept; - virtual ~BloomFilterCompression() noexcept; + ~BloomFilterCompression() noexcept; Uncompressed UNCOMPRESSED; _BloomFilterCompression__isset __isset; @@ -2371,7 +2373,7 @@ class BloomFilterCompression { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(BloomFilterCompression &a, BloomFilterCompression &b); @@ -2393,7 +2395,7 @@ class BloomFilterHeader { BloomFilterHeader& operator=(BloomFilterHeader&&) noexcept; BloomFilterHeader() noexcept; - virtual ~BloomFilterHeader() noexcept; + ~BloomFilterHeader() noexcept; /** * The size of bitset in bytes * */ @@ -2431,7 +2433,7 @@ class BloomFilterHeader { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(BloomFilterHeader &a, BloomFilterHeader &b); @@ -2456,7 +2458,7 @@ class PageHeader { PageHeader& operator=(PageHeader&&) noexcept; PageHeader() noexcept; - virtual ~PageHeader() noexcept; + ~PageHeader() noexcept; /** * the type of the page: indicates which of the *_header fields is set * * @@ -2525,7 +2527,7 @@ class PageHeader { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(PageHeader &a, PageHeader &b); @@ -2549,7 +2551,7 @@ class KeyValue { KeyValue& operator=(KeyValue&&) noexcept; KeyValue() noexcept; - virtual ~KeyValue() noexcept; + ~KeyValue() noexcept; std::string key; std::string value; @@ -2571,7 +2573,7 @@ class KeyValue { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(KeyValue &a, KeyValue &b); @@ -2591,7 +2593,7 @@ class SortingColumn { SortingColumn& operator=(SortingColumn&&) noexcept; SortingColumn() noexcept; - virtual ~SortingColumn() noexcept; + ~SortingColumn() noexcept; /** * The ordinal position of the column (in this row group) * */ @@ -2624,7 +2626,7 @@ class SortingColumn { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(SortingColumn &a, SortingColumn &b); @@ -2644,7 +2646,7 @@ class PageEncodingStats { PageEncodingStats& operator=(PageEncodingStats&&) noexcept; PageEncodingStats() noexcept; - virtual ~PageEncodingStats() noexcept; + ~PageEncodingStats() noexcept; /** * the page type (data/dic/...) * * @@ -2680,7 +2682,7 @@ class PageEncodingStats { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(PageEncodingStats &a, PageEncodingStats &b); @@ -2712,7 +2714,7 @@ class ColumnMetaData { ColumnMetaData& operator=(ColumnMetaData&&) noexcept; ColumnMetaData() noexcept; - virtual ~ColumnMetaData() noexcept; + ~ColumnMetaData() noexcept; /** * Type of this column * * @@ -2845,13 +2847,208 @@ class ColumnMetaData { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(ColumnMetaData &a, ColumnMetaData &b); std::ostream& operator<<(std::ostream& out, const ColumnMetaData& obj); +typedef struct _AesGcmV1__isset { + _AesGcmV1__isset() : aad_prefix(false), aad_file_unique(false), supply_aad_prefix(false) {} + bool aad_prefix :1; + bool aad_file_unique :1; + bool supply_aad_prefix :1; +} _AesGcmV1__isset; + +class AesGcmV1 { + public: + + AesGcmV1(const AesGcmV1&); + AesGcmV1(AesGcmV1&&) noexcept; + AesGcmV1& operator=(const AesGcmV1&); + AesGcmV1& operator=(AesGcmV1&&) noexcept; + AesGcmV1() noexcept; + + ~AesGcmV1() noexcept; + /** + * AAD prefix * + */ + std::string aad_prefix; + /** + * Unique file identifier part of AAD suffix * + */ + std::string aad_file_unique; + /** + * In files encrypted with AAD prefix without storing it, + * readers must supply the prefix * + */ + bool supply_aad_prefix; + + _AesGcmV1__isset __isset; + + void __set_aad_prefix(const std::string& val); + + void __set_aad_file_unique(const std::string& val); + + void __set_supply_aad_prefix(const bool val); + + bool operator == (const AesGcmV1 & rhs) const; + bool operator != (const AesGcmV1 &rhs) const { + return !(*this == rhs); + } + + bool operator < (const AesGcmV1 & ) const; + + template + uint32_t read(Protocol_* iprot); + template + uint32_t write(Protocol_* oprot) const; + + void printTo(std::ostream& out) const; +}; + +void swap(AesGcmV1 &a, AesGcmV1 &b); + +std::ostream& operator<<(std::ostream& out, const AesGcmV1& obj); + +typedef struct _AesGcmCtrV1__isset { + _AesGcmCtrV1__isset() : aad_prefix(false), aad_file_unique(false), supply_aad_prefix(false) {} + bool aad_prefix :1; + bool aad_file_unique :1; + bool supply_aad_prefix :1; +} _AesGcmCtrV1__isset; + +class AesGcmCtrV1 { + public: + + AesGcmCtrV1(const AesGcmCtrV1&); + AesGcmCtrV1(AesGcmCtrV1&&) noexcept; + AesGcmCtrV1& operator=(const AesGcmCtrV1&); + AesGcmCtrV1& operator=(AesGcmCtrV1&&) noexcept; + AesGcmCtrV1() noexcept; + + ~AesGcmCtrV1() noexcept; + /** + * AAD prefix * + */ + std::string aad_prefix; + /** + * Unique file identifier part of AAD suffix * + */ + std::string aad_file_unique; + /** + * In files encrypted with AAD prefix without storing it, + * readers must supply the prefix * + */ + bool supply_aad_prefix; + + _AesGcmCtrV1__isset __isset; + + void __set_aad_prefix(const std::string& val); + + void __set_aad_file_unique(const std::string& val); + + void __set_supply_aad_prefix(const bool val); + + bool operator == (const AesGcmCtrV1 & rhs) const; + bool operator != (const AesGcmCtrV1 &rhs) const { + return !(*this == rhs); + } + + bool operator < (const AesGcmCtrV1 & ) const; + + template + uint32_t read(Protocol_* iprot); + template + uint32_t write(Protocol_* oprot) const; + + void printTo(std::ostream& out) const; +}; + +void swap(AesGcmCtrV1 &a, AesGcmCtrV1 &b); + +std::ostream& operator<<(std::ostream& out, const AesGcmCtrV1& obj); + + +class ExternalDBPAV1 { + public: + + ExternalDBPAV1(const ExternalDBPAV1&) noexcept; + ExternalDBPAV1(ExternalDBPAV1&&) noexcept; + ExternalDBPAV1& operator=(const ExternalDBPAV1&) noexcept; + ExternalDBPAV1& operator=(ExternalDBPAV1&&) noexcept; + ExternalDBPAV1() noexcept; + + ~ExternalDBPAV1() noexcept; + + bool operator == (const ExternalDBPAV1 & /* rhs */) const; + bool operator != (const ExternalDBPAV1 &rhs) const { + return !(*this == rhs); + } + + bool operator < (const ExternalDBPAV1 & ) const; + + template + uint32_t read(Protocol_* iprot); + template + uint32_t write(Protocol_* oprot) const; + + void printTo(std::ostream& out) const; +}; + +void swap(ExternalDBPAV1 &a, ExternalDBPAV1 &b); + +std::ostream& operator<<(std::ostream& out, const ExternalDBPAV1& obj); + +typedef struct _EncryptionAlgorithm__isset { + _EncryptionAlgorithm__isset() : AES_GCM_V1(false), AES_GCM_CTR_V1(false), EXTERNAL_DBPA_V1(false) {} + bool AES_GCM_V1 :1; + bool AES_GCM_CTR_V1 :1; + bool EXTERNAL_DBPA_V1 :1; +} _EncryptionAlgorithm__isset; + +class EncryptionAlgorithm { + public: + + EncryptionAlgorithm(const EncryptionAlgorithm&); + EncryptionAlgorithm(EncryptionAlgorithm&&) noexcept; + EncryptionAlgorithm& operator=(const EncryptionAlgorithm&); + EncryptionAlgorithm& operator=(EncryptionAlgorithm&&) noexcept; + EncryptionAlgorithm() noexcept; + + ~EncryptionAlgorithm() noexcept; + AesGcmV1 AES_GCM_V1; + AesGcmCtrV1 AES_GCM_CTR_V1; + ExternalDBPAV1 EXTERNAL_DBPA_V1; + + _EncryptionAlgorithm__isset __isset; + + void __set_AES_GCM_V1(const AesGcmV1& val); + + void __set_AES_GCM_CTR_V1(const AesGcmCtrV1& val); + + void __set_EXTERNAL_DBPA_V1(const ExternalDBPAV1& val); + + bool operator == (const EncryptionAlgorithm & rhs) const; + bool operator != (const EncryptionAlgorithm &rhs) const { + return !(*this == rhs); + } + + bool operator < (const EncryptionAlgorithm & ) const; + + template + uint32_t read(Protocol_* iprot); + template + uint32_t write(Protocol_* oprot) const; + + void printTo(std::ostream& out) const; +}; + +void swap(EncryptionAlgorithm &a, EncryptionAlgorithm &b); + +std::ostream& operator<<(std::ostream& out, const EncryptionAlgorithm& obj); + class EncryptionWithFooterKey { public: @@ -2862,7 +3059,7 @@ class EncryptionWithFooterKey { EncryptionWithFooterKey& operator=(EncryptionWithFooterKey&&) noexcept; EncryptionWithFooterKey() noexcept; - virtual ~EncryptionWithFooterKey() noexcept; + ~EncryptionWithFooterKey() noexcept; bool operator == (const EncryptionWithFooterKey & /* rhs */) const; bool operator != (const EncryptionWithFooterKey &rhs) const { @@ -2876,7 +3073,7 @@ class EncryptionWithFooterKey { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(EncryptionWithFooterKey &a, EncryptionWithFooterKey &b); @@ -2884,8 +3081,9 @@ void swap(EncryptionWithFooterKey &a, EncryptionWithFooterKey &b); std::ostream& operator<<(std::ostream& out, const EncryptionWithFooterKey& obj); typedef struct _EncryptionWithColumnKey__isset { - _EncryptionWithColumnKey__isset() : key_metadata(false) {} + _EncryptionWithColumnKey__isset() : key_metadata(false), encryption_algorithm(false) {} bool key_metadata :1; + bool encryption_algorithm :1; } _EncryptionWithColumnKey__isset; class EncryptionWithColumnKey { @@ -2897,7 +3095,7 @@ class EncryptionWithColumnKey { EncryptionWithColumnKey& operator=(EncryptionWithColumnKey&&) noexcept; EncryptionWithColumnKey() noexcept; - virtual ~EncryptionWithColumnKey() noexcept; + ~EncryptionWithColumnKey() noexcept; /** * Column path in schema * */ @@ -2906,6 +3104,10 @@ class EncryptionWithColumnKey { * Retrieval metadata of column encryption key * */ std::string key_metadata; + /** + * Column specific encryption algorithm * + */ + EncryptionAlgorithm encryption_algorithm; _EncryptionWithColumnKey__isset __isset; @@ -2913,6 +3115,8 @@ class EncryptionWithColumnKey { void __set_key_metadata(const std::string& val); + void __set_encryption_algorithm(const EncryptionAlgorithm& val); + bool operator == (const EncryptionWithColumnKey & rhs) const; bool operator != (const EncryptionWithColumnKey &rhs) const { return !(*this == rhs); @@ -2925,7 +3129,7 @@ class EncryptionWithColumnKey { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(EncryptionWithColumnKey &a, EncryptionWithColumnKey &b); @@ -2947,7 +3151,7 @@ class ColumnCryptoMetaData { ColumnCryptoMetaData& operator=(ColumnCryptoMetaData&&) noexcept; ColumnCryptoMetaData() noexcept; - virtual ~ColumnCryptoMetaData() noexcept; + ~ColumnCryptoMetaData() noexcept; EncryptionWithFooterKey ENCRYPTION_WITH_FOOTER_KEY; EncryptionWithColumnKey ENCRYPTION_WITH_COLUMN_KEY; @@ -2969,7 +3173,7 @@ class ColumnCryptoMetaData { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(ColumnCryptoMetaData &a, ColumnCryptoMetaData &b); @@ -2997,7 +3201,7 @@ class ColumnChunk { ColumnChunk& operator=(ColumnChunk&&) noexcept; ColumnChunk() noexcept; - virtual ~ColumnChunk() noexcept; + ~ColumnChunk() noexcept; /** * File where column data is stored. If not set, assumed to be same file as * metadata. This path is relative to the current file. @@ -3080,7 +3284,7 @@ class ColumnChunk { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(ColumnChunk &a, ColumnChunk &b); @@ -3104,7 +3308,7 @@ class RowGroup { RowGroup& operator=(RowGroup&&) noexcept; RowGroup() noexcept; - virtual ~RowGroup() noexcept; + ~RowGroup() noexcept; /** * Metadata for each column chunk in this row group. * This list must have the same order as the SchemaElement list in FileMetaData. @@ -3167,7 +3371,7 @@ class RowGroup { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(RowGroup &a, RowGroup &b); @@ -3187,7 +3391,7 @@ class TypeDefinedOrder { TypeDefinedOrder& operator=(TypeDefinedOrder&&) noexcept; TypeDefinedOrder() noexcept; - virtual ~TypeDefinedOrder() noexcept; + ~TypeDefinedOrder() noexcept; bool operator == (const TypeDefinedOrder & /* rhs */) const; bool operator != (const TypeDefinedOrder &rhs) const { @@ -3201,7 +3405,7 @@ class TypeDefinedOrder { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(TypeDefinedOrder &a, TypeDefinedOrder &b); @@ -3234,7 +3438,7 @@ class ColumnOrder { ColumnOrder& operator=(ColumnOrder&&) noexcept; ColumnOrder() noexcept; - virtual ~ColumnOrder() noexcept; + ~ColumnOrder() noexcept; /** * The sort orders for logical types are: * UTF8 - unsigned byte-wise comparison @@ -3306,7 +3510,7 @@ class ColumnOrder { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(ColumnOrder &a, ColumnOrder &b); @@ -3323,7 +3527,7 @@ class PageLocation { PageLocation& operator=(PageLocation&&) noexcept; PageLocation() noexcept; - virtual ~PageLocation() noexcept; + ~PageLocation() noexcept; /** * Offset of the page in the file * */ @@ -3358,7 +3562,7 @@ class PageLocation { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(PageLocation &a, PageLocation &b); @@ -3386,7 +3590,7 @@ class OffsetIndex { OffsetIndex& operator=(OffsetIndex&&) noexcept; OffsetIndex() noexcept; - virtual ~OffsetIndex() noexcept; + ~OffsetIndex() noexcept; /** * PageLocations, ordered by increasing PageLocation.offset. It is required * that page_locations[i].first_row_index < page_locations[i+1].first_row_index. @@ -3418,7 +3622,7 @@ class OffsetIndex { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(OffsetIndex &a, OffsetIndex &b); @@ -3451,7 +3655,7 @@ class ColumnIndex { ColumnIndex& operator=(ColumnIndex&&) noexcept; ColumnIndex() noexcept; - virtual ~ColumnIndex() noexcept; + ~ColumnIndex() noexcept; /** * A list of Boolean values to determine the validity of the corresponding * min and max values. If true, a page contains only null values, and writers @@ -3540,173 +3744,13 @@ class ColumnIndex { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(ColumnIndex &a, ColumnIndex &b); std::ostream& operator<<(std::ostream& out, const ColumnIndex& obj); -typedef struct _AesGcmV1__isset { - _AesGcmV1__isset() : aad_prefix(false), aad_file_unique(false), supply_aad_prefix(false) {} - bool aad_prefix :1; - bool aad_file_unique :1; - bool supply_aad_prefix :1; -} _AesGcmV1__isset; - -class AesGcmV1 { - public: - - AesGcmV1(const AesGcmV1&); - AesGcmV1(AesGcmV1&&) noexcept; - AesGcmV1& operator=(const AesGcmV1&); - AesGcmV1& operator=(AesGcmV1&&) noexcept; - AesGcmV1() noexcept; - - virtual ~AesGcmV1() noexcept; - /** - * AAD prefix * - */ - std::string aad_prefix; - /** - * Unique file identifier part of AAD suffix * - */ - std::string aad_file_unique; - /** - * In files encrypted with AAD prefix without storing it, - * readers must supply the prefix * - */ - bool supply_aad_prefix; - - _AesGcmV1__isset __isset; - - void __set_aad_prefix(const std::string& val); - - void __set_aad_file_unique(const std::string& val); - - void __set_supply_aad_prefix(const bool val); - - bool operator == (const AesGcmV1 & rhs) const; - bool operator != (const AesGcmV1 &rhs) const { - return !(*this == rhs); - } - - bool operator < (const AesGcmV1 & ) const; - - template - uint32_t read(Protocol_* iprot); - template - uint32_t write(Protocol_* oprot) const; - - virtual void printTo(std::ostream& out) const; -}; - -void swap(AesGcmV1 &a, AesGcmV1 &b); - -std::ostream& operator<<(std::ostream& out, const AesGcmV1& obj); - -typedef struct _AesGcmCtrV1__isset { - _AesGcmCtrV1__isset() : aad_prefix(false), aad_file_unique(false), supply_aad_prefix(false) {} - bool aad_prefix :1; - bool aad_file_unique :1; - bool supply_aad_prefix :1; -} _AesGcmCtrV1__isset; - -class AesGcmCtrV1 { - public: - - AesGcmCtrV1(const AesGcmCtrV1&); - AesGcmCtrV1(AesGcmCtrV1&&) noexcept; - AesGcmCtrV1& operator=(const AesGcmCtrV1&); - AesGcmCtrV1& operator=(AesGcmCtrV1&&) noexcept; - AesGcmCtrV1() noexcept; - - virtual ~AesGcmCtrV1() noexcept; - /** - * AAD prefix * - */ - std::string aad_prefix; - /** - * Unique file identifier part of AAD suffix * - */ - std::string aad_file_unique; - /** - * In files encrypted with AAD prefix without storing it, - * readers must supply the prefix * - */ - bool supply_aad_prefix; - - _AesGcmCtrV1__isset __isset; - - void __set_aad_prefix(const std::string& val); - - void __set_aad_file_unique(const std::string& val); - - void __set_supply_aad_prefix(const bool val); - - bool operator == (const AesGcmCtrV1 & rhs) const; - bool operator != (const AesGcmCtrV1 &rhs) const { - return !(*this == rhs); - } - - bool operator < (const AesGcmCtrV1 & ) const; - - template - uint32_t read(Protocol_* iprot); - template - uint32_t write(Protocol_* oprot) const; - - virtual void printTo(std::ostream& out) const; -}; - -void swap(AesGcmCtrV1 &a, AesGcmCtrV1 &b); - -std::ostream& operator<<(std::ostream& out, const AesGcmCtrV1& obj); - -typedef struct _EncryptionAlgorithm__isset { - _EncryptionAlgorithm__isset() : AES_GCM_V1(false), AES_GCM_CTR_V1(false) {} - bool AES_GCM_V1 :1; - bool AES_GCM_CTR_V1 :1; -} _EncryptionAlgorithm__isset; - -class EncryptionAlgorithm { - public: - - EncryptionAlgorithm(const EncryptionAlgorithm&); - EncryptionAlgorithm(EncryptionAlgorithm&&) noexcept; - EncryptionAlgorithm& operator=(const EncryptionAlgorithm&); - EncryptionAlgorithm& operator=(EncryptionAlgorithm&&) noexcept; - EncryptionAlgorithm() noexcept; - - virtual ~EncryptionAlgorithm() noexcept; - AesGcmV1 AES_GCM_V1; - AesGcmCtrV1 AES_GCM_CTR_V1; - - _EncryptionAlgorithm__isset __isset; - - void __set_AES_GCM_V1(const AesGcmV1& val); - - void __set_AES_GCM_CTR_V1(const AesGcmCtrV1& val); - - bool operator == (const EncryptionAlgorithm & rhs) const; - bool operator != (const EncryptionAlgorithm &rhs) const { - return !(*this == rhs); - } - - bool operator < (const EncryptionAlgorithm & ) const; - - template - uint32_t read(Protocol_* iprot); - template - uint32_t write(Protocol_* oprot) const; - - virtual void printTo(std::ostream& out) const; -}; - -void swap(EncryptionAlgorithm &a, EncryptionAlgorithm &b); - -std::ostream& operator<<(std::ostream& out, const EncryptionAlgorithm& obj); - typedef struct _FileMetaData__isset { _FileMetaData__isset() : key_value_metadata(false), created_by(false), column_orders(false), encryption_algorithm(false), footer_signing_key_metadata(false) {} bool key_value_metadata :1; @@ -3728,7 +3772,7 @@ class FileMetaData { FileMetaData& operator=(FileMetaData&&) noexcept; FileMetaData() noexcept; - virtual ~FileMetaData() noexcept; + ~FileMetaData() noexcept; /** * Version of this file * */ @@ -3822,7 +3866,7 @@ class FileMetaData { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(FileMetaData &a, FileMetaData &b); @@ -3846,7 +3890,7 @@ class FileCryptoMetaData { FileCryptoMetaData& operator=(FileCryptoMetaData&&) noexcept; FileCryptoMetaData() noexcept; - virtual ~FileCryptoMetaData() noexcept; + ~FileCryptoMetaData() noexcept; /** * Encryption algorithm. This field is only used for files * with encrypted footer. Files with plaintext footer store algorithm id @@ -3877,7 +3921,7 @@ class FileCryptoMetaData { template uint32_t write(Protocol_* oprot) const; - virtual void printTo(std::ostream& out) const; + void printTo(std::ostream& out) const; }; void swap(FileCryptoMetaData &a, FileCryptoMetaData &b); diff --git a/cpp/src/generated/parquet_types.tcc b/cpp/src/generated/parquet_types.tcc index 78e3e2549394..e99ded1647e3 100644 --- a/cpp/src/generated/parquet_types.tcc +++ b/cpp/src/generated/parquet_types.tcc @@ -1,5 +1,5 @@ /** - * Autogenerated by Thrift Compiler (0.21.0) + * Autogenerated by Thrift Compiler (0.22.0) * * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated @@ -3673,7 +3673,7 @@ uint32_t ColumnMetaData::write(Protocol_* oprot) const { } template -uint32_t EncryptionWithFooterKey::read(Protocol_* iprot) { +uint32_t AesGcmV1::read(Protocol_* iprot) { ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); uint32_t xfer = 0; @@ -3692,7 +3692,36 @@ uint32_t EncryptionWithFooterKey::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_STOP) { break; } - xfer += iprot->skip(ftype); + switch (fid) + { + case 1: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readBinary(this->aad_prefix); + this->__isset.aad_prefix = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readBinary(this->aad_file_unique); + this->__isset.aad_file_unique = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 3: + if (ftype == ::apache::thrift::protocol::T_BOOL) { + xfer += iprot->readBool(this->supply_aad_prefix); + this->__isset.supply_aad_prefix = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } xfer += iprot->readFieldEnd(); } @@ -3702,18 +3731,33 @@ uint32_t EncryptionWithFooterKey::read(Protocol_* iprot) { } template -uint32_t EncryptionWithFooterKey::write(Protocol_* oprot) const { +uint32_t AesGcmV1::write(Protocol_* oprot) const { uint32_t xfer = 0; ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); - xfer += oprot->writeStructBegin("EncryptionWithFooterKey"); + xfer += oprot->writeStructBegin("AesGcmV1"); + if (this->__isset.aad_prefix) { + xfer += oprot->writeFieldBegin("aad_prefix", ::apache::thrift::protocol::T_STRING, 1); + xfer += oprot->writeBinary(this->aad_prefix); + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.aad_file_unique) { + xfer += oprot->writeFieldBegin("aad_file_unique", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeBinary(this->aad_file_unique); + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.supply_aad_prefix) { + xfer += oprot->writeFieldBegin("supply_aad_prefix", ::apache::thrift::protocol::T_BOOL, 3); + xfer += oprot->writeBool(this->supply_aad_prefix); + xfer += oprot->writeFieldEnd(); + } xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; } template -uint32_t EncryptionWithColumnKey::read(Protocol_* iprot) { +uint32_t AesGcmCtrV1::read(Protocol_* iprot) { ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); uint32_t xfer = 0; @@ -3725,7 +3769,6 @@ uint32_t EncryptionWithColumnKey::read(Protocol_* iprot) { using ::apache::thrift::protocol::TProtocolException; - bool isset_path_in_schema = false; while (true) { @@ -3736,29 +3779,25 @@ uint32_t EncryptionWithColumnKey::read(Protocol_* iprot) { switch (fid) { case 1: - if (ftype == ::apache::thrift::protocol::T_LIST) { - { - this->path_in_schema.clear(); - uint32_t _size233; - ::apache::thrift::protocol::TType _etype236; - xfer += iprot->readListBegin(_etype236, _size233); - this->path_in_schema.resize(_size233); - uint32_t _i237; - for (_i237 = 0; _i237 < _size233; ++_i237) - { - xfer += iprot->readString(this->path_in_schema[_i237]); - } - xfer += iprot->readListEnd(); - } - isset_path_in_schema = true; + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readBinary(this->aad_prefix); + this->__isset.aad_prefix = true; } else { xfer += iprot->skip(ftype); } break; case 2: if (ftype == ::apache::thrift::protocol::T_STRING) { - xfer += iprot->readBinary(this->key_metadata); - this->__isset.key_metadata = true; + xfer += iprot->readBinary(this->aad_file_unique); + this->__isset.aad_file_unique = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 3: + if (ftype == ::apache::thrift::protocol::T_BOOL) { + xfer += iprot->readBool(this->supply_aad_prefix); + this->__isset.supply_aad_prefix = true; } else { xfer += iprot->skip(ftype); } @@ -3772,41 +3811,77 @@ uint32_t EncryptionWithColumnKey::read(Protocol_* iprot) { xfer += iprot->readStructEnd(); - if (!isset_path_in_schema) - throw TProtocolException(TProtocolException::INVALID_DATA); return xfer; } template -uint32_t EncryptionWithColumnKey::write(Protocol_* oprot) const { +uint32_t AesGcmCtrV1::write(Protocol_* oprot) const { uint32_t xfer = 0; ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); - xfer += oprot->writeStructBegin("EncryptionWithColumnKey"); + xfer += oprot->writeStructBegin("AesGcmCtrV1"); - xfer += oprot->writeFieldBegin("path_in_schema", ::apache::thrift::protocol::T_LIST, 1); + if (this->__isset.aad_prefix) { + xfer += oprot->writeFieldBegin("aad_prefix", ::apache::thrift::protocol::T_STRING, 1); + xfer += oprot->writeBinary(this->aad_prefix); + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.aad_file_unique) { + xfer += oprot->writeFieldBegin("aad_file_unique", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeBinary(this->aad_file_unique); + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.supply_aad_prefix) { + xfer += oprot->writeFieldBegin("supply_aad_prefix", ::apache::thrift::protocol::T_BOOL, 3); + xfer += oprot->writeBool(this->supply_aad_prefix); + xfer += oprot->writeFieldEnd(); + } + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +template +uint32_t ExternalDBPAV1::read(Protocol_* iprot) { + + ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) { - xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->path_in_schema.size())); - std::vector ::const_iterator _iter238; - for (_iter238 = this->path_in_schema.begin(); _iter238 != this->path_in_schema.end(); ++_iter238) - { - xfer += oprot->writeString((*_iter238)); + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; } - xfer += oprot->writeListEnd(); + xfer += iprot->skip(ftype); + xfer += iprot->readFieldEnd(); } - xfer += oprot->writeFieldEnd(); - if (this->__isset.key_metadata) { - xfer += oprot->writeFieldBegin("key_metadata", ::apache::thrift::protocol::T_STRING, 2); - xfer += oprot->writeBinary(this->key_metadata); - xfer += oprot->writeFieldEnd(); - } + xfer += iprot->readStructEnd(); + + return xfer; +} + +template +uint32_t ExternalDBPAV1::write(Protocol_* oprot) const { + uint32_t xfer = 0; + ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); + xfer += oprot->writeStructBegin("ExternalDBPAV1"); + xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; } template -uint32_t ColumnCryptoMetaData::read(Protocol_* iprot) { +uint32_t EncryptionAlgorithm::read(Protocol_* iprot) { ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); uint32_t xfer = 0; @@ -3829,16 +3904,24 @@ uint32_t ColumnCryptoMetaData::read(Protocol_* iprot) { { case 1: if (ftype == ::apache::thrift::protocol::T_STRUCT) { - xfer += this->ENCRYPTION_WITH_FOOTER_KEY.read(iprot); - this->__isset.ENCRYPTION_WITH_FOOTER_KEY = true; + xfer += this->AES_GCM_V1.read(iprot); + this->__isset.AES_GCM_V1 = true; } else { xfer += iprot->skip(ftype); } break; case 2: if (ftype == ::apache::thrift::protocol::T_STRUCT) { - xfer += this->ENCRYPTION_WITH_COLUMN_KEY.read(iprot); - this->__isset.ENCRYPTION_WITH_COLUMN_KEY = true; + xfer += this->AES_GCM_CTR_V1.read(iprot); + this->__isset.AES_GCM_CTR_V1 = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 3: + if (ftype == ::apache::thrift::protocol::T_STRUCT) { + xfer += this->EXTERNAL_DBPA_V1.read(iprot); + this->__isset.EXTERNAL_DBPA_V1 = true; } else { xfer += iprot->skip(ftype); } @@ -3856,19 +3939,24 @@ uint32_t ColumnCryptoMetaData::read(Protocol_* iprot) { } template -uint32_t ColumnCryptoMetaData::write(Protocol_* oprot) const { +uint32_t EncryptionAlgorithm::write(Protocol_* oprot) const { uint32_t xfer = 0; ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); - xfer += oprot->writeStructBegin("ColumnCryptoMetaData"); + xfer += oprot->writeStructBegin("EncryptionAlgorithm"); - if (this->__isset.ENCRYPTION_WITH_FOOTER_KEY) { - xfer += oprot->writeFieldBegin("ENCRYPTION_WITH_FOOTER_KEY", ::apache::thrift::protocol::T_STRUCT, 1); - xfer += this->ENCRYPTION_WITH_FOOTER_KEY.write(oprot); + if (this->__isset.AES_GCM_V1) { + xfer += oprot->writeFieldBegin("AES_GCM_V1", ::apache::thrift::protocol::T_STRUCT, 1); + xfer += this->AES_GCM_V1.write(oprot); xfer += oprot->writeFieldEnd(); } - if (this->__isset.ENCRYPTION_WITH_COLUMN_KEY) { - xfer += oprot->writeFieldBegin("ENCRYPTION_WITH_COLUMN_KEY", ::apache::thrift::protocol::T_STRUCT, 2); - xfer += this->ENCRYPTION_WITH_COLUMN_KEY.write(oprot); + if (this->__isset.AES_GCM_CTR_V1) { + xfer += oprot->writeFieldBegin("AES_GCM_CTR_V1", ::apache::thrift::protocol::T_STRUCT, 2); + xfer += this->AES_GCM_CTR_V1.write(oprot); + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.EXTERNAL_DBPA_V1) { + xfer += oprot->writeFieldBegin("EXTERNAL_DBPA_V1", ::apache::thrift::protocol::T_STRUCT, 3); + xfer += this->EXTERNAL_DBPA_V1.write(oprot); xfer += oprot->writeFieldEnd(); } xfer += oprot->writeFieldStop(); @@ -3877,7 +3965,7 @@ uint32_t ColumnCryptoMetaData::write(Protocol_* oprot) const { } template -uint32_t ColumnChunk::read(Protocol_* iprot) { +uint32_t EncryptionWithFooterKey::read(Protocol_* iprot) { ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); uint32_t xfer = 0; @@ -3889,7 +3977,47 @@ uint32_t ColumnChunk::read(Protocol_* iprot) { using ::apache::thrift::protocol::TProtocolException; - bool isset_file_offset = false; + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + xfer += iprot->skip(ftype); + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +template +uint32_t EncryptionWithFooterKey::write(Protocol_* oprot) const { + uint32_t xfer = 0; + ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); + xfer += oprot->writeStructBegin("EncryptionWithFooterKey"); + + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +template +uint32_t EncryptionWithColumnKey::read(Protocol_* iprot) { + + ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + bool isset_path_in_schema = false; while (true) { @@ -3900,43 +4028,220 @@ uint32_t ColumnChunk::read(Protocol_* iprot) { switch (fid) { case 1: - if (ftype == ::apache::thrift::protocol::T_STRING) { - xfer += iprot->readString(this->file_path); - this->__isset.file_path = true; + if (ftype == ::apache::thrift::protocol::T_LIST) { + { + this->path_in_schema.clear(); + uint32_t _size249; + ::apache::thrift::protocol::TType _etype252; + xfer += iprot->readListBegin(_etype252, _size249); + this->path_in_schema.resize(_size249); + uint32_t _i253; + for (_i253 = 0; _i253 < _size249; ++_i253) + { + xfer += iprot->readString(this->path_in_schema[_i253]); + } + xfer += iprot->readListEnd(); + } + isset_path_in_schema = true; } else { xfer += iprot->skip(ftype); } break; case 2: - if (ftype == ::apache::thrift::protocol::T_I64) { - xfer += iprot->readI64(this->file_offset); - isset_file_offset = true; + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readBinary(this->key_metadata); + this->__isset.key_metadata = true; } else { xfer += iprot->skip(ftype); } break; case 3: if (ftype == ::apache::thrift::protocol::T_STRUCT) { - xfer += this->meta_data.read(iprot); - this->__isset.meta_data = true; + xfer += this->encryption_algorithm.read(iprot); + this->__isset.encryption_algorithm = true; } else { xfer += iprot->skip(ftype); } break; - case 4: - if (ftype == ::apache::thrift::protocol::T_I64) { - xfer += iprot->readI64(this->offset_index_offset); - this->__isset.offset_index_offset = true; - } else { - xfer += iprot->skip(ftype); - } + default: + xfer += iprot->skip(ftype); break; - case 5: - if (ftype == ::apache::thrift::protocol::T_I32) { - xfer += iprot->readI32(this->offset_index_length); - this->__isset.offset_index_length = true; - } else { - xfer += iprot->skip(ftype); + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + if (!isset_path_in_schema) + throw TProtocolException(TProtocolException::INVALID_DATA); + return xfer; +} + +template +uint32_t EncryptionWithColumnKey::write(Protocol_* oprot) const { + uint32_t xfer = 0; + ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); + xfer += oprot->writeStructBegin("EncryptionWithColumnKey"); + + xfer += oprot->writeFieldBegin("path_in_schema", ::apache::thrift::protocol::T_LIST, 1); + { + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->path_in_schema.size())); + std::vector ::const_iterator _iter254; + for (_iter254 = this->path_in_schema.begin(); _iter254 != this->path_in_schema.end(); ++_iter254) + { + xfer += oprot->writeString((*_iter254)); + } + xfer += oprot->writeListEnd(); + } + xfer += oprot->writeFieldEnd(); + + if (this->__isset.key_metadata) { + xfer += oprot->writeFieldBegin("key_metadata", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeBinary(this->key_metadata); + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.encryption_algorithm) { + xfer += oprot->writeFieldBegin("encryption_algorithm", ::apache::thrift::protocol::T_STRUCT, 3); + xfer += this->encryption_algorithm.write(oprot); + xfer += oprot->writeFieldEnd(); + } + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +template +uint32_t ColumnCryptoMetaData::read(Protocol_* iprot) { + + ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == ::apache::thrift::protocol::T_STRUCT) { + xfer += this->ENCRYPTION_WITH_FOOTER_KEY.read(iprot); + this->__isset.ENCRYPTION_WITH_FOOTER_KEY = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == ::apache::thrift::protocol::T_STRUCT) { + xfer += this->ENCRYPTION_WITH_COLUMN_KEY.read(iprot); + this->__isset.ENCRYPTION_WITH_COLUMN_KEY = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +template +uint32_t ColumnCryptoMetaData::write(Protocol_* oprot) const { + uint32_t xfer = 0; + ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); + xfer += oprot->writeStructBegin("ColumnCryptoMetaData"); + + if (this->__isset.ENCRYPTION_WITH_FOOTER_KEY) { + xfer += oprot->writeFieldBegin("ENCRYPTION_WITH_FOOTER_KEY", ::apache::thrift::protocol::T_STRUCT, 1); + xfer += this->ENCRYPTION_WITH_FOOTER_KEY.write(oprot); + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.ENCRYPTION_WITH_COLUMN_KEY) { + xfer += oprot->writeFieldBegin("ENCRYPTION_WITH_COLUMN_KEY", ::apache::thrift::protocol::T_STRUCT, 2); + xfer += this->ENCRYPTION_WITH_COLUMN_KEY.write(oprot); + xfer += oprot->writeFieldEnd(); + } + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +template +uint32_t ColumnChunk::read(Protocol_* iprot) { + + ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + bool isset_file_offset = false; + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readString(this->file_path); + this->__isset.file_path = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == ::apache::thrift::protocol::T_I64) { + xfer += iprot->readI64(this->file_offset); + isset_file_offset = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 3: + if (ftype == ::apache::thrift::protocol::T_STRUCT) { + xfer += this->meta_data.read(iprot); + this->__isset.meta_data = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 4: + if (ftype == ::apache::thrift::protocol::T_I64) { + xfer += iprot->readI64(this->offset_index_offset); + this->__isset.offset_index_offset = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 5: + if (ftype == ::apache::thrift::protocol::T_I32) { + xfer += iprot->readI32(this->offset_index_length); + this->__isset.offset_index_length = true; + } else { + xfer += iprot->skip(ftype); } break; case 6: @@ -4069,14 +4374,14 @@ uint32_t RowGroup::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->columns.clear(); - uint32_t _size251; - ::apache::thrift::protocol::TType _etype254; - xfer += iprot->readListBegin(_etype254, _size251); - this->columns.resize(_size251); - uint32_t _i255; - for (_i255 = 0; _i255 < _size251; ++_i255) + uint32_t _size267; + ::apache::thrift::protocol::TType _etype270; + xfer += iprot->readListBegin(_etype270, _size267); + this->columns.resize(_size267); + uint32_t _i271; + for (_i271 = 0; _i271 < _size267; ++_i271) { - xfer += this->columns[_i255].read(iprot); + xfer += this->columns[_i271].read(iprot); } xfer += iprot->readListEnd(); } @@ -4105,14 +4410,14 @@ uint32_t RowGroup::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->sorting_columns.clear(); - uint32_t _size256; - ::apache::thrift::protocol::TType _etype259; - xfer += iprot->readListBegin(_etype259, _size256); - this->sorting_columns.resize(_size256); - uint32_t _i260; - for (_i260 = 0; _i260 < _size256; ++_i260) + uint32_t _size272; + ::apache::thrift::protocol::TType _etype275; + xfer += iprot->readListBegin(_etype275, _size272); + this->sorting_columns.resize(_size272); + uint32_t _i276; + for (_i276 = 0; _i276 < _size272; ++_i276) { - xfer += this->sorting_columns[_i260].read(iprot); + xfer += this->sorting_columns[_i276].read(iprot); } xfer += iprot->readListEnd(); } @@ -4172,10 +4477,10 @@ uint32_t RowGroup::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("columns", ::apache::thrift::protocol::T_LIST, 1); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->columns.size())); - std::vector ::const_iterator _iter261; - for (_iter261 = this->columns.begin(); _iter261 != this->columns.end(); ++_iter261) + std::vector ::const_iterator _iter277; + for (_iter277 = this->columns.begin(); _iter277 != this->columns.end(); ++_iter277) { - xfer += (*_iter261).write(oprot); + xfer += (*_iter277).write(oprot); } xfer += oprot->writeListEnd(); } @@ -4193,10 +4498,10 @@ uint32_t RowGroup::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("sorting_columns", ::apache::thrift::protocol::T_LIST, 4); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->sorting_columns.size())); - std::vector ::const_iterator _iter262; - for (_iter262 = this->sorting_columns.begin(); _iter262 != this->sorting_columns.end(); ++_iter262) + std::vector ::const_iterator _iter278; + for (_iter278 = this->sorting_columns.begin(); _iter278 != this->sorting_columns.end(); ++_iter278) { - xfer += (*_iter262).write(oprot); + xfer += (*_iter278).write(oprot); } xfer += oprot->writeListEnd(); } @@ -4437,14 +4742,14 @@ uint32_t OffsetIndex::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->page_locations.clear(); - uint32_t _size279; - ::apache::thrift::protocol::TType _etype282; - xfer += iprot->readListBegin(_etype282, _size279); - this->page_locations.resize(_size279); - uint32_t _i283; - for (_i283 = 0; _i283 < _size279; ++_i283) + uint32_t _size295; + ::apache::thrift::protocol::TType _etype298; + xfer += iprot->readListBegin(_etype298, _size295); + this->page_locations.resize(_size295); + uint32_t _i299; + for (_i299 = 0; _i299 < _size295; ++_i299) { - xfer += this->page_locations[_i283].read(iprot); + xfer += this->page_locations[_i299].read(iprot); } xfer += iprot->readListEnd(); } @@ -4457,14 +4762,14 @@ uint32_t OffsetIndex::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->unencoded_byte_array_data_bytes.clear(); - uint32_t _size284; - ::apache::thrift::protocol::TType _etype287; - xfer += iprot->readListBegin(_etype287, _size284); - this->unencoded_byte_array_data_bytes.resize(_size284); - uint32_t _i288; - for (_i288 = 0; _i288 < _size284; ++_i288) + uint32_t _size300; + ::apache::thrift::protocol::TType _etype303; + xfer += iprot->readListBegin(_etype303, _size300); + this->unencoded_byte_array_data_bytes.resize(_size300); + uint32_t _i304; + for (_i304 = 0; _i304 < _size300; ++_i304) { - xfer += iprot->readI64(this->unencoded_byte_array_data_bytes[_i288]); + xfer += iprot->readI64(this->unencoded_byte_array_data_bytes[_i304]); } xfer += iprot->readListEnd(); } @@ -4496,10 +4801,10 @@ uint32_t OffsetIndex::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("page_locations", ::apache::thrift::protocol::T_LIST, 1); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->page_locations.size())); - std::vector ::const_iterator _iter289; - for (_iter289 = this->page_locations.begin(); _iter289 != this->page_locations.end(); ++_iter289) + std::vector ::const_iterator _iter305; + for (_iter305 = this->page_locations.begin(); _iter305 != this->page_locations.end(); ++_iter305) { - xfer += (*_iter289).write(oprot); + xfer += (*_iter305).write(oprot); } xfer += oprot->writeListEnd(); } @@ -4509,10 +4814,10 @@ uint32_t OffsetIndex::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("unencoded_byte_array_data_bytes", ::apache::thrift::protocol::T_LIST, 2); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I64, static_cast(this->unencoded_byte_array_data_bytes.size())); - std::vector ::const_iterator _iter290; - for (_iter290 = this->unencoded_byte_array_data_bytes.begin(); _iter290 != this->unencoded_byte_array_data_bytes.end(); ++_iter290) + std::vector ::const_iterator _iter306; + for (_iter306 = this->unencoded_byte_array_data_bytes.begin(); _iter306 != this->unencoded_byte_array_data_bytes.end(); ++_iter306) { - xfer += oprot->writeI64((*_iter290)); + xfer += oprot->writeI64((*_iter306)); } xfer += oprot->writeListEnd(); } @@ -4553,14 +4858,14 @@ uint32_t ColumnIndex::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->null_pages.clear(); - uint32_t _size295; - ::apache::thrift::protocol::TType _etype298; - xfer += iprot->readListBegin(_etype298, _size295); - this->null_pages.resize(_size295); - uint32_t _i299; - for (_i299 = 0; _i299 < _size295; ++_i299) + uint32_t _size311; + ::apache::thrift::protocol::TType _etype314; + xfer += iprot->readListBegin(_etype314, _size311); + this->null_pages.resize(_size311); + uint32_t _i315; + for (_i315 = 0; _i315 < _size311; ++_i315) { - xfer += iprot->readBool(this->null_pages[_i299]); + xfer += iprot->readBool(this->null_pages[_i315]); } xfer += iprot->readListEnd(); } @@ -4573,14 +4878,14 @@ uint32_t ColumnIndex::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->min_values.clear(); - uint32_t _size300; - ::apache::thrift::protocol::TType _etype303; - xfer += iprot->readListBegin(_etype303, _size300); - this->min_values.resize(_size300); - uint32_t _i304; - for (_i304 = 0; _i304 < _size300; ++_i304) + uint32_t _size316; + ::apache::thrift::protocol::TType _etype319; + xfer += iprot->readListBegin(_etype319, _size316); + this->min_values.resize(_size316); + uint32_t _i320; + for (_i320 = 0; _i320 < _size316; ++_i320) { - xfer += iprot->readBinary(this->min_values[_i304]); + xfer += iprot->readBinary(this->min_values[_i320]); } xfer += iprot->readListEnd(); } @@ -4593,14 +4898,14 @@ uint32_t ColumnIndex::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->max_values.clear(); - uint32_t _size305; - ::apache::thrift::protocol::TType _etype308; - xfer += iprot->readListBegin(_etype308, _size305); - this->max_values.resize(_size305); - uint32_t _i309; - for (_i309 = 0; _i309 < _size305; ++_i309) + uint32_t _size321; + ::apache::thrift::protocol::TType _etype324; + xfer += iprot->readListBegin(_etype324, _size321); + this->max_values.resize(_size321); + uint32_t _i325; + for (_i325 = 0; _i325 < _size321; ++_i325) { - xfer += iprot->readBinary(this->max_values[_i309]); + xfer += iprot->readBinary(this->max_values[_i325]); } xfer += iprot->readListEnd(); } @@ -4611,9 +4916,9 @@ uint32_t ColumnIndex::read(Protocol_* iprot) { break; case 4: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast310; - xfer += iprot->readI32(ecast310); - this->boundary_order = static_cast(ecast310); + int32_t ecast326; + xfer += iprot->readI32(ecast326); + this->boundary_order = static_cast(ecast326); isset_boundary_order = true; } else { xfer += iprot->skip(ftype); @@ -4623,14 +4928,14 @@ uint32_t ColumnIndex::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->null_counts.clear(); - uint32_t _size311; - ::apache::thrift::protocol::TType _etype314; - xfer += iprot->readListBegin(_etype314, _size311); - this->null_counts.resize(_size311); - uint32_t _i315; - for (_i315 = 0; _i315 < _size311; ++_i315) + uint32_t _size327; + ::apache::thrift::protocol::TType _etype330; + xfer += iprot->readListBegin(_etype330, _size327); + this->null_counts.resize(_size327); + uint32_t _i331; + for (_i331 = 0; _i331 < _size327; ++_i331) { - xfer += iprot->readI64(this->null_counts[_i315]); + xfer += iprot->readI64(this->null_counts[_i331]); } xfer += iprot->readListEnd(); } @@ -4643,14 +4948,14 @@ uint32_t ColumnIndex::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->repetition_level_histograms.clear(); - uint32_t _size316; - ::apache::thrift::protocol::TType _etype319; - xfer += iprot->readListBegin(_etype319, _size316); - this->repetition_level_histograms.resize(_size316); - uint32_t _i320; - for (_i320 = 0; _i320 < _size316; ++_i320) + uint32_t _size332; + ::apache::thrift::protocol::TType _etype335; + xfer += iprot->readListBegin(_etype335, _size332); + this->repetition_level_histograms.resize(_size332); + uint32_t _i336; + for (_i336 = 0; _i336 < _size332; ++_i336) { - xfer += iprot->readI64(this->repetition_level_histograms[_i320]); + xfer += iprot->readI64(this->repetition_level_histograms[_i336]); } xfer += iprot->readListEnd(); } @@ -4663,14 +4968,14 @@ uint32_t ColumnIndex::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->definition_level_histograms.clear(); - uint32_t _size321; - ::apache::thrift::protocol::TType _etype324; - xfer += iprot->readListBegin(_etype324, _size321); - this->definition_level_histograms.resize(_size321); - uint32_t _i325; - for (_i325 = 0; _i325 < _size321; ++_i325) + uint32_t _size337; + ::apache::thrift::protocol::TType _etype340; + xfer += iprot->readListBegin(_etype340, _size337); + this->definition_level_histograms.resize(_size337); + uint32_t _i341; + for (_i341 = 0; _i341 < _size337; ++_i341) { - xfer += iprot->readI64(this->definition_level_histograms[_i325]); + xfer += iprot->readI64(this->definition_level_histograms[_i341]); } xfer += iprot->readListEnd(); } @@ -4708,10 +5013,10 @@ uint32_t ColumnIndex::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("null_pages", ::apache::thrift::protocol::T_LIST, 1); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_BOOL, static_cast(this->null_pages.size())); - std::vector ::const_iterator _iter326; - for (_iter326 = this->null_pages.begin(); _iter326 != this->null_pages.end(); ++_iter326) + std::vector ::const_iterator _iter342; + for (_iter342 = this->null_pages.begin(); _iter342 != this->null_pages.end(); ++_iter342) { - xfer += oprot->writeBool((*_iter326)); + xfer += oprot->writeBool((*_iter342)); } xfer += oprot->writeListEnd(); } @@ -4720,10 +5025,10 @@ uint32_t ColumnIndex::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("min_values", ::apache::thrift::protocol::T_LIST, 2); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->min_values.size())); - std::vector ::const_iterator _iter327; - for (_iter327 = this->min_values.begin(); _iter327 != this->min_values.end(); ++_iter327) + std::vector ::const_iterator _iter343; + for (_iter343 = this->min_values.begin(); _iter343 != this->min_values.end(); ++_iter343) { - xfer += oprot->writeBinary((*_iter327)); + xfer += oprot->writeBinary((*_iter343)); } xfer += oprot->writeListEnd(); } @@ -4732,10 +5037,10 @@ uint32_t ColumnIndex::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("max_values", ::apache::thrift::protocol::T_LIST, 3); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->max_values.size())); - std::vector ::const_iterator _iter328; - for (_iter328 = this->max_values.begin(); _iter328 != this->max_values.end(); ++_iter328) + std::vector ::const_iterator _iter344; + for (_iter344 = this->max_values.begin(); _iter344 != this->max_values.end(); ++_iter344) { - xfer += oprot->writeBinary((*_iter328)); + xfer += oprot->writeBinary((*_iter344)); } xfer += oprot->writeListEnd(); } @@ -4749,10 +5054,10 @@ uint32_t ColumnIndex::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("null_counts", ::apache::thrift::protocol::T_LIST, 5); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I64, static_cast(this->null_counts.size())); - std::vector ::const_iterator _iter329; - for (_iter329 = this->null_counts.begin(); _iter329 != this->null_counts.end(); ++_iter329) + std::vector ::const_iterator _iter345; + for (_iter345 = this->null_counts.begin(); _iter345 != this->null_counts.end(); ++_iter345) { - xfer += oprot->writeI64((*_iter329)); + xfer += oprot->writeI64((*_iter345)); } xfer += oprot->writeListEnd(); } @@ -4762,10 +5067,10 @@ uint32_t ColumnIndex::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("repetition_level_histograms", ::apache::thrift::protocol::T_LIST, 6); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I64, static_cast(this->repetition_level_histograms.size())); - std::vector ::const_iterator _iter330; - for (_iter330 = this->repetition_level_histograms.begin(); _iter330 != this->repetition_level_histograms.end(); ++_iter330) + std::vector ::const_iterator _iter346; + for (_iter346 = this->repetition_level_histograms.begin(); _iter346 != this->repetition_level_histograms.end(); ++_iter346) { - xfer += oprot->writeI64((*_iter330)); + xfer += oprot->writeI64((*_iter346)); } xfer += oprot->writeListEnd(); } @@ -4775,10 +5080,10 @@ uint32_t ColumnIndex::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("definition_level_histograms", ::apache::thrift::protocol::T_LIST, 7); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I64, static_cast(this->definition_level_histograms.size())); - std::vector ::const_iterator _iter331; - for (_iter331 = this->definition_level_histograms.begin(); _iter331 != this->definition_level_histograms.end(); ++_iter331) + std::vector ::const_iterator _iter347; + for (_iter347 = this->definition_level_histograms.begin(); _iter347 != this->definition_level_histograms.end(); ++_iter347) { - xfer += oprot->writeI64((*_iter331)); + xfer += oprot->writeI64((*_iter347)); } xfer += oprot->writeListEnd(); } @@ -4789,245 +5094,6 @@ uint32_t ColumnIndex::write(Protocol_* oprot) const { return xfer; } -template -uint32_t AesGcmV1::read(Protocol_* iprot) { - - ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); - uint32_t xfer = 0; - std::string fname; - ::apache::thrift::protocol::TType ftype; - int16_t fid; - - xfer += iprot->readStructBegin(fname); - - using ::apache::thrift::protocol::TProtocolException; - - - while (true) - { - xfer += iprot->readFieldBegin(fname, ftype, fid); - if (ftype == ::apache::thrift::protocol::T_STOP) { - break; - } - switch (fid) - { - case 1: - if (ftype == ::apache::thrift::protocol::T_STRING) { - xfer += iprot->readBinary(this->aad_prefix); - this->__isset.aad_prefix = true; - } else { - xfer += iprot->skip(ftype); - } - break; - case 2: - if (ftype == ::apache::thrift::protocol::T_STRING) { - xfer += iprot->readBinary(this->aad_file_unique); - this->__isset.aad_file_unique = true; - } else { - xfer += iprot->skip(ftype); - } - break; - case 3: - if (ftype == ::apache::thrift::protocol::T_BOOL) { - xfer += iprot->readBool(this->supply_aad_prefix); - this->__isset.supply_aad_prefix = true; - } else { - xfer += iprot->skip(ftype); - } - break; - default: - xfer += iprot->skip(ftype); - break; - } - xfer += iprot->readFieldEnd(); - } - - xfer += iprot->readStructEnd(); - - return xfer; -} - -template -uint32_t AesGcmV1::write(Protocol_* oprot) const { - uint32_t xfer = 0; - ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); - xfer += oprot->writeStructBegin("AesGcmV1"); - - if (this->__isset.aad_prefix) { - xfer += oprot->writeFieldBegin("aad_prefix", ::apache::thrift::protocol::T_STRING, 1); - xfer += oprot->writeBinary(this->aad_prefix); - xfer += oprot->writeFieldEnd(); - } - if (this->__isset.aad_file_unique) { - xfer += oprot->writeFieldBegin("aad_file_unique", ::apache::thrift::protocol::T_STRING, 2); - xfer += oprot->writeBinary(this->aad_file_unique); - xfer += oprot->writeFieldEnd(); - } - if (this->__isset.supply_aad_prefix) { - xfer += oprot->writeFieldBegin("supply_aad_prefix", ::apache::thrift::protocol::T_BOOL, 3); - xfer += oprot->writeBool(this->supply_aad_prefix); - xfer += oprot->writeFieldEnd(); - } - xfer += oprot->writeFieldStop(); - xfer += oprot->writeStructEnd(); - return xfer; -} - -template -uint32_t AesGcmCtrV1::read(Protocol_* iprot) { - - ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); - uint32_t xfer = 0; - std::string fname; - ::apache::thrift::protocol::TType ftype; - int16_t fid; - - xfer += iprot->readStructBegin(fname); - - using ::apache::thrift::protocol::TProtocolException; - - - while (true) - { - xfer += iprot->readFieldBegin(fname, ftype, fid); - if (ftype == ::apache::thrift::protocol::T_STOP) { - break; - } - switch (fid) - { - case 1: - if (ftype == ::apache::thrift::protocol::T_STRING) { - xfer += iprot->readBinary(this->aad_prefix); - this->__isset.aad_prefix = true; - } else { - xfer += iprot->skip(ftype); - } - break; - case 2: - if (ftype == ::apache::thrift::protocol::T_STRING) { - xfer += iprot->readBinary(this->aad_file_unique); - this->__isset.aad_file_unique = true; - } else { - xfer += iprot->skip(ftype); - } - break; - case 3: - if (ftype == ::apache::thrift::protocol::T_BOOL) { - xfer += iprot->readBool(this->supply_aad_prefix); - this->__isset.supply_aad_prefix = true; - } else { - xfer += iprot->skip(ftype); - } - break; - default: - xfer += iprot->skip(ftype); - break; - } - xfer += iprot->readFieldEnd(); - } - - xfer += iprot->readStructEnd(); - - return xfer; -} - -template -uint32_t AesGcmCtrV1::write(Protocol_* oprot) const { - uint32_t xfer = 0; - ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); - xfer += oprot->writeStructBegin("AesGcmCtrV1"); - - if (this->__isset.aad_prefix) { - xfer += oprot->writeFieldBegin("aad_prefix", ::apache::thrift::protocol::T_STRING, 1); - xfer += oprot->writeBinary(this->aad_prefix); - xfer += oprot->writeFieldEnd(); - } - if (this->__isset.aad_file_unique) { - xfer += oprot->writeFieldBegin("aad_file_unique", ::apache::thrift::protocol::T_STRING, 2); - xfer += oprot->writeBinary(this->aad_file_unique); - xfer += oprot->writeFieldEnd(); - } - if (this->__isset.supply_aad_prefix) { - xfer += oprot->writeFieldBegin("supply_aad_prefix", ::apache::thrift::protocol::T_BOOL, 3); - xfer += oprot->writeBool(this->supply_aad_prefix); - xfer += oprot->writeFieldEnd(); - } - xfer += oprot->writeFieldStop(); - xfer += oprot->writeStructEnd(); - return xfer; -} - -template -uint32_t EncryptionAlgorithm::read(Protocol_* iprot) { - - ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); - uint32_t xfer = 0; - std::string fname; - ::apache::thrift::protocol::TType ftype; - int16_t fid; - - xfer += iprot->readStructBegin(fname); - - using ::apache::thrift::protocol::TProtocolException; - - - while (true) - { - xfer += iprot->readFieldBegin(fname, ftype, fid); - if (ftype == ::apache::thrift::protocol::T_STOP) { - break; - } - switch (fid) - { - case 1: - if (ftype == ::apache::thrift::protocol::T_STRUCT) { - xfer += this->AES_GCM_V1.read(iprot); - this->__isset.AES_GCM_V1 = true; - } else { - xfer += iprot->skip(ftype); - } - break; - case 2: - if (ftype == ::apache::thrift::protocol::T_STRUCT) { - xfer += this->AES_GCM_CTR_V1.read(iprot); - this->__isset.AES_GCM_CTR_V1 = true; - } else { - xfer += iprot->skip(ftype); - } - break; - default: - xfer += iprot->skip(ftype); - break; - } - xfer += iprot->readFieldEnd(); - } - - xfer += iprot->readStructEnd(); - - return xfer; -} - -template -uint32_t EncryptionAlgorithm::write(Protocol_* oprot) const { - uint32_t xfer = 0; - ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); - xfer += oprot->writeStructBegin("EncryptionAlgorithm"); - - if (this->__isset.AES_GCM_V1) { - xfer += oprot->writeFieldBegin("AES_GCM_V1", ::apache::thrift::protocol::T_STRUCT, 1); - xfer += this->AES_GCM_V1.write(oprot); - xfer += oprot->writeFieldEnd(); - } - if (this->__isset.AES_GCM_CTR_V1) { - xfer += oprot->writeFieldBegin("AES_GCM_CTR_V1", ::apache::thrift::protocol::T_STRUCT, 2); - xfer += this->AES_GCM_CTR_V1.write(oprot); - xfer += oprot->writeFieldEnd(); - } - xfer += oprot->writeFieldStop(); - xfer += oprot->writeStructEnd(); - return xfer; -} - template uint32_t FileMetaData::read(Protocol_* iprot) { @@ -5066,14 +5132,14 @@ uint32_t FileMetaData::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->schema.clear(); - uint32_t _size348; - ::apache::thrift::protocol::TType _etype351; - xfer += iprot->readListBegin(_etype351, _size348); - this->schema.resize(_size348); - uint32_t _i352; - for (_i352 = 0; _i352 < _size348; ++_i352) + uint32_t _size352; + ::apache::thrift::protocol::TType _etype355; + xfer += iprot->readListBegin(_etype355, _size352); + this->schema.resize(_size352); + uint32_t _i356; + for (_i356 = 0; _i356 < _size352; ++_i356) { - xfer += this->schema[_i352].read(iprot); + xfer += this->schema[_i356].read(iprot); } xfer += iprot->readListEnd(); } @@ -5094,14 +5160,14 @@ uint32_t FileMetaData::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->row_groups.clear(); - uint32_t _size353; - ::apache::thrift::protocol::TType _etype356; - xfer += iprot->readListBegin(_etype356, _size353); - this->row_groups.resize(_size353); - uint32_t _i357; - for (_i357 = 0; _i357 < _size353; ++_i357) + uint32_t _size357; + ::apache::thrift::protocol::TType _etype360; + xfer += iprot->readListBegin(_etype360, _size357); + this->row_groups.resize(_size357); + uint32_t _i361; + for (_i361 = 0; _i361 < _size357; ++_i361) { - xfer += this->row_groups[_i357].read(iprot); + xfer += this->row_groups[_i361].read(iprot); } xfer += iprot->readListEnd(); } @@ -5114,14 +5180,14 @@ uint32_t FileMetaData::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->key_value_metadata.clear(); - uint32_t _size358; - ::apache::thrift::protocol::TType _etype361; - xfer += iprot->readListBegin(_etype361, _size358); - this->key_value_metadata.resize(_size358); - uint32_t _i362; - for (_i362 = 0; _i362 < _size358; ++_i362) + uint32_t _size362; + ::apache::thrift::protocol::TType _etype365; + xfer += iprot->readListBegin(_etype365, _size362); + this->key_value_metadata.resize(_size362); + uint32_t _i366; + for (_i366 = 0; _i366 < _size362; ++_i366) { - xfer += this->key_value_metadata[_i362].read(iprot); + xfer += this->key_value_metadata[_i366].read(iprot); } xfer += iprot->readListEnd(); } @@ -5142,14 +5208,14 @@ uint32_t FileMetaData::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->column_orders.clear(); - uint32_t _size363; - ::apache::thrift::protocol::TType _etype366; - xfer += iprot->readListBegin(_etype366, _size363); - this->column_orders.resize(_size363); - uint32_t _i367; - for (_i367 = 0; _i367 < _size363; ++_i367) + uint32_t _size367; + ::apache::thrift::protocol::TType _etype370; + xfer += iprot->readListBegin(_etype370, _size367); + this->column_orders.resize(_size367); + uint32_t _i371; + for (_i371 = 0; _i371 < _size367; ++_i371) { - xfer += this->column_orders[_i367].read(iprot); + xfer += this->column_orders[_i371].read(iprot); } xfer += iprot->readListEnd(); } @@ -5207,10 +5273,10 @@ uint32_t FileMetaData::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("schema", ::apache::thrift::protocol::T_LIST, 2); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->schema.size())); - std::vector ::const_iterator _iter368; - for (_iter368 = this->schema.begin(); _iter368 != this->schema.end(); ++_iter368) + std::vector ::const_iterator _iter372; + for (_iter372 = this->schema.begin(); _iter372 != this->schema.end(); ++_iter372) { - xfer += (*_iter368).write(oprot); + xfer += (*_iter372).write(oprot); } xfer += oprot->writeListEnd(); } @@ -5223,10 +5289,10 @@ uint32_t FileMetaData::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("row_groups", ::apache::thrift::protocol::T_LIST, 4); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->row_groups.size())); - std::vector ::const_iterator _iter369; - for (_iter369 = this->row_groups.begin(); _iter369 != this->row_groups.end(); ++_iter369) + std::vector ::const_iterator _iter373; + for (_iter373 = this->row_groups.begin(); _iter373 != this->row_groups.end(); ++_iter373) { - xfer += (*_iter369).write(oprot); + xfer += (*_iter373).write(oprot); } xfer += oprot->writeListEnd(); } @@ -5236,10 +5302,10 @@ uint32_t FileMetaData::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("key_value_metadata", ::apache::thrift::protocol::T_LIST, 5); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->key_value_metadata.size())); - std::vector ::const_iterator _iter370; - for (_iter370 = this->key_value_metadata.begin(); _iter370 != this->key_value_metadata.end(); ++_iter370) + std::vector ::const_iterator _iter374; + for (_iter374 = this->key_value_metadata.begin(); _iter374 != this->key_value_metadata.end(); ++_iter374) { - xfer += (*_iter370).write(oprot); + xfer += (*_iter374).write(oprot); } xfer += oprot->writeListEnd(); } @@ -5254,10 +5320,10 @@ uint32_t FileMetaData::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("column_orders", ::apache::thrift::protocol::T_LIST, 7); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->column_orders.size())); - std::vector ::const_iterator _iter371; - for (_iter371 = this->column_orders.begin(); _iter371 != this->column_orders.end(); ++_iter371) + std::vector ::const_iterator _iter375; + for (_iter375 = this->column_orders.begin(); _iter375 != this->column_orders.end(); ++_iter375) { - xfer += (*_iter371).write(oprot); + xfer += (*_iter375).write(oprot); } xfer += oprot->writeListEnd(); } diff --git a/cpp/src/parquet/CMakeLists.txt b/cpp/src/parquet/CMakeLists.txt index dc7d40d2a386..1faa96d291eb 100644 --- a/cpp/src/parquet/CMakeLists.txt +++ b/cpp/src/parquet/CMakeLists.txt @@ -239,8 +239,102 @@ endif() if(PARQUET_REQUIRE_ENCRYPTION) list(APPEND PARQUET_SHARED_PRIVATE_LINK_LIBS ${ARROW_OPENSSL_LIBS}) - set(PARQUET_SRCS ${PARQUET_SRCS} encryption/encryption_internal.cc - encryption/openssl_internal.cc) + set(PARQUET_SRCS ${PARQUET_SRCS} encryption/aes_encryption.cc + encryption/openssl_internal.cc + encryption/encryption_utils.cc + encryption/external_dbpa_encryption.cc + encryption/external/dbpa_library_wrapper.cc + encryption/encoding_properties.cc + encryption/external/dbpa_enum_utils.cc + encryption/external/dbpa_executor.cc + encryption/external/loadable_encryptor_utils.cc + ) + + # Header-only dependencies for external DBPA code + include(FetchContent) + + # magic_enum + FetchContent_Declare( + magic_enum_upstream + GIT_REPOSITORY https://github.com/Neargye/magic_enum.git + GIT_TAG v0.9.7 + ) + FetchContent_MakeAvailable(magic_enum_upstream) + add_library(magic_enum_header_only INTERFACE) + target_include_directories(magic_enum_header_only INTERFACE ${magic_enum_upstream_SOURCE_DIR}/include) + + # tcb/span + FetchContent_Declare( + tcb_span_upstream + GIT_REPOSITORY https://github.com/tcbrindle/span.git + GIT_TAG master + ) + + # Populate sources only; do not add subproject/tests + FetchContent_GetProperties(tcb_span_upstream) + if(NOT tcb_span_upstream_POPULATED) + FetchContent_Populate(tcb_span_upstream) + endif() + add_library(tcb_span INTERFACE) + target_include_directories(tcb_span INTERFACE ${tcb_span_upstream_SOURCE_DIR}/include) + + # DBPS interface (header-only) + FetchContent_Declare( + dbps_agent + GIT_REPOSITORY https://github.com/protegrity/DataBatchProtectionService.git + + #TODO: Change to a specific tag/commit when we have one. + #https://github.com/protegrity/arrow/issues/179 + GIT_TAG 4c808b2233ed0bc04529c3b0dbf7c214c4901043 + GIT_SHALLOW FALSE + ) + + FetchContent_GetProperties(dbps_agent) + if(NOT dbps_agent_POPULATED) + FetchContent_Populate(dbps_agent) + endif() + add_library(dbps_interface INTERFACE) + # Expose dbpa_interface.h and friends from DBPS + target_include_directories(dbps_interface INTERFACE + ${dbps_agent_SOURCE_DIR}/src/common) + + # Allows to disable building DBPS shared libraries + option(PARQUET_BUILD_DBPS_LIBS "Build DBPS external libraries" ON) + + if(PARQUET_BUILD_DBPS_LIBS) + include(ExternalProject) + + # Allow callers to inject additional CMake args for DBPS (e.g., BOOST_ROOT) + set(PARQUET_DBPS_CMAKE_ARGS "${PARQUET_DBPS_CMAKE_ARGS}" CACHE STRING "Extra CMake args for DBPS ExternalProject") + + #builds DBPS shared libraries alongside Parquet + #same OS, same architecture, same compiler, same build type, same CMake args + ExternalProject_Add(dbps_external + DOWNLOAD_COMMAND "" # disables download, assumes (correctly) that sources are already fetched + SOURCE_DIR ${dbps_agent_SOURCE_DIR} + BINARY_DIR ${dbps_agent_BINARY_DIR} + CMAKE_ARGS + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DBUILD_SHARED_LIBS=ON + -DBUILD_TESTING=OFF + + # Place DBPS outputs next to Arrow artifacts + -DCMAKE_LIBRARY_OUTPUT_DIRECTORY=${BUILD_OUTPUT_ROOT_DIRECTORY} + -DCMAKE_RUNTIME_OUTPUT_DIRECTORY=${BUILD_OUTPUT_ROOT_DIRECTORY} + -DCMAKE_ARCHIVE_OUTPUT_DIRECTORY=${BUILD_OUTPUT_ROOT_DIRECTORY} + ${PARQUET_DBPS_CMAKE_ARGS} + INSTALL_COMMAND "" #disables install - artifacts already copied to BUILD_OUTPUT_ROOT_DIRECTORY + ) + endif() + + list(APPEND PARQUET_SHARED_PRIVATE_LINK_LIBS magic_enum_header_only tcb_span) + list(APPEND PARQUET_STATIC_LINK_LIBS magic_enum_header_only tcb_span) + if(ARROW_TESTING) + list(APPEND PARQUET_TEST_LINK_LIBS magic_enum_header_only tcb_span dbps_interface) + endif() + list(APPEND PARQUET_SHARED_PRIVATE_LINK_LIBS dbps_interface) + list(APPEND PARQUET_STATIC_LINK_LIBS dbps_interface) + # Encryption key management set(PARQUET_SRCS ${PARQUET_SRCS} @@ -255,7 +349,7 @@ if(PARQUET_REQUIRE_ENCRYPTION) encryption/key_toolkit_internal.cc encryption/local_wrap_kms_client.cc) else() - set(PARQUET_SRCS ${PARQUET_SRCS} encryption/encryption_internal_nossl.cc) + set(PARQUET_SRCS ${PARQUET_SRCS} encryption/aes_encryption_nossl.cc) endif() list(APPEND PARQUET_SHARED_LINK_LIBS arrow_shared) @@ -306,6 +400,13 @@ add_arrow_lib(parquet STATIC_INSTALL_INTERFACE_LIBS ${PARQUET_STATIC_INSTALL_INTERFACE_LIBS}) +# Ensure DBPS builds when Parquet builds, if requested +if(PARQUET_REQUIRE_ENCRYPTION AND PARQUET_BUILD_DBPS_LIBS) + foreach(LIB_TARGET ${PARQUET_LIBRARIES}) + add_dependencies(${LIB_TARGET} dbps_external) + endforeach() +endif() + if(WIN32 AND NOT (ARROW_TEST_LINKAGE STREQUAL "static")) add_library(parquet_test_support STATIC "${PARQUET_THRIFT_SOURCE_DIR}/parquet_types.cpp") @@ -397,6 +498,7 @@ add_parquet_test(reader-test add_parquet_test(writer-test SOURCES column_writer_test.cc + encryption/external/test_utils.cc file_serialize_test.cc stream_writer_test.cc) @@ -406,7 +508,8 @@ add_parquet_test(arrow-reader-writer-test SOURCES arrow/arrow_reader_writer_test.cc arrow/arrow_statistics_test.cc - arrow/variant_test.cc) + arrow/variant_test.cc + $<$:encryption/external/test_utils.cc>) add_parquet_test(arrow-internals-test SOURCES arrow/path_internal_test.cc arrow/reconstruct_internal_test.cc) @@ -417,11 +520,16 @@ add_parquet_test(arrow-metadata-test SOURCES arrow/arrow_metadata_test.cc if(PARQUET_REQUIRE_ENCRYPTION) add_parquet_test(encryption-test SOURCES - encryption/encryption_internal_test.cc + encryption/aes_encryption_test.cc + encryption/encoding_properties_test.cc + encryption/external_dbpa_encryption_test.cc encryption/write_configurations_test.cc encryption/read_configurations_test.cc + encryption/external/test_utils.cc encryption/properties_test.cc - encryption/test_encryption_util.cc) + encryption/crypto_factory_test.cc + encryption/test_encryption_util.cc + encryption/test_in_memory_kms.cc) add_parquet_test(encryption-key-management-test SOURCES encryption/key_management_test.cc @@ -434,7 +542,10 @@ endif() # Those tests need to use static linking as they access thrift-generated # symbols which are not exported by parquet.dll on Windows (PARQUET-1420). -add_parquet_test(file_deserialize_test SOURCES file_deserialize_test.cc) +add_parquet_test(file_deserialize_test + SOURCES + file_deserialize_test.cc + encryption/external/test_utils.cc) add_parquet_test(schema_test) add_parquet_benchmark(bloom_filter_benchmark SOURCES bloom_filter_benchmark.cc diff --git a/cpp/src/parquet/arrow/arrow_reader_writer_test.cc b/cpp/src/parquet/arrow/arrow_reader_writer_test.cc index e081b428e24f..8e2e99303f46 100644 --- a/cpp/src/parquet/arrow/arrow_reader_writer_test.cc +++ b/cpp/src/parquet/arrow/arrow_reader_writer_test.cc @@ -56,6 +56,7 @@ #include "arrow/util/key_value_metadata.h" #include "arrow/util/logging_internal.h" #include "arrow/util/range.h" +#include "arrow/util/secure_string.h" #ifdef ARROW_CSV # include "arrow/csv/api.h" @@ -75,6 +76,10 @@ #include "parquet/properties.h" #include "parquet/test_util.h" +#ifdef PARQUET_REQUIRE_ENCRYPTION +#include "parquet/encryption/external/test_utils.h" +#endif + using arrow::Array; using arrow::ArrayData; using arrow::ArrayFromJSON; @@ -1960,6 +1965,70 @@ TEST(TestArrowReadWrite, UseDeprecatedInt96) { ASSERT_NO_FATAL_FAILURE(::arrow::AssertTablesEqual(*ex_result, *result)); } +#ifdef PARQUET_REQUIRE_ENCRYPTION +TEST(ExternalDbpaConcurrencyTest, FailsWhenUseThreadsTrue) { + std::shared_ptr<::arrow::Array> arr; + ::arrow::Int32Builder b; + ASSERT_OK(b.AppendValues({1, 2, 3})); + ASSERT_OK(b.Finish(&arr)); + auto schema = ::arrow::schema({::arrow::field("f0", ::arrow::int32())}); + auto table = ::arrow::Table::Make(schema, {arr}); + + ::arrow::util::SecureString column_key(std::string("key1234567890123")); + ::arrow::util::SecureString footer_key(std::string("footer_key123456")); + + std::map> enc_cols; + parquet::ColumnEncryptionProperties::Builder col_builder("f0"); + col_builder.parquet_cipher(parquet::ParquetCipher::EXTERNAL_DBPA_V1); + col_builder.key_id("key1234567890123"); + col_builder.key(column_key); + enc_cols["f0"] = col_builder.build(); + + const std::string lib_path = + parquet::encryption::external::test::TestUtils::GetTestLibraryPath(); + + parquet::ExternalFileEncryptionProperties::Builder fep_builder(footer_key); + fep_builder.footer_key_metadata("kf") + ->encrypted_columns(enc_cols) + ->algorithm(parquet::ParquetCipher::AES_GCM_V1) + ->connection_config({{parquet::ParquetCipher::EXTERNAL_DBPA_V1, + {{"agent_library_path", lib_path}, + {"file_path", "/tmp/test"}}}}); + + auto writer_props = + parquet::WriterProperties::Builder().encryption(fep_builder.build_external())->build(); + + ASSERT_OK_AND_ASSIGN( + auto sink, ::arrow::io::BufferOutputStream::Create(1 << 16, default_memory_pool())); + ASSERT_OK(parquet::arrow::WriteTable(*table, default_memory_pool(), sink, /*chunk_size=*/1024, + writer_props)); + ASSERT_OK_AND_ASSIGN(auto buffer, sink->Finish()); + + auto kr = std::make_shared(); + kr->PutKey("kf", footer_key); + kr->PutKey("key1234567890123", column_key); + + parquet::ExternalFileDecryptionProperties::Builder dep_builder; + dep_builder.key_retriever(kr) + ->app_context("{}") + ->connection_config({{parquet::ParquetCipher::EXTERNAL_DBPA_V1, + {{"agent_library_path", lib_path}, + {"file_path", "/tmp/test"}}}}); + parquet::ReaderProperties rp = parquet::default_reader_properties(); + rp.file_decryption_properties(dep_builder.build_external()); + + ArrowReaderProperties arp; + arp.set_use_threads(true); + + parquet::arrow::FileReaderBuilder frb; + ASSERT_OK(frb.Open(std::make_shared(buffer), rp)); + frb.properties(arp); + + std::unique_ptr fr; + ASSERT_RAISES(Invalid, frb.Build(&fr)); +} +#endif + TEST(TestArrowReadWrite, DownsampleDeprecatedInt96) { using ::arrow::ArrayFromJSON; using ::arrow::field; diff --git a/cpp/src/parquet/arrow/reader.cc b/cpp/src/parquet/arrow/reader.cc index d42fdc5034ab..d9a810de4b80 100644 --- a/cpp/src/parquet/arrow/reader.cc +++ b/cpp/src/parquet/arrow/reader.cc @@ -131,6 +131,23 @@ std::shared_ptr> VectorToSharedSet( return result; } +bool IsExternalDBPAEncryptionUsedInColumns(std::shared_ptr metadata) { + for (int row_group = 0; row_group < metadata->num_row_groups(); row_group++) { + auto row_group_metadata = metadata->RowGroup(row_group); + for (int column = 0; column < row_group_metadata->num_columns(); column++) { + auto column_metadata = row_group_metadata->ColumnChunk(column); + if (column_metadata->crypto_metadata()) { + auto crypto_metadata = column_metadata->crypto_metadata(); + if (crypto_metadata->is_encryption_algorithm_set() && + crypto_metadata->encryption_algorithm().algorithm == ParquetCipher::EXTERNAL_DBPA_V1) { + return true; + } + } + } + } + return false; +} + // Forward declaration Status GetReader(const SchemaField& field, const std::shared_ptr& context, std::unique_ptr* out); @@ -147,6 +164,14 @@ class FileReaderImpl : public FileReader { reader_properties_(std::move(properties)) {} Status Init() { + // If the file is encrypted using EXTERNAL_DBPA_V1 on any of its columns, then it is not safe + // to use multiple threads to read the file. + if (reader_properties_.use_threads()) { + auto metadata = reader_->metadata(); + if (IsExternalDBPAEncryptionUsedInColumns(metadata)) { + return Status::Invalid("EXTERNAL_DBPA_V1 encryption does not support multiple threads"); + } + } return SchemaManifest::Make(reader_->metadata()->schema(), reader_->metadata()->key_value_metadata(), reader_properties_, &manifest_); diff --git a/cpp/src/parquet/column_reader.cc b/cpp/src/parquet/column_reader.cc index 9c314cf81817..2cb40e2594d4 100644 --- a/cpp/src/parquet/column_reader.cc +++ b/cpp/src/parquet/column_reader.cc @@ -47,7 +47,7 @@ #include "arrow/util/unreachable.h" #include "parquet/column_page.h" #include "parquet/encoding.h" -#include "parquet/encryption/encryption_internal.h" +#include "parquet/encryption/encryption_utils.h" #include "parquet/encryption/internal_file_decryptor.h" #include "parquet/exception.h" #include "parquet/level_comparison.h" @@ -57,6 +57,11 @@ #include "parquet/thrift_internal.h" // IWYU pragma: keep #include "parquet/windows_fixup.h" // for OPTIONAL +#include "parquet/encryption/encoding_properties.h" + +using parquet::encryption::EncodingProperties; +using parquet::encryption::EncodingPropertiesBuilder; + using arrow::MemoryPool; using arrow::internal::AddWithOverflow; using arrow::internal::checked_cast; @@ -68,6 +73,31 @@ namespace parquet { namespace { +static parquet::Encoding::type ToParquetEncoding(::parquet::format::Encoding::type format_encoding) { + switch (format_encoding) { + case ::parquet::format::Encoding::PLAIN: + return parquet::Encoding::PLAIN; + case ::parquet::format::Encoding::PLAIN_DICTIONARY: + return parquet::Encoding::PLAIN_DICTIONARY; + case ::parquet::format::Encoding::RLE: + return parquet::Encoding::RLE; + case ::parquet::format::Encoding::BIT_PACKED: + return parquet::Encoding::BIT_PACKED; + case ::parquet::format::Encoding::DELTA_BINARY_PACKED: + return parquet::Encoding::DELTA_BINARY_PACKED; + case ::parquet::format::Encoding::DELTA_LENGTH_BYTE_ARRAY: + return parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY; + case ::parquet::format::Encoding::DELTA_BYTE_ARRAY: + return parquet::Encoding::DELTA_BYTE_ARRAY; + case ::parquet::format::Encoding::RLE_DICTIONARY: + return parquet::Encoding::RLE_DICTIONARY; + case ::parquet::format::Encoding::BYTE_STREAM_SPLIT: + return parquet::Encoding::BYTE_STREAM_SPLIT; + default: + throw ParquetException("ToParquetEncoding: Invalid encoding: " + std::to_string(format_encoding)); + } +} + // The minimum number of repetition/definition levels to decode at a time, for // better vectorized performance when doing many smaller record reads constexpr int64_t kMinLevelBatchSize = 1024; @@ -246,6 +276,8 @@ class SerializedPageReader : public PageReader { void set_max_page_header_size(uint32_t size) override { max_page_header_size_ = size; } + std::unique_ptr GetEncodingProperties(format::PageHeader& page_header); + private: void UpdateDecryption(Decryptor* decryptor, int8_t module_type, std::string* page_aad); @@ -388,6 +420,64 @@ bool SerializedPageReader::ShouldSkipPage(EncodedStatistics* data_page_statistic return false; } +// While ideally we would have written the builder code for PageHeader-based properties within +// encoding_properties.cc, Arrow frowns upon including thrift headers in the public API, and the +// types used in this function are not defined in the public API. This is verified via unit tests. +// Therefore, we have to define this function here (as opposed to **any** .h file). +std::unique_ptr SerializedPageReader::GetEncodingProperties( + format::PageHeader& page_header) { + EncodingPropertiesBuilder builder; + + format::PageType::type page_type_from_header = page_header.type; + + if (page_type_from_header == format::PageType::DICTIONARY_PAGE) { + format::DictionaryPageHeader dictionary_page_header = page_header.dictionary_page_header; + + builder.PageType(parquet::PageType::type::DICTIONARY_PAGE); + builder.PageEncoding(ToParquetEncoding(dictionary_page_header.encoding)); + } + else if (page_type_from_header == format::PageType::DATA_PAGE) { // this is DataPageV1 + format::DataPageHeader data_page_header = page_header.data_page_header; + + builder.PageType(parquet::PageType::type::DATA_PAGE); + builder.PageEncoding(ToParquetEncoding(data_page_header.encoding)); + builder.DataPageNumValues(data_page_header.num_values); + builder.PageV1DefinitionLevelEncoding( + ToParquetEncoding(data_page_header.definition_level_encoding)); + builder.PageV1RepetitionLevelEncoding( + ToParquetEncoding(data_page_header.repetition_level_encoding)); + + if (crypto_ctx_.column_descriptor) { + builder.DataPageMaxDefinitionLevel(crypto_ctx_.column_descriptor->max_definition_level()); + builder.DataPageMaxRepetitionLevel(crypto_ctx_.column_descriptor->max_repetition_level()); + } + else { + //TODO (argmarco): handle this case. + } + } + else if (page_type_from_header == format::PageType::DATA_PAGE_V2) { + format::DataPageHeaderV2 data_page_header_v2 = page_header.data_page_header_v2; + + builder.PageType(parquet::PageType::type::DATA_PAGE_V2); + builder.PageEncoding(ToParquetEncoding(data_page_header_v2.encoding)); + builder.DataPageNumValues(data_page_header_v2.num_values); + builder.PageV2NumNulls(data_page_header_v2.num_nulls); + builder.PageV2DefinitionLevelsByteLength(data_page_header_v2.definition_levels_byte_length); + builder.PageV2RepetitionLevelsByteLength(data_page_header_v2.repetition_levels_byte_length); + builder.PageV2IsCompressed(data_page_header_v2.is_compressed); + + if (crypto_ctx_.column_descriptor) { + builder.DataPageMaxDefinitionLevel(crypto_ctx_.column_descriptor->max_definition_level()); + builder.DataPageMaxRepetitionLevel(crypto_ctx_.column_descriptor->max_repetition_level()); + } + else { + //TODO (argmarco): handle this case. + } + } + + return builder.Build(); +} //SerializedPageReader::GetEncodingProperties + std::shared_ptr SerializedPageReader::NextPage() { ThriftDeserializer deserializer(properties_); @@ -474,10 +564,23 @@ std::shared_ptr SerializedPageReader::NextPage() { // Decrypt it if we need to if (data_decryptor_ != nullptr) { - auto decryption_buffer = AllocateBuffer( + std::unique_ptr encoding_properties = GetEncodingProperties( + current_page_header_); + + data_decryptor_->UpdateEncodingProperties(std::move(encoding_properties)); + + std::shared_ptr decryption_buffer; + if (data_decryptor_->CanCalculateLengths()) { + decryption_buffer = AllocateBuffer( properties_.memory_pool(), data_decryptor_->PlaintextLength(compressed_len)); - compressed_len = data_decryptor_->Decrypt( + compressed_len = data_decryptor_->Decrypt( page_buffer->span_as(), decryption_buffer->mutable_span_as()); + } else { + decryption_buffer = AllocateBuffer(properties_.memory_pool(), 0); + compressed_len = + data_decryptor_->DecryptWithManagedBuffer(page_buffer->span_as(), + decryption_buffer.get()); + } page_buffer = decryption_buffer; } diff --git a/cpp/src/parquet/column_reader.h b/cpp/src/parquet/column_reader.h index ac4469b1904f..38fc8958adef 100644 --- a/cpp/src/parquet/column_reader.h +++ b/cpp/src/parquet/column_reader.h @@ -105,6 +105,8 @@ struct CryptoContext { bool start_decrypt_with_dictionary_page = false; int16_t row_group_ordinal = -1; int16_t column_ordinal = -1; + // Optional: descriptor for the column; may be used to enrich encoding properties + const ColumnDescriptor* column_descriptor = nullptr; std::function()> meta_decryptor_factory; std::function()> data_decryptor_factory; }; diff --git a/cpp/src/parquet/column_writer.cc b/cpp/src/parquet/column_writer.cc index 1f3d64f6228c..6c19eca54b5c 100644 --- a/cpp/src/parquet/column_writer.cc +++ b/cpp/src/parquet/column_writer.cc @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -48,7 +49,7 @@ #include "parquet/chunker_internal.h" #include "parquet/column_page.h" #include "parquet/encoding.h" -#include "parquet/encryption/encryption_internal.h" +#include "parquet/encryption/encryption_utils.h" #include "parquet/encryption/internal_file_encryptor.h" #include "parquet/level_conversion.h" #include "parquet/metadata.h" @@ -305,12 +306,46 @@ class SerializedPageWriter : public PageWriter { if (data_encryptor_.get()) { UpdateEncryption(encryption::kDictionaryPage); - PARQUET_THROW_NOT_OK(encryption_buffer_->Resize( + + // Creating an EncodingProperties object from the metadata. + // We're retrieving the column descriptor and writer properties + // from the metadata_ object to simplify the code. + // + // WriterProperties is created using WriterProperties::Builder::build() (in parquet/properties.h ) + // via ParquetFileFormat::MakeWriter (in arrow/dataset/file_parquet.cc), + // and passed down to ColumnChunkMetaDataBuilder::Make(). + + // A SchemaDescriptor is created in FileWriter::Open() (parquet/file_writer.cc). + // The SchemaDescriptor is passed down to RowGroupMetadataBuilder NextColumnChunk() (parquet/metadata.cc) + // where a ColumnDescriptor is extracted from the SchemaDescriptor, and passed into + // ColumnChunkMetaDataBuilder::Make() + + std::unique_ptr encoding_properties = + EncodingProperties::MakeFromMetadata( + metadata_->descr(), + metadata_->properties(), + static_cast(page)); + data_encryptor_->UpdateEncodingProperties(std::move(encoding_properties)); + + if (data_encryptor_->CanCalculateCiphertextLength()) { + PARQUET_THROW_NOT_OK(encryption_buffer_->Resize( data_encryptor_->CiphertextLength(output_data_len), false)); - output_data_len = - data_encryptor_->Encrypt(compressed_data->span_as(), - encryption_buffer_->mutable_span_as()); + output_data_len = + data_encryptor_->Encrypt(compressed_data->span_as(), + encryption_buffer_->mutable_span_as()); + } else { + output_data_len = + data_encryptor_->EncryptWithManagedBuffer(compressed_data->span_as(), + encryption_buffer_.get()); + } + + output_data_buffer = encryption_buffer_->data(); + + // after the call to encrypt(), add the column encryption metadata to the metadata_ object + auto data_encryptor_metadata = data_encryptor_->GetKeyValueMetadata( + encryption::kDictionaryPage); + UpdateDataEncryptorMetadata(data_encryptor_metadata); } format::PageHeader page_header; @@ -396,13 +431,44 @@ class SerializedPageWriter : public PageWriter { } if (data_encryptor_.get()) { - PARQUET_THROW_NOT_OK(encryption_buffer_->Resize( - data_encryptor_->CiphertextLength(output_data_len), false)); UpdateEncryption(encryption::kDataPage); - output_data_len = - data_encryptor_->Encrypt(compressed_data->span_as(), - encryption_buffer_->mutable_span_as()); + + // Creating an EncodingProperties object from the metadata. + // We're retrieving the column descriptor and writer properties + // from the metadata_ object to simplify the code. + // + // WriterProperties is created using WriterProperties::Builder::build() (in parquet/properties.h ) + // via ParquetFileFormat::MakeWriter (in arrow/dataset/file_parquet.cc), + // and passed down to ColumnChunkMetaDataBuilder::Make(). + + // A SchemaDescriptor is created in FileWriter::Open() (parquet/file_writer.cc). + // The SchemaDescriptor is passed down to RowGroupMetadataBuilder NextColumnChunk() (parquet/metadata.cc) + // where a ColumnDescriptor is extracted from the SchemaDescriptor, and passed into + // ColumnChunkMetaDataBuilder::Make() + std::unique_ptr encoding_properties = + EncodingProperties::MakeFromMetadata( + metadata_->descr(), + metadata_->properties(), + static_cast(page)); + data_encryptor_->UpdateEncodingProperties(std::move(encoding_properties)); + + if (data_encryptor_->CanCalculateCiphertextLength()) { + PARQUET_THROW_NOT_OK(encryption_buffer_->Resize( + data_encryptor_->CiphertextLength(output_data_len), false)); + output_data_len = + data_encryptor_->Encrypt(compressed_data->span_as(), + encryption_buffer_->mutable_span_as()); + } else { + output_data_len = + data_encryptor_->EncryptWithManagedBuffer(compressed_data->span_as(), + encryption_buffer_.get()); + } output_data_buffer = encryption_buffer_->data(); + + // after the call to encrypt(), add the column encryption metadata to the metadata_ object + auto data_encryptor_metadata = data_encryptor_->GetKeyValueMetadata( + encryption::kDataPage); + UpdateDataEncryptorMetadata(data_encryptor_metadata); } format::PageHeader page_header; @@ -589,6 +655,42 @@ class SerializedPageWriter : public PageWriter { } } + // Updates metadata with encryptor-provided KeyValueMetadata, checking for conflicts. + // Throws ParquetException if a key exists with a different value (same value is allowed). + void UpdateDataEncryptorMetadata( + const std::shared_ptr& data_encryptor_metadata) { + if (data_encryptor_metadata == nullptr) { + return; + } + + // Prevent overriding an existing key with a different value after obtaining + // metadata coming from the encryptor (same value is fine). + const auto& keys = data_encryptor_metadata->keys(); + const auto& values = data_encryptor_metadata->values(); + for (size_t i = 0; i < keys.size(); ++i) { + const auto& key = keys[i]; + const auto& value = values[i]; + auto it = encryptor_seen_kvmetadata_.find(key); + if (it != encryptor_seen_kvmetadata_.end()) { + + //if we're here, the key already exists in the encryptor_seen_kvmetadata_ map + // we need to check if the value is the same. If it is not, throw an exception. + if (it->second != value) { + std::stringstream ss; + ss << "Encryptor-provided metadata attempts to override key '" << key + << "' with a different value (old='" << it->second + << "', new='" << value << "')"; + throw ParquetException(ss.str()); + } + } else { + //if we're here, the key does not exist in the encryptor_seen_kvmetadata_ map + // add it to the map. + encryptor_seen_kvmetadata_.emplace(key, value); + } + } + metadata_->AddKeyValueMetadata(data_encryptor_metadata); + } + std::shared_ptr sink_; ColumnChunkMetaDataBuilder* metadata_; MemoryPool* pool_; @@ -626,6 +728,10 @@ class SerializedPageWriter : public PageWriter { ColumnIndexBuilder* column_index_builder_; OffsetIndexBuilder* offset_index_builder_; + + // Tracks keys inserted from the encryptor so we can detect conflicting overrides + // across modules (dictionary/data pages). Overriding with an identical value is allowed. + std::unordered_map encryptor_seen_kvmetadata_; }; // This implementation of the PageWriter writes to the final sink on Close . @@ -1128,7 +1234,9 @@ int64_t ColumnWriterImpl::Close() { } } - metadata_->SetKeyValueMetadata(key_value_metadata_); + if (key_value_metadata_ != nullptr) { + metadata_->AddKeyValueMetadata(key_value_metadata_); + } pager_->Close(has_dictionary_, fallback_); } diff --git a/cpp/src/parquet/column_writer_test.cc b/cpp/src/parquet/column_writer_test.cc index 990125df4e37..e301370206c2 100644 --- a/cpp/src/parquet/column_writer_test.cc +++ b/cpp/src/parquet/column_writer_test.cc @@ -33,6 +33,7 @@ #include "parquet/column_page.h" #include "parquet/column_reader.h" #include "parquet/column_writer.h" +#include "parquet/encryption/external/test_utils.h" #include "parquet/file_reader.h" #include "parquet/file_writer.h" #include "parquet/geospatial/statistics.h" @@ -1881,6 +1882,268 @@ TEST_F(TestValuesWriterInt32Type, AllNullsCompressionInPageV2) { } } +class TestColumnWriterEncryption : public ::testing::Test { + protected: + void SetUp() override { + sink_ = CreateOutputStream(); + node_ = std::static_pointer_cast( + GroupNode::Make("schema", Repetition::REQUIRED, + {schema::Int32("encrypted_column", Repetition::REQUIRED)})); + schema_descriptor_ = std::make_unique(); + schema_descriptor_->Init(node_); + + kColumnEncryptionKey_ = ::arrow::util::SecureString(std::string("0123456789012345")); + kFooterEncryptionKey_ = ::arrow::util::SecureString(std::string("1234567890123456")); + values_ = {1, 2, 3, 4, 5}; + + library_path_ = parquet::encryption::external::test::TestUtils::GetTestLibraryPath(); + app_context_ = "{\"user_id\": \"test_user\", \"location\": {\"lat\": 0.0, \"lon\": 0.0}}"; + connection_config_ = {{"config_path", "test/path"}, {"agent_library_path", library_path_}}; + } + + std::shared_ptr<::arrow::io::BufferOutputStream> sink_; + std::shared_ptr node_; + std::unique_ptr schema_descriptor_; + + ::arrow::util::SecureString kColumnEncryptionKey_; + ::arrow::util::SecureString kFooterEncryptionKey_; + std::vector values_; + + std::string library_path_; + std::string app_context_; + std::map connection_config_; +}; + +TEST_F(TestColumnWriterEncryption, AESEncryption) { + auto column_properties_builder = ColumnEncryptionProperties::Builder("encrypted_column"); + column_properties_builder.key(kColumnEncryptionKey_) + ->parquet_cipher(ParquetCipher::AES_GCM_V1); + auto column_properties = column_properties_builder.build(); + + std::map> encryption_columns; + encryption_columns["encrypted_column"] = column_properties; + + auto fep_builder = FileEncryptionProperties::Builder(kFooterEncryptionKey_); + fep_builder.encrypted_columns(encryption_columns); + auto file_encryption_properties = fep_builder.build(); + + auto writer_properties_builder = WriterProperties::Builder(); + writer_properties_builder.encryption(file_encryption_properties); + auto writer_properties = writer_properties_builder.build(); + + auto file_writer = ParquetFileWriter::Open(sink_, node_, writer_properties); + auto rg_writer = file_writer->AppendRowGroup(); + auto col_writer = static_cast(rg_writer->NextColumn()); + + col_writer->WriteBatch(values_.size(), nullptr, nullptr, values_.data()); + col_writer->Close(); + file_writer->Close(); + + ASSERT_OK_AND_ASSIGN(auto buffer, sink_->Finish()); + ASSERT_GT(buffer->size(), 0); + + std::map> decryption_cols; + auto decryption_col_builder = ColumnDecryptionProperties::Builder("encrypted_column"); + decryption_cols["encrypted_column"] = decryption_col_builder.key(kColumnEncryptionKey_)->build(); + + auto reader_properties = ReaderProperties(); + auto decryption_properties_builder = FileDecryptionProperties::Builder(); + decryption_properties_builder.footer_key(kFooterEncryptionKey_) + ->column_keys(decryption_cols) + ->build(); + auto decryption_properties = decryption_properties_builder.build(); + reader_properties.file_decryption_properties(decryption_properties); + + auto file_reader = ParquetFileReader::Open( + std::make_shared<::arrow::io::BufferReader>(buffer), reader_properties); + ASSERT_NE(file_reader, nullptr); + + auto rg_reader = file_reader->RowGroup(0); + auto col_reader = std::static_pointer_cast>(rg_reader->Column(0)); + + auto key_value_metadata = rg_reader->metadata()->ColumnChunk(0)->key_value_metadata(); + ASSERT_THAT(key_value_metadata, nullptr); + + std::vector read_values(values_.size()); + int64_t values_read; + col_reader->ReadBatch(values_.size(), nullptr, nullptr, read_values.data(), &values_read); + + ASSERT_EQ(values_read, static_cast(values_.size())); + ASSERT_EQ(values_, read_values); +} + +TEST_F(TestColumnWriterEncryption, ExternalDBPAEncryption) { + ::arrow::util::SecureString kColumnKeyId(std::string("test_column_key1")); + + auto column_properties_builder = ColumnEncryptionProperties::Builder("encrypted_column"); + column_properties_builder.key(kColumnKeyId)->key_id(std::string(kColumnKeyId.as_view())) + ->parquet_cipher(ParquetCipher::EXTERNAL_DBPA_V1); + auto column_properties = column_properties_builder.build(); + + std::map> encryption_columns; + encryption_columns["encrypted_column"] = column_properties; + + auto fep_builder = ExternalFileEncryptionProperties::Builder(kFooterEncryptionKey_); + fep_builder.encrypted_columns(encryption_columns) + ->footer_key_metadata(std::string(kFooterEncryptionKey_.as_view())) + ->set_plaintext_footer() + ->algorithm(ParquetCipher::AES_GCM_V1) + ->app_context(app_context_) + ->connection_config({{ParquetCipher::EXTERNAL_DBPA_V1, connection_config_}}); + auto file_encryption_properties = fep_builder.build_external(); + + auto writer_properties_builder = WriterProperties::Builder(); + writer_properties_builder.encryption(file_encryption_properties); + auto writer_properties = writer_properties_builder.build(); + + auto file_writer = ParquetFileWriter::Open(sink_, node_, writer_properties); + auto rg_writer = file_writer->AppendRowGroup(); + auto col_writer = static_cast(rg_writer->NextColumn()); + + col_writer->WriteBatch(values_.size(), nullptr, nullptr, values_.data()); + col_writer->Close(); + file_writer->Close(); + + ASSERT_OK_AND_ASSIGN(auto buffer, sink_->Finish()); + ASSERT_GT(buffer->size(), 0); + + std::map> decryption_columns; + auto decryption_column_builder = ColumnDecryptionProperties::Builder("encrypted_column"); + decryption_column_builder.parquet_cipher(ParquetCipher::EXTERNAL_DBPA_V1); + decryption_columns["encrypted_column"] = decryption_column_builder.key(kColumnKeyId)->build(); + + auto reader_properties = ReaderProperties(); + auto decryption_properties_builder = ExternalFileDecryptionProperties::Builder(); + decryption_properties_builder.footer_key(kFooterEncryptionKey_) + ->column_keys(decryption_columns) + ->app_context(app_context_) + ->connection_config({{ParquetCipher::EXTERNAL_DBPA_V1, connection_config_}}); + reader_properties.file_decryption_properties(decryption_properties_builder.build_external()); + + auto file_reader = ParquetFileReader::Open( + std::make_shared<::arrow::io::BufferReader>(buffer), reader_properties); + ASSERT_NE(file_reader, nullptr); + + auto rg_reader = file_reader->RowGroup(0); + auto col_reader = std::static_pointer_cast>(rg_reader->Column(0)); + + auto key_value_metadata = rg_reader->metadata()->ColumnChunk(0)->key_value_metadata(); + ASSERT_THAT(key_value_metadata, NotNull()); + ASSERT_EQ(2, key_value_metadata->size()); + ASSERT_OK_AND_ASSIGN(auto test_v1, key_value_metadata->Get("test_key1")); + ASSERT_EQ("test_value1", test_v1); + ASSERT_OK_AND_ASSIGN(auto test_v2, key_value_metadata->Get("test_key2")); + ASSERT_EQ("test_value2", test_v2); + + std::vector read_values(values_.size()); + int64_t values_read; + col_reader->ReadBatch(values_.size(), nullptr, nullptr, read_values.data(), &values_read); + + ASSERT_EQ(values_read, static_cast(values_.size())); + ASSERT_EQ(values_, read_values); +} + +TEST_F(TestColumnWriterEncryption, ExternalDBPAEncryption_MultiplePagesNoConflictingOverrides) { + ::arrow::util::SecureString kColumnKeyId(std::string("test_column_key1")); + + // Set up EXTERNAL_DBPA encryption on a single column so that the encryptor + // returns key/value metadata on dictionary and data pages. + auto column_properties_builder = ColumnEncryptionProperties::Builder("encrypted_column"); + column_properties_builder.key(kColumnKeyId)->key_id(std::string(kColumnKeyId.as_view())) + ->parquet_cipher(ParquetCipher::EXTERNAL_DBPA_V1); + auto column_properties = column_properties_builder.build(); + + std::map> encryption_columns; + encryption_columns["encrypted_column"] = column_properties; + + auto fep_builder = ExternalFileEncryptionProperties::Builder(kFooterEncryptionKey_); + fep_builder.encrypted_columns(encryption_columns) + ->footer_key_metadata(std::string(kFooterEncryptionKey_.as_view())) + ->set_plaintext_footer() + ->algorithm(ParquetCipher::AES_GCM_V1) + ->app_context(app_context_) + ->connection_config({{ParquetCipher::EXTERNAL_DBPA_V1, connection_config_}}); + auto file_encryption_properties = fep_builder.build_external(); + + auto writer_properties_builder = WriterProperties::Builder(); + + // Force many very small pages so the writer will obtain encryptor metadata + // multiple times (dictionary page + several data pages). Disable dictionary + // so we always produce multiple data pages. + writer_properties_builder.disable_dictionary() + ->data_pagesize(100) + ->encryption(file_encryption_properties); + auto writer_properties = writer_properties_builder.build(); + + auto file_writer = ParquetFileWriter::Open(sink_, node_, writer_properties); + auto rg_writer = file_writer->AppendRowGroup(); + auto col_writer = static_cast(rg_writer->NextColumn()); + + // Write enough values to guarantee multiple data pages are emitted. + // (we're setting page size to 100 above) + std::vector many_values; + many_values.reserve(1000); + for (int i = 0; i < 1000; ++i) { + many_values.push_back(values_[i % values_.size()]); + } + + col_writer->WriteBatch(many_values.size(), nullptr, nullptr, many_values.data()); + col_writer->Close(); + + // Closing should not throw even across many pages with repeated encryptor metadata, + // as long as the encryptor does not change values for the same keys. + ASSERT_NO_THROW(file_writer->Close()); +} + +TEST_F(TestColumnWriterEncryption, ExternalDBPAEncryption_ConflictingMetadataThrows) { + ::arrow::util::SecureString kColumnKeyId(std::string("test_column_key1")); + + // Set up EXTERNAL_DBPA with a test-agent flag that forces a conflicting metadata value + auto column_properties_builder = ColumnEncryptionProperties::Builder("encrypted_column"); + column_properties_builder.key(kColumnKeyId)->key_id(std::string(kColumnKeyId.as_view())) + ->parquet_cipher(ParquetCipher::EXTERNAL_DBPA_V1); + auto column_properties = column_properties_builder.build(); + + std::map> encryption_columns; + encryption_columns["encrypted_column"] = column_properties; + + auto fep_builder = ExternalFileEncryptionProperties::Builder(kFooterEncryptionKey_); + fep_builder.encrypted_columns(encryption_columns) + ->footer_key_metadata(std::string(kFooterEncryptionKey_.as_view())) + ->set_plaintext_footer() + ->algorithm(ParquetCipher::AES_GCM_V1) + ->app_context(app_context_) + ->connection_config({{ParquetCipher::EXTERNAL_DBPA_V1, { + {"dbpa_test_force_conflicting_metadata", "1"}, + {"agent_library_path", library_path_} + }}}); + + auto file_encryption_properties = fep_builder.build_external(); + + auto writer_properties_builder = WriterProperties::Builder(); + // Force tiny pages and small write batches so multiple Encrypt() calls occur + writer_properties_builder.disable_dictionary() + ->data_pagesize(1) + ->write_batch_size(1) + ->encryption(file_encryption_properties); + auto writer_properties = writer_properties_builder.build(); + + auto file_writer = ParquetFileWriter::Open(sink_, node_, writer_properties); + auto rg_writer = file_writer->AppendRowGroup(); + auto col_writer = static_cast(rg_writer->NextColumn()); + + // Write enough values to guarantee multiple Encrypt() invocations. + // (we're setting page size to 1 above) + std::vector many_values; + many_values.reserve(1000); + for (int i = 0; i < 1000; ++i) { + many_values.push_back(values_[i % values_.size()]); + } + + // Expect that a conflicting encryptor metadata value triggers an exception + EXPECT_THROW(col_writer->WriteBatch(many_values.size(), nullptr, nullptr, many_values.data()), ParquetException); +} + #ifdef ARROW_WITH_ZSTD TEST_F(TestValuesWriterInt32Type, AvoidCompressedInDataPageV2) { Compression::type compression = Compression::ZSTD; diff --git a/cpp/src/parquet/encryption/CMakeLists.txt b/cpp/src/parquet/encryption/CMakeLists.txt index b4c977fccd18..d211d778c91e 100644 --- a/cpp/src/parquet/encryption/CMakeLists.txt +++ b/cpp/src/parquet/encryption/CMakeLists.txt @@ -17,3 +17,40 @@ # Headers: public api arrow_install_all_headers("parquet/encryption") + +if(ARROW_TESTING) + # Add library for DBPATestAgent + add_library(DBPATestAgent SHARED + external/dbpa_test_agent.cc) + + # DBPATestAgent configuration + target_link_libraries(DBPATestAgent PUBLIC + arrow_shared + magic_enum_header_only + tcb_span + dbps_interface) + + set_target_properties(DBPATestAgent PROPERTIES OUTPUT_NAME "DBPATestAgent") + + # Add test for LoadableEncryptorUtils + # (depends on DBPATestAgent) + add_parquet_test(loadable-encryptor-utils-test + SOURCES external/loadable_encryptor_utils_test.cc + external/test_utils.cc + LABELS "parquet-tests" "encryption-tests") + + # Add test for DBPALibraryWrapper + add_parquet_test(dbpa-library-wrapper-test + SOURCES external/dbpa_library_wrapper_test.cc + LABELS "parquet-tests" "encryption-tests") + + # Add a test for dbpa_enum_utils + add_parquet_test(dbpa-utils-test + SOURCES external/dbpa_enum_utils_test.cc + LABELS "parquet-tests" "encryption-tests") + + # Add a test for dbpa_executor_test + add_parquet_test(dbpa-executor-test + SOURCES external/dbpa_executor_test.cc + LABELS "parquet-tests" "encryption-tests") +endif() diff --git a/cpp/src/parquet/encryption/encryption_internal.cc b/cpp/src/parquet/encryption/aes_encryption.cc similarity index 57% rename from cpp/src/parquet/encryption/encryption_internal.cc rename to cpp/src/parquet/encryption/aes_encryption.cc index 9400fae0adf7..2fe6e71832a4 100644 --- a/cpp/src/parquet/encryption/encryption_internal.cc +++ b/cpp/src/parquet/encryption/aes_encryption.cc @@ -15,21 +15,18 @@ // specific language governing permissions and limitations // under the License. -#include "parquet/encryption/encryption_internal.h" +#include "parquet/encryption/aes_encryption.h" -#include -#include -#include #include #include #include #include +#include #include #include -#include -#include +#include "parquet/encryption/encryption_utils.h" #include "parquet/encryption/openssl_internal.h" #include "parquet/exception.h" @@ -53,103 +50,63 @@ constexpr int32_t kBufferSizeLength = 4; throw ParquetException("Couldn't init ALG decryption"); \ } -class AesCryptoContext { - public: - AesCryptoContext(ParquetCipher::type alg_id, int32_t key_len, bool metadata, - bool include_length) { - openssl::EnsureInitialized(); - - length_buffer_length_ = include_length ? kBufferSizeLength : 0; - ciphertext_size_delta_ = length_buffer_length_ + kNonceLength; - - if (ParquetCipher::AES_GCM_V1 != alg_id && ParquetCipher::AES_GCM_CTR_V1 != alg_id) { - std::stringstream ss; - ss << "Crypto algorithm " << alg_id << " is not supported"; - throw ParquetException(ss.str()); - } - if (16 != key_len && 24 != key_len && 32 != key_len) { - std::stringstream ss; - ss << "Wrong key length: " << key_len; - throw ParquetException(ss.str()); - } - - if (metadata || (ParquetCipher::AES_GCM_V1 == alg_id)) { - aes_mode_ = kGcmMode; - ciphertext_size_delta_ += kGcmTagLength; - } else { - aes_mode_ = kCtrMode; - } - - key_length_ = key_len; +AesCryptoContext::AesCryptoContext( + ParquetCipher::type alg_id, int32_t key_len, bool metadata, bool include_length) { + openssl::EnsureInitialized(); + + length_buffer_length_ = include_length ? kBufferSizeLength : 0; + ciphertext_size_delta_ = length_buffer_length_ + kNonceLength; + + // Not all encryptors support metadata encryption. When that happens, even if the ParquetCipher + // is not AES, the metadata is encrypted using AES. This check should pass. + bool is_aes_algorithm = ParquetCipher::AES_GCM_V1 == alg_id + || ParquetCipher::AES_GCM_CTR_V1 == alg_id; + if (!is_aes_algorithm && !metadata) { + std::stringstream ss; + ss << "Crypto algorithm " << alg_id << " is not supported"; + throw ParquetException(ss.str()); } - - virtual ~AesCryptoContext() = default; - - protected: - static void DeleteCipherContext(EVP_CIPHER_CTX* ctx) { EVP_CIPHER_CTX_free(ctx); } - - using CipherContext = std::unique_ptr; - - static CipherContext NewCipherContext() { - auto ctx = CipherContext(EVP_CIPHER_CTX_new(), DeleteCipherContext); - if (!ctx) { - throw ParquetException("Couldn't init cipher context"); - } - return ctx; + if (16 != key_len && 24 != key_len && 32 != key_len) { + std::stringstream ss; + ss << "Wrong key length: " << key_len; + throw ParquetException(ss.str()); } - int32_t aes_mode_; - int32_t key_length_; - int32_t ciphertext_size_delta_; - int32_t length_buffer_length_; -}; - -class AesEncryptor::AesEncryptorImpl : public AesCryptoContext { - public: - explicit AesEncryptorImpl(ParquetCipher::type alg_id, int32_t key_len, bool metadata, - bool write_length); - - int32_t Encrypt(span plaintext, span key, - span aad, span ciphertext); - - int32_t SignedFooterEncrypt(span footer, span key, - span aad, span nonce, - span encrypted_footer); - - [[nodiscard]] int32_t CiphertextLength(int64_t plaintext_len) const { - if (plaintext_len < 0) { - std::stringstream ss; - ss << "Negative plaintext length " << plaintext_len; - throw ParquetException(ss.str()); - } else if (plaintext_len > - std::numeric_limits::max() - ciphertext_size_delta_) { - std::stringstream ss; - ss << "Plaintext length " << plaintext_len << " plus ciphertext size delta " - << ciphertext_size_delta_ << " overflows int32"; - throw ParquetException(ss.str()); - } - - return static_cast(plaintext_len + ciphertext_size_delta_); + if (metadata || (ParquetCipher::AES_GCM_V1 == alg_id)) { + aes_mode_ = kGcmMode; + ciphertext_size_delta_ += kGcmTagLength; + } else { + aes_mode_ = kCtrMode; } - private: - [[nodiscard]] CipherContext MakeCipherContext() const; + key_length_ = key_len; +} - int32_t GcmEncrypt(span plaintext, span key, - span nonce, span aad, - span ciphertext); +AesEncryptor::AesEncryptor( + ParquetCipher::type alg_id, int32_t key_len, bool metadata, bool write_length) + : AesCryptoContext(alg_id, key_len, metadata, write_length) {} - int32_t CtrEncrypt(span plaintext, span key, - span nonce, span ciphertext); -}; +std::unique_ptr AesEncryptor::Make( + ParquetCipher::type alg_id, int32_t key_len, bool metadata, bool write_length) { + return std::make_unique(alg_id, key_len, metadata, write_length); +} -AesEncryptor::AesEncryptorImpl::AesEncryptorImpl(ParquetCipher::type alg_id, - int32_t key_len, bool metadata, - bool write_length) - : AesCryptoContext(alg_id, key_len, metadata, write_length) {} +int32_t AesEncryptor::CiphertextLength(int64_t plaintext_len) const { + if (plaintext_len < 0) { + std::stringstream ss; + ss << "Negative plaintext length " << plaintext_len; + throw ParquetException(ss.str()); + } else if (plaintext_len > + std::numeric_limits::max() - ciphertext_size_delta_) { + std::stringstream ss; + ss << "Plaintext length " << plaintext_len << " plus ciphertext size delta " + << ciphertext_size_delta_ << " overflows int32"; + throw ParquetException(ss.str()); + } + return static_cast(plaintext_len + ciphertext_size_delta_); +} -AesCryptoContext::CipherContext AesEncryptor::AesEncryptorImpl::MakeCipherContext() - const { +AesCryptoContext::CipherContext AesEncryptor::MakeCipherContext() const { auto ctx = NewCipherContext(); if (kGcmMode == aes_mode_) { // Init AES-GCM with specified key length @@ -173,7 +130,7 @@ AesCryptoContext::CipherContext AesEncryptor::AesEncryptorImpl::MakeCipherContex return ctx; } -int32_t AesEncryptor::AesEncryptorImpl::SignedFooterEncrypt( +int32_t AesEncryptor::SignedFooterEncrypt( span footer, span key, span aad, span nonce, span encrypted_footer) { if (static_cast(key_length_) != key.size()) { @@ -196,10 +153,9 @@ int32_t AesEncryptor::AesEncryptorImpl::SignedFooterEncrypt( return GcmEncrypt(footer, key, nonce, aad, encrypted_footer); } -int32_t AesEncryptor::AesEncryptorImpl::Encrypt(span plaintext, - span key, - span aad, - span ciphertext) { +int32_t AesEncryptor::Encrypt( + span plaintext, span key, span aad, + span ciphertext) { if (static_cast(key_length_) != key.size()) { std::stringstream ss; ss << "Wrong key length " << key.size() << ". Should be " << key_length_; @@ -225,11 +181,9 @@ int32_t AesEncryptor::AesEncryptorImpl::Encrypt(span plaintext, return CtrEncrypt(plaintext, key, nonce, ciphertext); } -int32_t AesEncryptor::AesEncryptorImpl::GcmEncrypt(span plaintext, - span key, - span nonce, - span aad, - span ciphertext) { +int32_t AesEncryptor::GcmEncrypt( + span plaintext, span key, span nonce, + span aad, span ciphertext) { int len; int32_t ciphertext_len; @@ -304,10 +258,9 @@ int32_t AesEncryptor::AesEncryptorImpl::GcmEncrypt(span plaintext return length_buffer_length_ + buffer_size; } -int32_t AesEncryptor::AesEncryptorImpl::CtrEncrypt(span plaintext, - span key, - span nonce, - span ciphertext) { +int32_t AesEncryptor::CtrEncrypt( + span plaintext, span key, span nonce, + span ciphertext) { int len; int32_t ciphertext_len; @@ -369,90 +322,106 @@ int32_t AesEncryptor::AesEncryptorImpl::CtrEncrypt(span plaintext return length_buffer_length_ + buffer_size; } -AesEncryptor::~AesEncryptor() = default; - -int32_t AesEncryptor::SignedFooterEncrypt(span footer, - span key, - span aad, - span nonce, - span encrypted_footer) { - return impl_->SignedFooterEncrypt(footer, key, aad, nonce, encrypted_footer); -} - -int32_t AesEncryptor::CiphertextLength(int64_t plaintext_len) const { - return impl_->CiphertextLength(plaintext_len); +uint64_t AesEncryptorFactory::MakeCacheKey( + ParquetCipher::type alg_id, int32_t key_len, bool metadata) { + uint64_t key = 0; + // Set the algorithm id in the most significant 32 bits. + key |= static_cast(static_cast(alg_id)) << 32; + // Set the key length in the next 8 bits. + key |= static_cast(static_cast(key_len)); + // Set the metadata flag in the next 8 bits. + key |= static_cast(metadata ? 1 : 0) << 8; + return key; } -int32_t AesEncryptor::Encrypt(span plaintext, span key, - span aad, span ciphertext) { - return impl_->Encrypt(plaintext, key, aad, ciphertext); -} - -AesEncryptor::AesEncryptor(ParquetCipher::type alg_id, int32_t key_len, bool metadata, - bool write_length) - : impl_{std::unique_ptr( - new AesEncryptorImpl(alg_id, key_len, metadata, write_length))} {} +AesEncryptor* AesEncryptorFactory::GetMetaAesEncryptor( + ParquetCipher::type alg_id, size_t key_size) { + if (key_size > static_cast(std::numeric_limits::max())) { + throw ParquetException("Invalid key length: exceeds int32_t max"); + } + auto key_len = static_cast(key_size); + // Create the cache key using the algorithm id, key length, and metadata flag + // to avoid collisions for encryptors with the same key length. + uint64_t cache_key = MakeCacheKey(alg_id, key_len, /*metadata=*/true); -class AesDecryptor::AesDecryptorImpl : AesCryptoContext { - public: - explicit AesDecryptorImpl(ParquetCipher::type alg_id, int32_t key_len, bool metadata, - bool contains_length); + // If no encryptor exists for this cache key, create one. + if (encryptor_cache_.find(cache_key) == encryptor_cache_.end()) { + encryptor_cache_[cache_key] = AesEncryptor::Make( + alg_id, key_len, /*metadata=*/true); + } - int32_t Decrypt(span ciphertext, span key, - span aad, span plaintext); + return encryptor_cache_[cache_key].get(); +} - [[nodiscard]] int32_t PlaintextLength(int32_t ciphertext_len) const { - if (ciphertext_len < ciphertext_size_delta_) { - std::stringstream ss; - ss << "Ciphertext length " << ciphertext_len << " is invalid, expected at least " - << ciphertext_size_delta_; - throw ParquetException(ss.str()); - } - return ciphertext_len - ciphertext_size_delta_; +AesEncryptor* AesEncryptorFactory::GetDataAesEncryptor( + ParquetCipher::type alg_id, size_t key_size) { + if (key_size > static_cast(std::numeric_limits::max())) { + throw ParquetException("Invalid key length: exceeds int32_t max"); } + auto key_len = static_cast(key_size); + // Create the cache key using the algorithm id, key length, and metadata flag + // to avoid collisions for encryptors with the same key length. + uint64_t cache_key = MakeCacheKey(alg_id, key_len, /*metadata=*/false); - [[nodiscard]] int32_t CiphertextLength(int32_t plaintext_len) const { - if (plaintext_len < 0) { - std::stringstream ss; - ss << "Negative plaintext length " << plaintext_len; - throw ParquetException(ss.str()); - } else if (plaintext_len > - std::numeric_limits::max() - ciphertext_size_delta_) { - std::stringstream ss; - ss << "Plaintext length " << plaintext_len << " plus ciphertext size delta " - << ciphertext_size_delta_ << " overflows int32"; - throw ParquetException(ss.str()); - } - return plaintext_len + ciphertext_size_delta_; + // If no encryptor exists for this cache key, create one. + if (encryptor_cache_.find(cache_key) == encryptor_cache_.end()) { + encryptor_cache_[cache_key] = AesEncryptor::Make( + alg_id, key_len, /*metadata=*/false); } + return encryptor_cache_[cache_key].get(); +} - private: - [[nodiscard]] CipherContext MakeCipherContext() const; - - /// Get the actual ciphertext length, inclusive of the length buffer length, - /// and validate that the provided buffer size is large enough. - [[nodiscard]] int32_t GetCiphertextLength(span ciphertext) const; +AesDecryptor::AesDecryptor( + ParquetCipher::type alg_id, int32_t key_len, bool metadata, bool contains_length) + : AesCryptoContext(alg_id, key_len, metadata, contains_length) {} - int32_t GcmDecrypt(span ciphertext, span key, - span aad, span plaintext); +std::unique_ptr AesDecryptor::Make( + ParquetCipher::type alg_id, int32_t key_len, bool metadata) { + return std::make_unique(alg_id, key_len, metadata); +} - int32_t CtrDecrypt(span ciphertext, span key, - span plaintext); -}; +int32_t AesDecryptor::PlaintextLength(int32_t ciphertext_len) const { + if (ciphertext_len < ciphertext_size_delta_) { + std::stringstream ss; + ss << "Ciphertext length " << ciphertext_len << " is invalid, expected at least " + << ciphertext_size_delta_; + throw ParquetException(ss.str()); + } + return ciphertext_len - ciphertext_size_delta_; +} -int32_t AesDecryptor::Decrypt(span ciphertext, span key, - span aad, span plaintext) { - return impl_->Decrypt(ciphertext, key, aad, plaintext); +int32_t AesDecryptor::CiphertextLength(int32_t plaintext_len) const { + if (plaintext_len < 0) { + std::stringstream ss; + ss << "Negative plaintext length " << plaintext_len; + throw ParquetException(ss.str()); + } else if (plaintext_len > + std::numeric_limits::max() - ciphertext_size_delta_) { + std::stringstream ss; + ss << "Plaintext length " << plaintext_len << " plus ciphertext size delta " + << ciphertext_size_delta_ << " overflows int32"; + throw ParquetException(ss.str()); + } + return plaintext_len + ciphertext_size_delta_; } -AesDecryptor::~AesDecryptor() {} +int32_t AesDecryptor::Decrypt( + span ciphertext, span key, span aad, + span plaintext) { + if (static_cast(key_length_) != key.size()) { + std::stringstream ss; + ss << "Wrong key length " << key.size() << ". Should be " << key_length_; + throw ParquetException(ss.str()); + } -AesDecryptor::AesDecryptorImpl::AesDecryptorImpl(ParquetCipher::type alg_id, - int32_t key_len, bool metadata, - bool contains_length) - : AesCryptoContext(alg_id, key_len, metadata, contains_length) {} + if (kGcmMode == aes_mode_) { + return GcmDecrypt(ciphertext, key, aad, plaintext); + } + + return CtrDecrypt(ciphertext, key, plaintext); +} -AesCryptoContext::CipherContext AesDecryptor::AesDecryptorImpl::MakeCipherContext() +AesCryptoContext::CipherContext AesDecryptor::MakeCipherContext() const { auto ctx = NewCipherContext(); if (kGcmMode == aes_mode_) { @@ -477,32 +446,7 @@ AesCryptoContext::CipherContext AesDecryptor::AesDecryptorImpl::MakeCipherContex return ctx; } -std::unique_ptr AesEncryptor::Make(ParquetCipher::type alg_id, - int32_t key_len, bool metadata, - bool write_length) { - return std::make_unique(alg_id, key_len, metadata, write_length); -} - -AesDecryptor::AesDecryptor(ParquetCipher::type alg_id, int32_t key_len, bool metadata, - bool contains_length) - : impl_{std::make_unique(alg_id, key_len, metadata, - contains_length)} {} - -std::unique_ptr AesDecryptor::Make(ParquetCipher::type alg_id, - int32_t key_len, bool metadata) { - return std::make_unique(alg_id, key_len, metadata); -} - -int32_t AesDecryptor::PlaintextLength(int32_t ciphertext_len) const { - return impl_->PlaintextLength(ciphertext_len); -} - -int32_t AesDecryptor::CiphertextLength(int32_t plaintext_len) const { - return impl_->CiphertextLength(plaintext_len); -} - -int32_t AesDecryptor::AesDecryptorImpl::GetCiphertextLength( - span ciphertext) const { +int32_t AesDecryptor::GetCiphertextLength(span ciphertext) const { if (length_buffer_length_ > 0) { // Note: length_buffer_length_ must be either 0 or kBufferSizeLength if (ciphertext.size() < static_cast(kBufferSizeLength)) { @@ -547,10 +491,9 @@ int32_t AesDecryptor::AesDecryptorImpl::GetCiphertextLength( } } -int32_t AesDecryptor::AesDecryptorImpl::GcmDecrypt(span ciphertext, - span key, - span aad, - span plaintext) { +int32_t AesDecryptor::GcmDecrypt( + span ciphertext, span key, span aad, + span plaintext) { int len; int32_t plaintext_len; @@ -622,9 +565,8 @@ int32_t AesDecryptor::AesDecryptorImpl::GcmDecrypt(span ciphertex return plaintext_len; } -int32_t AesDecryptor::AesDecryptorImpl::CtrDecrypt(span ciphertext, - span key, - span plaintext) { +int32_t AesDecryptor::CtrDecrypt( + span ciphertext, span key, span plaintext) { int len; int32_t plaintext_len; @@ -681,102 +623,6 @@ int32_t AesDecryptor::AesDecryptorImpl::CtrDecrypt(span ciphertex return plaintext_len; } -int32_t AesDecryptor::AesDecryptorImpl::Decrypt(span ciphertext, - span key, - span aad, - span plaintext) { - if (static_cast(key_length_) != key.size()) { - std::stringstream ss; - ss << "Wrong key length " << key.size() << ". Should be " << key_length_; - throw ParquetException(ss.str()); - } - - if (kGcmMode == aes_mode_) { - return GcmDecrypt(ciphertext, key, aad, plaintext); - } - - return CtrDecrypt(ciphertext, key, plaintext); -} - -static std::string ShortToBytesLe(int16_t input) { - int8_t output[2]; - memset(output, 0, 2); - output[1] = static_cast(0xff & (input >> 8)); - output[0] = static_cast(0xff & (input)); - - return std::string(reinterpret_cast(output), 2); -} - -static void CheckPageOrdinal(int32_t page_ordinal) { - if (ARROW_PREDICT_FALSE(page_ordinal > std::numeric_limits::max())) { - throw ParquetException("Encrypted Parquet files can't have more than " + - std::to_string(std::numeric_limits::max()) + - " pages per chunk: got " + std::to_string(page_ordinal)); - } -} - -std::string CreateModuleAad(const std::string& file_aad, int8_t module_type, - int16_t row_group_ordinal, int16_t column_ordinal, - int32_t page_ordinal) { - CheckPageOrdinal(page_ordinal); - const int16_t page_ordinal_short = static_cast(page_ordinal); - int8_t type_ordinal_bytes[1]; - type_ordinal_bytes[0] = module_type; - std::string type_ordinal_bytes_str(reinterpret_cast(type_ordinal_bytes), - 1); - if (kFooter == module_type) { - std::string result = file_aad + type_ordinal_bytes_str; - return result; - } - std::string row_group_ordinal_bytes = ShortToBytesLe(row_group_ordinal); - std::string column_ordinal_bytes = ShortToBytesLe(column_ordinal); - if (kDataPage != module_type && kDataPageHeader != module_type) { - std::ostringstream out; - out << file_aad << type_ordinal_bytes_str << row_group_ordinal_bytes - << column_ordinal_bytes; - return out.str(); - } - std::string page_ordinal_bytes = ShortToBytesLe(page_ordinal_short); - std::ostringstream out; - out << file_aad << type_ordinal_bytes_str << row_group_ordinal_bytes - << column_ordinal_bytes << page_ordinal_bytes; - return out.str(); -} - -std::string CreateFooterAad(const std::string& aad_prefix_bytes) { - return CreateModuleAad(aad_prefix_bytes, kFooter, static_cast(-1), - static_cast(-1), static_cast(-1)); -} - -// Update last two bytes with new page ordinal (instead of creating new page AAD -// from scratch) -void QuickUpdatePageAad(int32_t new_page_ordinal, std::string* AAD) { - CheckPageOrdinal(new_page_ordinal); - const std::string page_ordinal_bytes = - ShortToBytesLe(static_cast(new_page_ordinal)); - std::memcpy(AAD->data() + AAD->length() - 2, page_ordinal_bytes.data(), 2); -} - -void RandBytes(unsigned char* buf, size_t num) { - if (num > static_cast(std::numeric_limits::max())) { - std::stringstream ss; - ss << "Length " << num << " for RandBytes overflows int"; - throw ParquetException(ss.str()); - } - openssl::EnsureInitialized(); - int status = RAND_bytes(buf, static_cast(num)); - if (status != 1) { - const auto error_code = ERR_get_error(); - char buffer[256]; - ERR_error_string_n(error_code, buffer, sizeof(buffer)); - std::stringstream ss; - ss << "Failed to generate random bytes: " << buffer; - throw ParquetException(ss.str()); - } -} - -void EnsureBackendInitialized() { openssl::EnsureInitialized(); } - #undef ENCRYPT_INIT #undef DECRYPT_INIT diff --git a/cpp/src/parquet/encryption/aes_encryption.h b/cpp/src/parquet/encryption/aes_encryption.h new file mode 100644 index 000000000000..e741a89ee791 --- /dev/null +++ b/cpp/src/parquet/encryption/aes_encryption.h @@ -0,0 +1,203 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "arrow/util/span.h" +#include "parquet/encryption/encryptor_interface.h" +#include "parquet/encryption/decryptor_interface.h" +#include "parquet/types.h" +#include "parquet/exception.h" + +using parquet::ParquetCipher; + +namespace parquet::encryption { + +class AesCryptoContext { + public: + AesCryptoContext(ParquetCipher::type alg_id, int32_t key_len, bool metadata, + bool include_length); + + virtual ~AesCryptoContext() = default; + + protected: + static void DeleteCipherContext(EVP_CIPHER_CTX* ctx) { EVP_CIPHER_CTX_free(ctx); } + + using CipherContext = std::unique_ptr; + + static CipherContext NewCipherContext() { + auto ctx = CipherContext(EVP_CIPHER_CTX_new(), DeleteCipherContext); + if (!ctx) { + throw ParquetException("Couldn't init cipher context"); + } + return ctx; + } + + int32_t aes_mode_; + int32_t key_length_; + int32_t ciphertext_size_delta_; + int32_t length_buffer_length_; +}; + +/// Performs AES encryption operations with GCM or CTR ciphers. +class PARQUET_EXPORT AesEncryptor : public AesCryptoContext, public EncryptorInterface { + public: + /// Can serve one key length only. Possible values: 16, 24, 32 bytes. + /// If write_length is true, prepend ciphertext length to the ciphertext + explicit AesEncryptor(ParquetCipher::type alg_id, int32_t key_len, bool metadata, + bool write_length = true); + + static std::unique_ptr Make(ParquetCipher::type alg_id, int32_t key_len, + bool metadata, bool write_length = true); + + ~AesEncryptor() = default; + + /// Start of Encryptor Interface methods. + + /// Signal whether the encryptor can calculate a valid ciphertext length before performing + /// encryption. + [[nodiscard]] bool CanCalculateCiphertextLength() const override { return true; } + + /// The size of the ciphertext, for this cipher and the specified plaintext length. + [[nodiscard]] int32_t CiphertextLength(int64_t plaintext_len) const override; + + /// Encrypts plaintext with the key and aad. Key length is passed only for validation. + /// If different from value in constructor, exception will be thrown. + int32_t Encrypt(::arrow::util::span plaintext, + ::arrow::util::span key, + ::arrow::util::span aad, + ::arrow::util::span ciphertext) override; + + /// Encrypt the plaintext and leave the results in the ciphertext buffer. This method is + /// not supported as we can calculate the ciphertext length before encryption. + int32_t EncryptWithManagedBuffer(::arrow::util::span plaintext, + ::arrow::ResizableBuffer* ciphertext) override { + throw ParquetException( + "EncryptWithManagedBuffer is not supported in AesEncryptor, use Encrypt instead"); + } + + /// Encrypts plaintext footer, in order to compute footer signature (tag). + int32_t SignedFooterEncrypt(::arrow::util::span footer, + ::arrow::util::span key, + ::arrow::util::span aad, + ::arrow::util::span nonce, + ::arrow::util::span encrypted_footer) override; + + /// End of Encryptor Interface methods. + + private: + [[nodiscard]] CipherContext MakeCipherContext() const; + + int32_t GcmEncrypt(::arrow::util::span plaintext, + ::arrow::util::span key, + ::arrow::util::span nonce, + ::arrow::util::span aad, + ::arrow::util::span ciphertext); + + int32_t CtrEncrypt(::arrow::util::span plaintext, + ::arrow::util::span key, + ::arrow::util::span nonce, + ::arrow::util::span ciphertext); +}; + +// AesEncryptor supports only three key lengths: 16, 24, 32 bytes, so at most there could be +// up to three types of meta_encryptors and data_encryptors. This factory uses a cache to +// store the encryptors for the different key lengths. +class AesEncryptorFactory { + public: + AesEncryptor* GetMetaAesEncryptor(ParquetCipher::type alg_id, size_t key_size); + AesEncryptor* GetDataAesEncryptor(ParquetCipher::type alg_id, size_t key_size); + + private: + /// Build a cache key including algorithm id, key length, and metadata flag. + static uint64_t MakeCacheKey( + ParquetCipher::type alg_id, int32_t key_len, bool metadata); + + std::unordered_map> encryptor_cache_; +}; + +/// Performs AES decryption operations with GCM or CTR ciphers. +class PARQUET_EXPORT AesDecryptor : public AesCryptoContext, public DecryptorInterface { + public: + /// \brief Construct an AesDecryptor + /// + /// \param alg_id the encryption algorithm to use + /// \param key_len key length. Possible values: 16, 24, 32 bytes. + /// \param metadata if true then this is a metadata decryptor + /// \param contains_length if true, expect ciphertext length prepended to the ciphertext + explicit AesDecryptor(ParquetCipher::type alg_id, int32_t key_len, bool metadata, + bool contains_length = true); + + static std::unique_ptr Make(ParquetCipher::type alg_id, int32_t key_len, + bool metadata); + + ~AesDecryptor() = default; + + /// Start of Decryptor Interface methods. + + /// Signal whether the decryptor can calculate a valid plaintext or ciphertext length before + /// performing decryption or not. If false, a proper sized buffer cannot be allocated before + /// calling the Decrypt method, and Arrow must use this decryptor's DecryptWithManagedBuffer + /// method instead of Decrypt. + [[nodiscard]] bool CanCalculateLengths() const override { return true; } + + /// The size of the plaintext, for this cipher and the specified ciphertext length. + [[nodiscard]] int32_t PlaintextLength(int32_t ciphertext_len) const override; + + /// The size of the ciphertext, for this cipher and the specified plaintext length. + [[nodiscard]] int32_t CiphertextLength(int32_t plaintext_len) const override; + + /// Decrypts ciphertext with the key and aad. Key length is passed only for + /// validation. If different from value in constructor, exception will be thrown. + /// The caller is responsible for ensuring that the plaintext buffer is at least as + /// large as PlaintextLength(ciphertext_len). + int32_t Decrypt(::arrow::util::span ciphertext, + ::arrow::util::span key, + ::arrow::util::span aad, + ::arrow::util::span plaintext) override; + + /// Decrypt the ciphertext and leave the results in the plaintext buffer. + /// This method is not supported as we can calculate the plaintext length before decryption. + int32_t DecryptWithManagedBuffer(::arrow::util::span ciphertext, + ::arrow::ResizableBuffer* plaintext) override { + throw ParquetException( + "DecryptWithManagedBuffer is not supported in AesDecryptor, use Decrypt instead"); + } + + /// End of Decryptor Interface methods. + + private: + [[nodiscard]] CipherContext MakeCipherContext() const; + + /// Get the actual ciphertext length, inclusive of the length buffer length, + /// and validate that the provided buffer size is large enough. + [[nodiscard]] int32_t GetCiphertextLength(::arrow::util::span ciphertext) const; + + int32_t GcmDecrypt(::arrow::util::span ciphertext, + ::arrow::util::span key, + ::arrow::util::span aad, + ::arrow::util::span plaintext); + + int32_t CtrDecrypt(::arrow::util::span ciphertext, + ::arrow::util::span key, + ::arrow::util::span plaintext); +}; + +} // namespace parquet::encryption diff --git a/cpp/src/parquet/encryption/encryption_internal_nossl.cc b/cpp/src/parquet/encryption/aes_encryption_nossl.cc similarity index 90% rename from cpp/src/parquet/encryption/encryption_internal_nossl.cc rename to cpp/src/parquet/encryption/aes_encryption_nossl.cc index 2450f8654d6f..099cddfbfdfd 100644 --- a/cpp/src/parquet/encryption/encryption_internal_nossl.cc +++ b/cpp/src/parquet/encryption/aes_encryption_nossl.cc @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "parquet/encryption/encryption_internal.h" +#include "parquet/encryption/aes_encryption.h" #include "parquet/exception.h" namespace parquet::encryption { @@ -29,9 +29,10 @@ void ThrowOpenSSLRequiredException() { } // namespace -class AesEncryptor::AesEncryptorImpl {}; - -AesEncryptor::~AesEncryptor() {} +AesCryptoContext::AesCryptoContext( + ParquetCipher::type alg_id, int32_t key_len, bool metadata, bool include_length) { +ThrowOpenSSLRequiredException(); +} int32_t AesEncryptor::SignedFooterEncrypt(::arrow::util::span footer, ::arrow::util::span key, @@ -56,12 +57,11 @@ int32_t AesEncryptor::Encrypt(::arrow::util::span plaintext, } AesEncryptor::AesEncryptor(ParquetCipher::type alg_id, int32_t key_len, bool metadata, - bool write_length) { + bool write_length) + : AesCryptoContext(alg_id, key_len, metadata, write_length) { ThrowOpenSSLRequiredException(); } -class AesDecryptor::AesDecryptorImpl {}; - int32_t AesDecryptor::Decrypt(::arrow::util::span ciphertext, ::arrow::util::span key, ::arrow::util::span aad, @@ -70,8 +70,6 @@ int32_t AesDecryptor::Decrypt(::arrow::util::span ciphertext, return -1; } -AesDecryptor::~AesDecryptor() {} - std::unique_ptr AesEncryptor::Make(ParquetCipher::type alg_id, int32_t key_len, bool metadata, bool write_length) { @@ -80,7 +78,8 @@ std::unique_ptr AesEncryptor::Make(ParquetCipher::type alg_id, } AesDecryptor::AesDecryptor(ParquetCipher::type alg_id, int32_t key_len, bool metadata, - bool contains_length) { + bool contains_length) + : AesCryptoContext(alg_id, key_len, metadata, contains_length) { ThrowOpenSSLRequiredException(); } diff --git a/cpp/src/parquet/encryption/encryption_internal_test.cc b/cpp/src/parquet/encryption/aes_encryption_test.cc similarity index 84% rename from cpp/src/parquet/encryption/encryption_internal_test.cc rename to cpp/src/parquet/encryption/aes_encryption_test.cc index bf6607e32877..73d333d3afae 100644 --- a/cpp/src/parquet/encryption/encryption_internal_test.cc +++ b/cpp/src/parquet/encryption/aes_encryption_test.cc @@ -17,7 +17,8 @@ #include -#include "parquet/encryption/encryption_internal.h" +#include "parquet/encryption/aes_encryption.h" +#include "parquet/encryption/encryption.h" namespace parquet::encryption::test { @@ -137,4 +138,24 @@ TEST_F(TestAesEncryption, AesGcmCtrDecryptCiphertextBufferTooSmall) { DecryptCiphertextBufferTooSmall(ParquetCipher::AES_GCM_CTR_V1); } +TEST_F(TestAesEncryption, AesGcmEncryptWithManagedBuffer) { + AesEncryptor encryptor( + ParquetCipher::AES_GCM_V1, /*key_length*/ 16, /*metadata*/ false, /*write_length*/ true); + std::unique_ptr<::arrow::ResizableBuffer> ciphertext_buffer; + ASSERT_TRUE(encryptor.CanCalculateCiphertextLength()); + EXPECT_THROW( + encryptor.EncryptWithManagedBuffer(str2span("plain_text_"), ciphertext_buffer.get()), + ParquetException); +} + +TEST_F(TestAesEncryption, AesGcmDecryptWithManagedBuffer) { + AesDecryptor decryptor( + ParquetCipher::AES_GCM_V1, /*key_length*/ 16, /*metadata*/ false, /*write_length*/ true); + std::unique_ptr<::arrow::ResizableBuffer> ciphertext_buffer; + ASSERT_TRUE(decryptor.CanCalculateLengths()); + EXPECT_THROW( + decryptor.DecryptWithManagedBuffer(str2span("plain_text_"), ciphertext_buffer.get()), + ParquetException); +} + } // namespace parquet::encryption::test diff --git a/cpp/src/parquet/encryption/crypto_factory.cc b/cpp/src/parquet/encryption/crypto_factory.cc index 50b074537884..5a9aa702fb0c 100644 --- a/cpp/src/parquet/encryption/crypto_factory.cc +++ b/cpp/src/parquet/encryption/crypto_factory.cc @@ -22,7 +22,7 @@ #include "arrow/util/string.h" #include "parquet/encryption/crypto_factory.h" -#include "parquet/encryption/encryption_internal.h" +#include "parquet/encryption/encryption_utils.h" #include "parquet/encryption/file_key_unwrapper.h" #include "parquet/encryption/file_system_key_material_store.h" #include "parquet/encryption/key_toolkit_internal.h" @@ -31,6 +31,78 @@ using arrow::util::SecureString; namespace parquet::encryption { +/// Extracting functionality common to both GetFileEncryptionProperties and +/// GetExternalFileEncryptionProperties here for reuse. +namespace { + +// Struct to simplify the returned objects in GetFileKeyUtils. +struct FileKeyUtils { + std::shared_ptr key_material_store; + FileKeyWrapper key_wrapper; +}; + +FileKeyUtils GetFileKeyUtils( + const std::shared_ptr& key_toolkit, + const KmsConnectionConfig& kms_connection_config, + const EncryptionConfiguration& encryption_config, + const std::string& file_path, const std::shared_ptr<::arrow::fs::FileSystem>& file_system) { + std::shared_ptr key_material_store = nullptr; + if (!encryption_config.internal_key_material) { + try { + key_material_store = + FileSystemKeyMaterialStore::Make(file_path, file_system, false); + } catch (ParquetException& e) { + std::stringstream ss; + ss << "Failed to get key material store.\n" << e.what() << "\n"; + throw ParquetException(ss.str()); + } + } + + FileKeyWrapper key_wrapper(key_toolkit.get(), kms_connection_config, + key_material_store, encryption_config.cache_lifetime_seconds, + encryption_config.double_wrapping); + return {key_material_store, std::move(key_wrapper)}; +} + +int ValidateAndGetKeyLength(int32_t dek_length_bits) { + if (!internal::ValidateKeyLength(dek_length_bits)) { + std::ostringstream ss; + ss << "Wrong data key length : " << dek_length_bits; + throw ParquetException(ss.str()); + } + return dek_length_bits / 8; +} + +std::map> ConvertConnectionConfig( + const std::unordered_map>& connection_config) { + + std::map> converted_config; + + for (const auto& [cipher_type, inner_config] : connection_config) { + if (!IsParquetCipherSupported(cipher_type)) { + throw ParquetException("Invalid ParquetCipher type: " + + std::to_string(static_cast(cipher_type))); + } + + std::map converted_inner; + for (const auto& [key, value] : inner_config) { + if (key.empty()) { + throw ParquetException("Empty key in connection config"); + } + if (value.empty()) { + throw ParquetException("Empty value for key '" + key + "' in connection config"); + } + converted_inner[key] = value; + } + converted_config[cipher_type] = converted_inner; + } + + return converted_config; +} + +} // Anonymous namespace + void CryptoFactory::RegisterKmsClientFactory( std::shared_ptr kms_client_factory) { key_toolkit_->RegisterKmsClientFactory(std::move(kms_client_factory)); @@ -49,30 +121,9 @@ std::shared_ptr CryptoFactory::GetFileEncryptionProper const std::string& footer_key_id = encryption_config.footer_key; const std::string& column_key_str = encryption_config.column_keys; - std::shared_ptr key_material_store = nullptr; - if (!encryption_config.internal_key_material) { - try { - key_material_store = - FileSystemKeyMaterialStore::Make(file_path, file_system, false); - } catch (ParquetException& e) { - std::stringstream ss; - ss << "Failed to get key material store.\n" << e.what() << "\n"; - throw ParquetException(ss.str()); - } - } - - FileKeyWrapper key_wrapper(key_toolkit_.get(), kms_connection_config, - key_material_store, encryption_config.cache_lifetime_seconds, - encryption_config.double_wrapping); - - int32_t dek_length_bits = encryption_config.data_key_length_bits; - if (!internal::ValidateKeyLength(dek_length_bits)) { - std::ostringstream ss; - ss << "Wrong data key length : " << dek_length_bits; - throw ParquetException(ss.str()); - } - - int dek_length = dek_length_bits / 8; + auto [key_material_store, key_wrapper] = GetFileKeyUtils( + key_toolkit_, kms_connection_config, encryption_config, file_path, file_system); + int dek_length = ValidateAndGetKeyLength(encryption_config.data_key_length_bits); SecureString footer_key(dek_length, '\0'); RandBytes(footer_key.as_span().data(), footer_key.size()); @@ -101,6 +152,99 @@ std::shared_ptr CryptoFactory::GetFileEncryptionProper return properties_builder.build(); } +std::shared_ptr +CryptoFactory::GetExternalFileEncryptionProperties( + const KmsConnectionConfig& kms_connection_config, + const ExternalEncryptionConfiguration& external_encryption_config, + const std::string& file_path, const std::shared_ptr<::arrow::fs::FileSystem>& file_system) { + // Validate the same rules as FileEncryptionProperties but considering per_column_encryption too. + // If uniform_encryption is not set then either column_keys or per_column_encryption must have + // values. + // If uniform_encryption is set, then both column_keys and per_column_encryption must be empty. + bool no_columns_encrypted = external_encryption_config.column_keys.empty() && + external_encryption_config.per_column_encryption.empty(); + if (!external_encryption_config.uniform_encryption && no_columns_encrypted) { + throw ParquetException( + "Either uniform_encryption must be set or column encryption must be specified in either " + "column_keys or per_column_encryption"); + } else if (external_encryption_config.uniform_encryption && !no_columns_encrypted) { + throw ParquetException("Cannot set both column encryption and uniform_encryption"); + } + + auto [key_material_store, key_wrapper] = GetFileKeyUtils( + key_toolkit_, kms_connection_config, external_encryption_config, file_path, file_system); + + int dek_length = ValidateAndGetKeyLength(external_encryption_config.data_key_length_bits); + + SecureString footer_key(dek_length, '\0'); + RandBytes(reinterpret_cast(footer_key.as_span().data()), footer_key.size()); + + std::string footer_key_metadata = + key_wrapper.GetEncryptionKeyMetadata(footer_key, external_encryption_config.footer_key, true); + + ExternalFileEncryptionProperties::Builder external_properties_builder = + ExternalFileEncryptionProperties::Builder(footer_key); + external_properties_builder.footer_key_metadata(footer_key_metadata); + external_properties_builder.algorithm(external_encryption_config.encryption_algorithm); + + if (!external_encryption_config.uniform_encryption && + external_encryption_config.plaintext_footer) { + external_properties_builder.set_plaintext_footer(); + } + + ColumnPathToEncryptionPropertiesMap encrypted_columns; + if (!external_encryption_config.column_keys.empty()) { + encrypted_columns = GetColumnEncryptionProperties( + dek_length, external_encryption_config.column_keys, &key_wrapper); + } + if (!external_encryption_config.per_column_encryption.empty()) { + for (const auto& pair : external_encryption_config.per_column_encryption) { + const std::string& column_name = pair.first; + const ColumnEncryptionAttributes& attributes = pair.second; + + // Validate column names are not in both column_keys and per_column_encryption maps. + if (encrypted_columns.find(column_name) != encrypted_columns.end()) { + std::stringstream string_stream; + string_stream << "Multiple keys defined for column [" << column_name << "]. "; + string_stream << "Keys found in column_keys and in per_column_encryption."; + throw ParquetException(string_stream.str()); + } + + SecureString column_key(dek_length, '\0'); + RandBytes(reinterpret_cast(column_key.as_span().data()), column_key.size()); + std::string column_key_metadata = + key_wrapper.GetEncryptionKeyMetadata(column_key, attributes.key_id, false); + + std::shared_ptr column_properties = + ColumnEncryptionProperties::Builder(column_name) + .key(column_key) + ->key_metadata(column_key_metadata) + ->parquet_cipher(attributes.parquet_cipher) + ->build(); + + encrypted_columns.insert({column_name, column_properties}); + } + } + if (!encrypted_columns.empty()) { + external_properties_builder.encrypted_columns(encrypted_columns); + } + + if (!external_encryption_config.app_context.empty()) { + external_properties_builder.app_context(external_encryption_config.app_context); + } + + if (!external_encryption_config.connection_config.empty()) { + external_properties_builder.connection_config(ConvertConnectionConfig( + external_encryption_config.connection_config)); + } + + if (key_material_store != nullptr) { + key_material_store->SaveMaterial(); + } + + return external_properties_builder.build_external(); +} + ColumnPathToEncryptionPropertiesMap CryptoFactory::GetColumnEncryptionProperties( int dek_length, const std::string& column_keys, FileKeyWrapper* key_wrapper) { ColumnPathToEncryptionPropertiesMap encrypted_columns; @@ -184,6 +328,33 @@ std::shared_ptr CryptoFactory::GetFileDecryptionProper ->build(); } +std::shared_ptr +CryptoFactory::GetExternalFileDecryptionProperties( + const KmsConnectionConfig& kms_connection_config, + const ExternalDecryptionConfiguration& external_decryption_config, + const std::string& file_path, const std::shared_ptr<::arrow::fs::FileSystem>& file_system) { + + // Use the same FileKeyUnwrapper as in the FileDecryptionProperties. + auto key_retriever = std::make_shared( + key_toolkit_, kms_connection_config, external_decryption_config.cache_lifetime_seconds, + file_path, file_system); + + ExternalFileDecryptionProperties::Builder builder; + builder.key_retriever(key_retriever); + builder.plaintext_files_allowed(); + + if (!external_decryption_config.app_context.empty()) { + builder.app_context(external_decryption_config.app_context); + } + + if (!external_decryption_config.connection_config.empty()) { + builder.connection_config(ConvertConnectionConfig( + external_decryption_config.connection_config)); + } + + return builder.build_external(); +} + void CryptoFactory::RotateMasterKeys( const KmsConnectionConfig& kms_connection_config, const std::string& parquet_file_path, diff --git a/cpp/src/parquet/encryption/crypto_factory.h b/cpp/src/parquet/encryption/crypto_factory.h index 3c6a5f29ea88..ff12eccdfb30 100644 --- a/cpp/src/parquet/encryption/crypto_factory.h +++ b/cpp/src/parquet/encryption/crypto_factory.h @@ -18,12 +18,14 @@ #pragma once #include +#include #include "parquet/encryption/encryption.h" #include "parquet/encryption/file_key_wrapper.h" #include "parquet/encryption/key_toolkit.h" #include "parquet/encryption/kms_client_factory.h" #include "parquet/platform.h" +#include "parquet/types.h" namespace parquet::encryption { @@ -86,6 +88,57 @@ struct PARQUET_EXPORT EncryptionConfiguration { int32_t data_key_length_bits = kDefaultDataKeyLengthBits; }; +/// Helper struct for use in per column encryption specification. +/// The ExternalEncryptionConfiguration will use this to send encryption parameters per column. +struct PARQUET_EXPORT ColumnEncryptionAttributes { + /// Which type of encryptor to use. + ParquetCipher::type parquet_cipher; + + /// The key_id to use for this column. + std::string key_id; +}; + +/// Encryption Configuration for use with External Encryptions. +/// Extends the already existing EncryptionConfiguration with more context and with +/// the capability of specifying encryption algorithm per column. +struct PARQUET_EXPORT ExternalEncryptionConfiguration : public EncryptionConfiguration { +explicit ExternalEncryptionConfiguration(const std::string& footer_key) + : EncryptionConfiguration(footer_key) {} + +/// Map of the columns to encrypt to their associated encryption parameters. The id of the map +/// is the column name, and the value is a ColumnEncryptionAttributes struct that can be +/// used to construct the ColumnEncryptionProperties in the CryptoFactory. +/// As with the EncryptionConfiguration, either: +/// (1) uniform_encryption = true +/// or +/// (2) column_keys and/or per_column_encryption is set +/// If none of (1) and (2) are true, or if both are true, an exception will be thrown. +/// If a column name appears in the original column_keys list, it will be encrypted with the +/// algorithm specified in the encryption_algorithm field. +/// If a column name appears in the new per_column_encryption map, it will be encrypted using the +/// per column specific algorithm and key. +/// If a column name appears in both, an exception will be thrown. +std::unordered_map per_column_encryption; + +/// External encryptors may use additional context provided by the application to +/// enforce robust access control. The values sent to the external encryptor depend on each +/// implementation. +/// This value must be a valid JSON-formatted string. +/// Validation of the string will be done by the external encryptor, Arrow will only +/// forward this value. +/// Format: "{\"user_id\": \"abc123\", \"location\": {\"lat\": 9.7489, \"lon\": -83.7534}}" +std::string app_context; + +/// Map of the encryption algorithms to the key/value map of the location of configuration files +/// needed by the external encryptors. This may include location of a dynamically-linked +/// library, or the location of a file where the external encryptor can find urls, certificates, +/// and parameters needed to make a remote call. +/// For security, these values should never be sent in this config, only the locations of +/// the files that the external encryptor will know how to access. +std::unordered_map> + connection_config; +}; + struct PARQUET_EXPORT DecryptionConfiguration { /// Lifetime of cached entities (key encryption keys, local wrapping keys, KMS client /// objects). @@ -93,6 +146,26 @@ struct PARQUET_EXPORT DecryptionConfiguration { double cache_lifetime_seconds = kDefaultCacheLifetimeSeconds; }; +struct PARQUET_EXPORT ExternalDecryptionConfiguration : public DecryptionConfiguration { + /// External decryptors may use additional context provided by the application to + /// enforce robust access control. The values sent to the external decryptor depend on each + /// implementation. + /// This value must be a valid JSON-formatted string. + /// Validation of the string will be done by the external decryptors, Arrow will only + /// forward this value. + /// Format: "{\"user_id\": \"abc123\", \"location\": {\"lat\": 9.7489, \"lon\": -83.7534}}" + std::string app_context; + + /// Map of the encryption algorithms to the key/value map of the location of configuration files + /// needed by the external decryptors. This may include location of a dynamically-linked + /// library, or the location of a file where the external decryptor can find urls, certificates, + /// and parameters needed to make a remote call. + /// For security, these values should never be sent in this config, only the locations of + /// the files that the external decryptor will know how to access. + std::unordered_map> + connection_config; +}; + /// This is a core class, that translates the parameters of high level encryption (like /// the names of encrypted columns, names of master keys, etc), into parameters of low /// level encryption (like the key metadata, DEK, etc). A factory that produces the low @@ -105,13 +178,21 @@ class PARQUET_EXPORT CryptoFactory { void RegisterKmsClientFactory(std::shared_ptr kms_client_factory); /// Get the encryption properties for a Parquet file. - /// If external key material is used then a file system and path to the + /// If key material from outside the file is used, then a file system and path to the /// parquet file must be provided. std::shared_ptr GetFileEncryptionProperties( const KmsConnectionConfig& kms_connection_config, const EncryptionConfiguration& encryption_config, const std::string& file_path = "", const std::shared_ptr<::arrow::fs::FileSystem>& file_system = NULLPTR); + /// Get the external encryption properties for a Parquet file. Used when an external encryptor + /// will be used to encrypt the file. + std::shared_ptr GetExternalFileEncryptionProperties( + const KmsConnectionConfig& kms_connection_config, + const ExternalEncryptionConfiguration& external_encryption_config, + const std::string& file_path = "", + const std::shared_ptr<::arrow::fs::FileSystem>& file_system = NULLPTR); + /// Get decryption properties for a Parquet file. /// If external key material is used then a file system and path to the /// parquet file must be provided. @@ -120,6 +201,14 @@ class PARQUET_EXPORT CryptoFactory { const DecryptionConfiguration& decryption_config, const std::string& file_path = "", const std::shared_ptr<::arrow::fs::FileSystem>& file_system = NULLPTR); + /// Get the external decryption properties for a Parquet file. Used when an external decryptor + /// will be used to decrypt the file. + std::shared_ptr GetExternalFileDecryptionProperties( + const KmsConnectionConfig& kms_connection_config, + const ExternalDecryptionConfiguration& external_decryption_config, + const std::string& file_path = "", + const std::shared_ptr<::arrow::fs::FileSystem>& file_system = NULLPTR); + void RemoveCacheEntriesForToken(const std::string& access_token) { key_toolkit_->RemoveCacheEntriesForToken(access_token); } diff --git a/cpp/src/parquet/encryption/crypto_factory_test.cc b/cpp/src/parquet/encryption/crypto_factory_test.cc new file mode 100644 index 000000000000..e229e22a635d --- /dev/null +++ b/cpp/src/parquet/encryption/crypto_factory_test.cc @@ -0,0 +1,257 @@ +#include +#include + +#include "arrow/util/secure_string.h" + +#include "parquet/encryption/crypto_factory.h" +#include "parquet/encryption/file_key_material_store.h" +#include "parquet/encryption/test_encryption_util.h" +#include "parquet/encryption/test_in_memory_kms.h" + +using ::testing::_; +using ::testing::HasSubstr; +using ::testing::Return; +using ::testing::StrEq; + +namespace parquet::encryption::test { + +class CryptoFactoryTest : public ::testing::Test { + + void SetUp() { + key_list_ = BuildKeyMap(kColumnMasterKeyIds, kColumnMasterKeys, kFooterMasterKeyId, + kFooterMasterKey); + crypto_factory_.RegisterKmsClientFactory(std::make_shared( + true, key_list_)); + } + + protected: + std::unordered_map key_list_; + KmsConnectionConfig kms_config_; + CryptoFactory crypto_factory_; +}; + +TEST_F(CryptoFactoryTest, UniformEncryptionAndColumnKeysThrowsException) { + ExternalEncryptionConfiguration config("kf"); + config.uniform_encryption = true; + config.column_keys = "kc1:col1,col2"; + + try { + auto properties = crypto_factory_.GetExternalFileEncryptionProperties(kms_config_, config); + FAIL() << "ParquetException should have been raised"; + } catch (const ParquetException& xcp) { + EXPECT_THAT(xcp.what(), HasSubstr("Cannot set both column encryption and uniform")); + } catch (...) { + FAIL() << "Caught unexpected exception type"; + } +} + +TEST_F(CryptoFactoryTest, UniformEncryptionAndPerColumnEncryptionThrowsException) { + ExternalEncryptionConfiguration config("kf"); + config.uniform_encryption = true; + + std::unordered_map per_column_encryption; + ColumnEncryptionAttributes attributes; + attributes.parquet_cipher = ParquetCipher::AES_GCM_CTR_V1; + attributes.key_id = "kc1"; + per_column_encryption["col1"] = attributes; + config.per_column_encryption = per_column_encryption; + + try { + auto properties = crypto_factory_.GetExternalFileEncryptionProperties(kms_config_, config); + FAIL() << "ParquetException should have been raised"; + } catch (const ParquetException& xcp) { + EXPECT_THAT(xcp.what(), HasSubstr("Cannot set both column encryption and uniform")); + } catch (...) { + FAIL() << "Caught unexpected exception type"; + } +} + +TEST_F(CryptoFactoryTest, NoUniformEncryptionAndNoColumnsThrowsException) { + ExternalEncryptionConfiguration config("kf"); + + try { + auto properties = crypto_factory_.GetExternalFileEncryptionProperties(kms_config_, config); + FAIL() << "ParquetException should have been raised"; + } catch (const ParquetException& xcp) { + EXPECT_THAT(xcp.what(), HasSubstr( + "uniform_encryption must be set or column encryption must be specified in either")); + } catch (...) { + FAIL() << "Caught unexpected exception type"; + } +} + +TEST_F(CryptoFactoryTest, BasicEncryptionConfig) { + ExternalEncryptionConfiguration config("kf"); + config.plaintext_footer = true; + config.column_keys = "kc1:col1,col2;kc2:col3,col4"; + + auto properties = crypto_factory_.GetExternalFileEncryptionProperties(kms_config_, config); + EXPECT_EQ(16, properties->footer_key().size()); + EXPECT_TRUE(properties->footer_key_metadata().size() > 0); + EXPECT_EQ(ParquetCipher::AES_GCM_V1, properties->algorithm().algorithm); + EXPECT_FALSE(properties->encrypted_footer()); + EXPECT_TRUE(properties->encrypted_columns().size() == 4); + + auto column_properties_1 = properties->column_encryption_properties("col1"); + EXPECT_EQ("col1", column_properties_1->column_path()); + EXPECT_TRUE(column_properties_1->is_encrypted()); + EXPECT_FALSE(column_properties_1->is_encrypted_with_footer_key()); + EXPECT_THAT(column_properties_1->key_metadata(), HasSubstr("kc1")); + EXPECT_FALSE(column_properties_1->parquet_cipher().has_value()); + + auto column_properties_2 = properties->column_encryption_properties("col4"); + EXPECT_EQ("col4", column_properties_2->column_path()); + EXPECT_TRUE(column_properties_2->is_encrypted()); + EXPECT_FALSE(column_properties_2->is_encrypted_with_footer_key()); + EXPECT_THAT(column_properties_2->key_metadata(), HasSubstr("kc2")); + EXPECT_FALSE(column_properties_2->parquet_cipher().has_value()); +} + +TEST_F(CryptoFactoryTest, ExternalEncryptionConfig) { + ExternalEncryptionConfiguration config("kf"); + config.plaintext_footer = true; + config.column_keys = "kc3:col3,col4"; + + std::unordered_map per_column_encryption; + ColumnEncryptionAttributes attributes_1; + attributes_1.parquet_cipher = ParquetCipher::AES_GCM_CTR_V1; + attributes_1.key_id = "kc1"; + per_column_encryption["col1"] = attributes_1; + + ColumnEncryptionAttributes attributes_2; + attributes_2.parquet_cipher = ParquetCipher::AES_GCM_V1; + attributes_2.key_id = "kc2"; + per_column_encryption["col2"] = attributes_2; + + config.per_column_encryption = per_column_encryption; + config.app_context = + "{\"user_id\": \"abc123\", \"location\": {\"lat\": 9.7489, \"lon\": -83.7534}}"; + config.connection_config = { + {ParquetCipher::EXTERNAL_DBPA_V1, {{"file_path", "path/to/file"}}} + }; + + auto properties = crypto_factory_.GetExternalFileEncryptionProperties(kms_config_, config); + EXPECT_EQ(16, properties->footer_key().size()); + EXPECT_TRUE(properties->footer_key_metadata().size() > 0); + EXPECT_EQ(ParquetCipher::AES_GCM_V1, properties->algorithm().algorithm); + EXPECT_FALSE(properties->encrypted_footer()); + EXPECT_TRUE(properties->encrypted_columns().size() == 4); + + auto column_properties_1 = properties->column_encryption_properties("col1"); + EXPECT_EQ("col1", column_properties_1->column_path()); + EXPECT_TRUE(column_properties_1->is_encrypted()); + EXPECT_FALSE(column_properties_1->is_encrypted_with_footer_key()); + EXPECT_THAT(column_properties_1->key_metadata(), HasSubstr("kc1")); + EXPECT_TRUE(column_properties_1->parquet_cipher().has_value()); + EXPECT_EQ(ParquetCipher::AES_GCM_CTR_V1, column_properties_1->parquet_cipher().value()); + + auto column_properties_2 = properties->column_encryption_properties("col2"); + EXPECT_EQ("col2", column_properties_2->column_path()); + EXPECT_TRUE(column_properties_2->is_encrypted()); + EXPECT_FALSE(column_properties_2->is_encrypted_with_footer_key()); + EXPECT_THAT(column_properties_2->key_metadata(), HasSubstr("kc2")); + EXPECT_TRUE(column_properties_2->parquet_cipher().has_value()); + EXPECT_EQ(ParquetCipher::AES_GCM_V1, column_properties_2->parquet_cipher().value()); + + auto column_properties_3 = properties->column_encryption_properties("col3"); + EXPECT_EQ("col3", column_properties_3->column_path()); + EXPECT_TRUE(column_properties_3->is_encrypted()); + EXPECT_FALSE(column_properties_3->is_encrypted_with_footer_key()); + EXPECT_THAT(column_properties_3->key_metadata(), HasSubstr("kc3")); + EXPECT_FALSE(column_properties_3->parquet_cipher().has_value()); + + EXPECT_FALSE(properties->app_context().empty()); + EXPECT_FALSE(properties->connection_config().empty()); + EXPECT_EQ(properties->app_context(), config.app_context); + EXPECT_NE(properties->connection_config().find(ParquetCipher::EXTERNAL_DBPA_V1), + properties->connection_config().end()); + EXPECT_EQ(properties->connection_config().find(ParquetCipher::AES_GCM_CTR_V1), + properties->connection_config().end()); + EXPECT_NE(properties->connection_config().at(ParquetCipher::EXTERNAL_DBPA_V1).find("file_path"), + properties->connection_config().at(ParquetCipher::EXTERNAL_DBPA_V1).end()); + EXPECT_EQ(properties->connection_config().at(ParquetCipher::EXTERNAL_DBPA_V1).at("file_path"), + "path/to/file"); +} + +TEST_F(CryptoFactoryTest, ColumnRepeatedInMapsThrowsException) { + ExternalEncryptionConfiguration config("kf"); + config.plaintext_footer = true; + config.column_keys = "kc3:col3,col2"; + + std::unordered_map per_column_encryption; + ColumnEncryptionAttributes attributes_1; + attributes_1.parquet_cipher = ParquetCipher::AES_GCM_CTR_V1; + attributes_1.key_id = "kc1"; + per_column_encryption["col1"] = attributes_1; + + ColumnEncryptionAttributes attributes_2; + attributes_2.parquet_cipher = ParquetCipher::AES_GCM_V1; + attributes_2.key_id = "kc2"; + per_column_encryption["col2"] = attributes_2; + + config.per_column_encryption = per_column_encryption; + + try { + auto properties = crypto_factory_.GetExternalFileEncryptionProperties(kms_config_, config); + FAIL() << "ParquetException should have been raised"; + } catch (const ParquetException& xcp) { + EXPECT_THAT(xcp.what(), HasSubstr("Multiple keys defined for column [col2]")); + } catch (...) { + FAIL() << "Caught unexpected exception type"; + } +} + +TEST_F(CryptoFactoryTest, BasicDecryptionConfig) { + ExternalDecryptionConfiguration config; + config.cache_lifetime_seconds = 600; + + auto properties = crypto_factory_.GetExternalFileDecryptionProperties(kms_config_, config); + EXPECT_TRUE(properties->check_plaintext_footer_integrity()); + EXPECT_TRUE(properties->plaintext_files_allowed()); + EXPECT_THAT(properties->key_retriever(), testing::NotNull()); + EXPECT_TRUE(properties->app_context().empty()); + EXPECT_TRUE(properties->connection_config().empty()); +} + +TEST_F(CryptoFactoryTest, ExternalDecryptionConfig) { + ExternalDecryptionConfiguration config; + config.cache_lifetime_seconds = 600; + config.app_context = + "{\"user_id\": \"abc123\", \"location\": {\"lat\": 9.7489, \"lon\": -83.7534}}"; + config.connection_config = { + {ParquetCipher::EXTERNAL_DBPA_V1, {{"file_path", "path/to/file"}}} + }; + + auto properties = crypto_factory_.GetExternalFileDecryptionProperties(kms_config_, config); + EXPECT_TRUE(properties->check_plaintext_footer_integrity()); + EXPECT_TRUE(properties->plaintext_files_allowed()); + EXPECT_THAT(properties->key_retriever(), testing::NotNull()); + EXPECT_FALSE(properties->app_context().empty()); + EXPECT_FALSE(properties->connection_config().empty()); + EXPECT_EQ(properties->app_context(), config.app_context); + EXPECT_NE(properties->connection_config().find(ParquetCipher::EXTERNAL_DBPA_V1), + properties->connection_config().end()); + EXPECT_EQ(properties->connection_config().find(ParquetCipher::AES_GCM_CTR_V1), + properties->connection_config().end()); + EXPECT_NE(properties->connection_config().at(ParquetCipher::EXTERNAL_DBPA_V1).find("file_path"), + properties->connection_config().at(ParquetCipher::EXTERNAL_DBPA_V1).end()); + EXPECT_EQ(properties->connection_config().at(ParquetCipher::EXTERNAL_DBPA_V1).at("file_path"), + "path/to/file"); +} + +TEST_F(CryptoFactoryTest, ExternalDecryptionConfigWithInvalidAppContextThrowsException) { + ExternalDecryptionConfiguration config; + config.cache_lifetime_seconds = 600; + config.app_context = "invalid_json"; + + try { + auto properties = crypto_factory_.GetExternalFileDecryptionProperties(kms_config_, config); + FAIL() << "ParquetException should have been raised"; + } catch (const ParquetException& xcp) { + EXPECT_THAT(xcp.what(), HasSubstr("App context is not a valid JSON string")); + } catch (...) { + FAIL() << "Caught unexpected exception type"; + } +} + +} // namespace parquet::encryption::test \ No newline at end of file diff --git a/cpp/src/parquet/encryption/decryptor_interface.h b/cpp/src/parquet/encryption/decryptor_interface.h new file mode 100644 index 000000000000..569bad46ddba --- /dev/null +++ b/cpp/src/parquet/encryption/decryptor_interface.h @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "parquet/platform.h" +#include "parquet/encryption/encoding_properties.h" + +namespace parquet::encryption { + +class PARQUET_EXPORT DecryptorInterface { + public: + virtual ~DecryptorInterface() = default; + + /// Signal whether the decryptor can calculate a valid plaintext or ciphertext length before + /// performing decryption or not. If false, a proper sized buffer cannot be allocated before + /// calling the Decrypt method, and Arrow must use this decryptor's DecryptWithManagedBuffer + /// method instead of Decrypt. + [[nodiscard]] virtual bool CanCalculateLengths() const = 0; + + /// Calculate the size of the plaintext for a given ciphertext length. + [[nodiscard]] virtual int32_t PlaintextLength(int32_t ciphertext_len) const = 0; + + /// Calculate the size of the ciphertext for a given plaintext length. + [[nodiscard]] virtual int32_t CiphertextLength(int32_t plaintext_len) const = 0; + + /// Decrypt the ciphertext and leave the results in the plaintext buffer. + /// Most implementations will require the key and aad to be provided, but it is up to + /// each decryptor whether to use them or not. + virtual int32_t Decrypt(::arrow::util::span ciphertext, + ::arrow::util::span key, + ::arrow::util::span aad, + ::arrow::util::span plaintext) = 0; + + /// Decrypt the ciphertext and leave the results in the plaintext buffer. + /// The buffer will be resized to the correct size during decryption. This method is used + /// when the decryptor cannot calculate the plaintext length before decryption. + virtual int32_t DecryptWithManagedBuffer(::arrow::util::span ciphertext, + ::arrow::ResizableBuffer* plaintext) = 0; + + // Some Encryptors may need to understand the page encoding before the encryption process. + // This method will be called from ColumnWriter before invoking the Encrypt method. + virtual void UpdateEncodingProperties(std::unique_ptr encoding_properties) {}; +}; + +} // namespace parquet::encryption diff --git a/cpp/src/parquet/encryption/encoding_properties.cc b/cpp/src/parquet/encryption/encoding_properties.cc new file mode 100644 index 000000000000..f033e9ae5c96 --- /dev/null +++ b/cpp/src/parquet/encryption/encoding_properties.cc @@ -0,0 +1,325 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include "parquet/encryption/encoding_properties.h" +#include "parquet/metadata.h" +#include "parquet/column_page.h" + +namespace parquet::encryption { + +using parquet::ColumnDescriptor; +using parquet::WriterProperties; + +// Private constructor for builder +EncodingProperties::EncodingProperties(const EncodingPropertiesBuilder& builder) + : column_path_(builder.column_path_), + physical_type_(builder.physical_type_), + compression_codec_(builder.compression_codec_), + fixed_length_bytes_(builder.fixed_length_bytes_), + page_type_(builder.page_type_.value()), + page_encoding_(builder.page_encoding_), + data_page_num_values_(builder.data_page_num_values_), + data_page_max_definition_level_(builder.data_page_max_definition_level_), + data_page_max_repetition_level_(builder.data_page_max_repetition_level_), + page_v1_definition_level_encoding_(builder.page_v1_definition_level_encoding_), + page_v1_repetition_level_encoding_(builder.page_v1_repetition_level_encoding_), + page_v2_definition_levels_byte_length_(builder.page_v2_definition_levels_byte_length_), + page_v2_repetition_levels_byte_length_(builder.page_v2_repetition_levels_byte_length_), + page_v2_num_nulls_(builder.page_v2_num_nulls_), + page_v2_is_compressed_(builder.page_v2_is_compressed_) { + } + +// Builder static method +EncodingPropertiesBuilder EncodingProperties::Builder() { + return EncodingPropertiesBuilder(); +} + +// private method to validate the properties +void EncodingProperties::validate() { + // Validate required fields + if ( (!column_path_.has_value()) || column_path_.value().empty()) { + throw std::invalid_argument("ColumnPath is required"); + } + + // page encoding is required for all page types (data and dictionary) + if (!page_encoding_.has_value()) { + throw std::invalid_argument("PageEncoding is required"); + } + + //Validate page-level properties + // first, let's validate common properties across data pages. + // DATA_PAGE is equivalent to DATA_PAGE_V1. + if ((page_type_ == parquet::PageType::DATA_PAGE) || (page_type_ == parquet::PageType::DATA_PAGE_V2)) { + if (!data_page_num_values_.has_value()) { + throw std::invalid_argument("DataPageNumValues is required"); + } + + if (physical_type_ == parquet::Type::type::FIXED_LEN_BYTE_ARRAY) { + if (!fixed_length_bytes_.has_value()) { + throw std::invalid_argument("FixedLengthBytes is required for column page with FIXED_LEN_BYTE_ARRAY data type"); + } + } + + if (fixed_length_bytes_.has_value()) { + if (physical_type_ != parquet::Type::type::FIXED_LEN_BYTE_ARRAY) { + throw std::invalid_argument("FixedLengthBytes is only allowed for FIXED_LEN_BYTE_ARRAY physical type"); + } + } + + // max levels may be required for decoding both data page types + if (!data_page_max_definition_level_.has_value()) { + throw std::invalid_argument("DataPageMaxDefinitionLevel is required"); + } + if (!data_page_max_repetition_level_.has_value()) { + throw std::invalid_argument("DataPageMaxRepetitionLevel is required"); + } + } + + // then, let's validate properties specific to each page type. + // DATA_PAGE is equivalent to DATA_PAGE_V1. + if (page_type_ == parquet::PageType::DATA_PAGE) { + if (!page_v1_definition_level_encoding_.has_value()) { + throw std::invalid_argument("PageV1DefinitionLevelEncoding is required"); + } + if (!page_v1_repetition_level_encoding_.has_value()) { + throw std::invalid_argument("PageV1RepetitionLevelEncoding is required"); + } + } + else if (page_type_ == parquet::PageType::DATA_PAGE_V2) { + if (!page_v2_num_nulls_.has_value()) { + throw std::invalid_argument("PageV2NumNulls is required"); + } + if (!page_v2_definition_levels_byte_length_.has_value()) { + throw std::invalid_argument("PageV2DefinitionLevelsByteLength is required"); + } + if (!page_v2_repetition_levels_byte_length_.has_value()) { + throw std::invalid_argument("PageV2RepetitionLevelsByteLength is required"); + } + if (!page_v2_is_compressed_.has_value()) { + throw std::invalid_argument("PageV2IsCompressed is required"); + } + } + else if (page_type_ == parquet::PageType::DICTIONARY_PAGE) { + // no validations required for DICTIONARY_PAGE + // (the requirement for 'encoding' is satisfied by the page_encoding check above) + } + } //validate() + +std::unique_ptr EncodingProperties::MakeFromMetadata( + //const ColumnChunkMetaDataBuilder* column_chunk_metadata, + const ColumnDescriptor* column_descriptor, + const WriterProperties* writer_properties, + const Page& column_page) { + + EncodingPropertiesBuilder builder; + + builder.ColumnPath(column_descriptor->path()->ToDotString()); + builder.PhysicalType(column_descriptor->physical_type()); + builder.CompressionCodec(writer_properties->compression(column_descriptor->path())); + builder.PageType(column_page.type()); + + // If the physical type is FIXED_LEN_BYTE_ARRAY, propagate the byte width + // from the column descriptor so downstream users (e.g., external agents) + // have access to the length. Validation also requires this for data pages. + if (column_descriptor->physical_type() == parquet::Type::FIXED_LEN_BYTE_ARRAY) { + builder.FixedLengthBytes(column_descriptor->type_length()); + } + + bool is_data_page = (column_page.type() == parquet::PageType::DATA_PAGE) || (column_page.type() == parquet::PageType::DATA_PAGE_V2); + + //properties common to V1 and V2 data pages. + if (is_data_page) { + DataPage data_page = static_cast(column_page); + builder.PageEncoding(data_page.encoding()); + builder.DataPageNumValues(data_page.num_values()); + builder.DataPageMaxDefinitionLevel(column_descriptor->max_definition_level()); + builder.DataPageMaxRepetitionLevel(column_descriptor->max_repetition_level()); + } + + //properties specific to each type of page + if (column_page.type() == parquet::PageType::DATA_PAGE) { + DataPageV1 data_page_v1 = static_cast(column_page); + builder.PageV1DefinitionLevelEncoding(data_page_v1.definition_level_encoding()); + builder.PageV1RepetitionLevelEncoding(data_page_v1.repetition_level_encoding()); + } + else if (column_page.type() == parquet::PageType::DATA_PAGE_V2) { + DataPageV2 data_page_v2 = static_cast(column_page); + builder.PageV2DefinitionLevelsByteLength(data_page_v2.definition_levels_byte_length()); + builder.PageV2RepetitionLevelsByteLength(data_page_v2.repetition_levels_byte_length()); + builder.PageV2NumNulls(data_page_v2.num_nulls()); + builder.PageV2IsCompressed(data_page_v2.is_compressed()); + } + else if (column_page.type() == parquet::PageType::DICTIONARY_PAGE) { + DictionaryPage dict_page = static_cast(column_page); + builder.PageEncoding(dict_page.encoding()); + } + else { + throw std::invalid_argument(std::string("Unknown Page Type:: ") + EnumToString(column_page.type())); + } + + return builder.Build(); +} + +std::map EncodingProperties::ToPropertiesMap() const { + std::map result; + + result["column_path"] = std::string(column_path_.value()); + result["physical_type"] = EnumToString(physical_type_.value()); + result["compression_codec"] = EnumToString(compression_codec_.value()); + result["page_type"] = EnumToString(page_type_); + result["page_encoding"] = EnumToString(page_encoding_.value()); + + if (fixed_length_bytes_.has_value()) { + result["fixed_length_bytes"] = std::to_string(fixed_length_bytes_.value()); + } + + if (page_type_ == parquet::PageType::DATA_PAGE || page_type_ == parquet::PageType::DATA_PAGE_V2) { + result["data_page_max_definition_level"] = std::to_string(data_page_max_definition_level_.value()); + result["data_page_max_repetition_level"] = std::to_string(data_page_max_repetition_level_.value()); + } + + if (page_type_ == parquet::PageType::DATA_PAGE) { //DATA_PAGE_V1 + result["data_page_num_values"] = std::to_string(data_page_num_values_.value()); + result["page_v1_definition_level_encoding"] = EnumToString(page_v1_definition_level_encoding_.value()); + result["page_v1_repetition_level_encoding"] = EnumToString(page_v1_repetition_level_encoding_.value()); + } + else if (page_type_ == parquet::PageType::DATA_PAGE_V2) { + result["data_page_num_values"] = std::to_string(data_page_num_values_.value()); + result["page_v2_definition_levels_byte_length"] = std::to_string(page_v2_definition_levels_byte_length_.value()); + result["page_v2_repetition_levels_byte_length"] = std::to_string(page_v2_repetition_levels_byte_length_.value()); + result["page_v2_num_nulls"] = std::to_string(page_v2_num_nulls_.value()); + result["page_v2_is_compressed"] = (page_v2_is_compressed_.value() ? "true" : "false"); + } + else if (page_type_ == parquet::PageType::DICTIONARY_PAGE) { + // no other properties are set for DICTIONARY_PAGE + } + + return result; +} + +//-------------------------------- +// Builder method implementations + +std::unique_ptr EncodingPropertiesBuilder::Build() { + // while we will perform validation upon construction, + // we know that these properties are required. + // validating here simplifies our code. + + if (!page_type_) { + throw std::invalid_argument("EncodingPropertiesBuilder::Build - PageType is required"); + } + + return std::unique_ptr(new EncodingProperties(*this)); +} + +EncodingPropertiesBuilder& EncodingPropertiesBuilder::ColumnPath(const std::string& column_path) { + column_path_ = column_path; + return *this; +} + +EncodingPropertiesBuilder& EncodingPropertiesBuilder::PhysicalType(parquet::Type::type physical_type) { + physical_type_ = physical_type; + return *this; +} + +EncodingPropertiesBuilder& EncodingPropertiesBuilder::CompressionCodec(::arrow::Compression::type compression_codec) { + compression_codec_ = compression_codec; + return *this; +} + +EncodingPropertiesBuilder& EncodingPropertiesBuilder::FixedLengthBytes(std::int64_t fixed_length_bytes) { + fixed_length_bytes_ = fixed_length_bytes; + return *this; +} + +EncodingPropertiesBuilder& EncodingPropertiesBuilder::PageType(parquet::PageType::type page_type) { + page_type_ = page_type; + return *this; +} + +EncodingPropertiesBuilder& EncodingPropertiesBuilder::PageEncoding(parquet::Encoding::type page_encoding) { + page_encoding_ = page_encoding; + return *this; +} + +EncodingPropertiesBuilder& EncodingPropertiesBuilder::DataPageNumValues(int64_t data_page_num_values) { + data_page_num_values_ = data_page_num_values; + return *this; +} + +EncodingPropertiesBuilder& EncodingPropertiesBuilder::PageV1DefinitionLevelEncoding(parquet::Encoding::type encoding) { + page_v1_definition_level_encoding_ = encoding; + return *this; +} + +EncodingPropertiesBuilder& EncodingPropertiesBuilder::PageV1RepetitionLevelEncoding(parquet::Encoding::type encoding) { + page_v1_repetition_level_encoding_ = encoding; + return *this; +} + +EncodingPropertiesBuilder& EncodingPropertiesBuilder::DataPageMaxDefinitionLevel(int16_t level) { + data_page_max_definition_level_ = level; + return *this; +} + +EncodingPropertiesBuilder& EncodingPropertiesBuilder::DataPageMaxRepetitionLevel(int16_t level) { + data_page_max_repetition_level_ = level; + return *this; +} + +EncodingPropertiesBuilder& EncodingPropertiesBuilder::PageV2DefinitionLevelsByteLength(int32_t byte_length) { + page_v2_definition_levels_byte_length_ = byte_length; + return *this; +} + +EncodingPropertiesBuilder& EncodingPropertiesBuilder::PageV2RepetitionLevelsByteLength(int32_t byte_length) { + page_v2_repetition_levels_byte_length_ = byte_length; + return *this; +} + +EncodingPropertiesBuilder& EncodingPropertiesBuilder::PageV2NumNulls(int32_t num_nulls) { + page_v2_num_nulls_ = num_nulls; + return *this; +} + +EncodingPropertiesBuilder& EncodingPropertiesBuilder::PageV2IsCompressed(bool is_compressed) { + page_v2_is_compressed_ = is_compressed; + return *this; +} + +//-------------------------------- +// Setters for column-level properties +// used to fill-in values provided in the encryptor/decryptor constructor. +void EncodingProperties::set_column_path(const std::string& column_path) { + column_path_ = column_path; +} + +void EncodingProperties::set_compression_codec(::arrow::Compression::type compression_codec) { + compression_codec_ = compression_codec; +} + +void EncodingProperties::set_physical_type(parquet::Type::type physical_type, + const std::optional& fixed_length_bytes) { + physical_type_ = physical_type; + if (fixed_length_bytes.has_value()) { + fixed_length_bytes_ = fixed_length_bytes; + } +} + +} // namespace parquet::encryption diff --git a/cpp/src/parquet/encryption/encoding_properties.h b/cpp/src/parquet/encryption/encoding_properties.h new file mode 100644 index 000000000000..01335ef3afdc --- /dev/null +++ b/cpp/src/parquet/encryption/encoding_properties.h @@ -0,0 +1,251 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "parquet/types.h" +#include "parquet/encoding.h" +#include "parquet/metadata.h" +#include "parquet/column_page.h" +#include "arrow/util/compression.h" + +namespace parquet::encryption { + +class EncodingPropertiesBuilder; + +class EncodingProperties { +public: + static std::unique_ptr MakeFromMetadata( + const ColumnDescriptor* column_descriptor, + const WriterProperties* writer_properties, + const Page& column_page); + + + // Builder pattern + static EncodingPropertiesBuilder Builder(); + + // Setters for column-level properties + void set_column_path(const std::string& column_path); + void set_physical_type(parquet::Type::type physical_type, + const std::optional& fixed_length_bytes = std::nullopt); + void set_compression_codec(::arrow::Compression::type compression_codec); + + void validate(); + + std::map ToPropertiesMap() const; + + // Lightweight accessor to avoid building maps when only page type is needed + parquet::PageType::type GetPageType() const { return page_type_; } + +private: + // Private constructor for builder + EncodingProperties(const EncodingPropertiesBuilder& builder); + + EncodingProperties( + std::optional column_path, + std::optional physical_type, + std::optional<::arrow::Compression::type> compression_codec, + std::int64_t fixed_length_bytes, + parquet::PageType::type page_type, + parquet::Encoding::type page_encoding, + int64_t data_page_num_values, + parquet::Encoding::type page_v1_definition_level_encoding, + parquet::Encoding::type page_v1_repetition_level_encoding, + int32_t page_v2_definition_levels_byte_length, + int32_t page_v2_repetition_levels_byte_length, + int32_t page_v2_num_nulls, + bool page_v2_is_compressed + ); + + // Allow the builder to access private constructor + friend class EncodingPropertiesBuilder; + + //-------------------------------- + //from column metadata. does not change across chunks nor data pages. + std::optional column_path_; + std::optional physical_type_; // BOOLEAN, INT32, INT64, INT96, FLOAT, DOUBLE, BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY, etc + std::optional<::arrow::Compression::type> compression_codec_; + + std::optional fixed_length_bytes_; // for FIXED_LEN_BYTE_ARRAY + + //-------------------------------- + //page type. - applies across all types of pages. non optional. + parquet::PageType::type page_type_; // V1, V2, DICTIONARY_PAGE + + //-------------------------------- + //from data page. changes across chunks and data pages. + //page-level properties can be seen in parquet/column_page.h + std::optional page_encoding_; + + //common between V1 and V2 data pages. + std::optional data_page_num_values_; + std::optional data_page_max_definition_level_; + std::optional data_page_max_repetition_level_; + + //-------------------------------- + // V1 data page properties. + std::optional page_v1_definition_level_encoding_; + std::optional page_v1_repetition_level_encoding_; + + //-------------------------------- + // V2 data page properties. + std::optional page_v2_definition_levels_byte_length_; //note that typing is different from V1 + std::optional page_v2_repetition_levels_byte_length_; //note that typing is different from V1 + std::optional page_v2_num_nulls_; + std::optional page_v2_is_compressed_; //this does not exist in V1 nor dictionary pages. + + //-------------------------------- + // Dictionary page properties. + + // there are not specific properties for dictionary pages, + // other than the page encoding (captured above). + + //-------------------------------- +}; //class EncodingProperties + +class EncodingPropertiesBuilder { +public: + EncodingPropertiesBuilder() = default; + + // Column-level properties (required) + EncodingPropertiesBuilder& ColumnPath(const std::string& column_path); + EncodingPropertiesBuilder& PhysicalType(parquet::Type::type physical_type); + EncodingPropertiesBuilder& CompressionCodec(::arrow::Compression::type compression_codec); + EncodingPropertiesBuilder& PageType(parquet::PageType::type page_type); + + // Column-level optional fields + EncodingPropertiesBuilder& FixedLengthBytes(std::int64_t fixed_length_bytes); + + // Data page properties + EncodingPropertiesBuilder& PageEncoding(parquet::Encoding::type page_encoding); + EncodingPropertiesBuilder& DataPageNumValues(int64_t data_page_num_values); + + // V1 data page properties + EncodingPropertiesBuilder& PageV1DefinitionLevelEncoding(parquet::Encoding::type encoding); + EncodingPropertiesBuilder& PageV1RepetitionLevelEncoding(parquet::Encoding::type encoding); + + // Data page common properties (apply to V1 and V2) + EncodingPropertiesBuilder& DataPageMaxDefinitionLevel(int16_t level); + EncodingPropertiesBuilder& DataPageMaxRepetitionLevel(int16_t level); + + // V2 data page properties + EncodingPropertiesBuilder& PageV2DefinitionLevelsByteLength(int32_t byte_length); + EncodingPropertiesBuilder& PageV2RepetitionLevelsByteLength(int32_t byte_length); + EncodingPropertiesBuilder& PageV2NumNulls(int32_t num_nulls); + EncodingPropertiesBuilder& PageV2IsCompressed(bool is_compressed); + + // Build the final object + std::unique_ptr Build(); + +private: + friend class EncodingProperties; + + // Required fields + std::optional column_path_; + std::optional physical_type_; + std::optional<::arrow::Compression::type> compression_codec_; + std::optional page_type_; + + // column metadata + std::optional fixed_length_bytes_; + + // data page properties + std::optional page_encoding_; + std::optional data_page_num_values_; + std::optional data_page_max_definition_level_; + std::optional data_page_max_repetition_level_; + + // V1 data page properties + std::optional page_v1_definition_level_encoding_; + std::optional page_v1_repetition_level_encoding_; + + // V2 data page properties + std::optional page_v2_definition_levels_byte_length_; + std::optional page_v2_repetition_levels_byte_length_; + std::optional page_v2_num_nulls_; + std::optional page_v2_is_compressed_; +}; // class EncodingPropertiesBuilder + +//-------------------------------- +// Enum to string helpers +// Kept inline in the header for convenient reuse and zero-link overhead. +inline std::string EnumToString(parquet::Type::type t) { + switch (t) { + case parquet::Type::BOOLEAN: return "BOOLEAN"; + case parquet::Type::INT32: return "INT32"; + case parquet::Type::INT64: return "INT64"; + case parquet::Type::INT96: return "INT96"; + case parquet::Type::FLOAT: return "FLOAT"; + case parquet::Type::DOUBLE: return "DOUBLE"; + case parquet::Type::BYTE_ARRAY: return "BYTE_ARRAY"; + case parquet::Type::FIXED_LEN_BYTE_ARRAY: return "FIXED_LEN_BYTE_ARRAY"; + case parquet::Type::UNDEFINED: return "UNDEFINED"; + default: throw std::invalid_argument(std::string("Unknown parquet Type:: ") + std::to_string(t)); + } +} + +inline std::string EnumToString(::arrow::Compression::type t) { + // Use uppercase names for consistency with other enums + switch (t) { + case ::arrow::Compression::UNCOMPRESSED: return "UNCOMPRESSED"; + case ::arrow::Compression::SNAPPY: return "SNAPPY"; + case ::arrow::Compression::GZIP: return "GZIP"; + case ::arrow::Compression::BROTLI: return "BROTLI"; + case ::arrow::Compression::ZSTD: return "ZSTD"; + case ::arrow::Compression::LZ4: return "LZ4"; + case ::arrow::Compression::LZ4_FRAME: return "LZ4_FRAME"; + case ::arrow::Compression::LZO: return "LZO"; + case ::arrow::Compression::BZ2: return "BZ2"; + case ::arrow::Compression::LZ4_HADOOP: return "LZ4_HADOOP"; + default: throw std::invalid_argument(std::string("Unknown arrow Compression::type:: ") + std::to_string(t)); + } +} + +inline std::string EnumToString(parquet::Encoding::type t) { + switch (t) { + case parquet::Encoding::PLAIN: return "PLAIN"; + case parquet::Encoding::PLAIN_DICTIONARY: return "PLAIN_DICTIONARY"; + case parquet::Encoding::RLE: return "RLE"; + case parquet::Encoding::BIT_PACKED: return "BIT_PACKED"; + case parquet::Encoding::DELTA_BINARY_PACKED: return "DELTA_BINARY_PACKED"; + case parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY: return "DELTA_LENGTH_BYTE_ARRAY"; + case parquet::Encoding::DELTA_BYTE_ARRAY: return "DELTA_BYTE_ARRAY"; + case parquet::Encoding::RLE_DICTIONARY: return "RLE_DICTIONARY"; + case parquet::Encoding::BYTE_STREAM_SPLIT: return "BYTE_STREAM_SPLIT"; + case parquet::Encoding::UNDEFINED: return "UNDEFINED"; + case parquet::Encoding::UNKNOWN: return "UNKNOWN"; + default: throw std::invalid_argument(std::string("Unknown parquet Encoding::type:: ") + std::to_string(t)); + } +} + +inline std::string EnumToString(parquet::PageType::type t) { + switch (t) { + case parquet::PageType::DATA_PAGE: return "DATA_PAGE_V1"; + case parquet::PageType::DATA_PAGE_V2: return "DATA_PAGE_V2"; + case parquet::PageType::DICTIONARY_PAGE: return "DICTIONARY_PAGE"; + case parquet::PageType::INDEX_PAGE: return "INDEX_PAGE"; + case parquet::PageType::UNDEFINED: return "UNDEFINED"; + default: throw std::invalid_argument(std::string("Unknown parquet PageType::type:: ") + std::to_string(t)); + } +} + +} //namespace parquet::encryption diff --git a/cpp/src/parquet/encryption/encoding_properties_test.cc b/cpp/src/parquet/encryption/encoding_properties_test.cc new file mode 100644 index 000000000000..062ece329c67 --- /dev/null +++ b/cpp/src/parquet/encryption/encoding_properties_test.cc @@ -0,0 +1,454 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include + +#include + +#include "parquet/column_page.h" +#include "parquet/types.h" +#include "parquet/properties.h" +#include "parquet/platform.h" +#include "parquet/encryption/encoding_properties.h" +#include "parquet/schema.h" + +namespace parquet::encryption::test { + +using ::parquet::Encoding; +using ::parquet::PageType; +using ::parquet::Type; +using ::parquet::ColumnDescriptor; + +static std::shared_ptr<::parquet::SchemaDescriptor> MakeSingleInt32Schema( + const std::string& col_name = "col") { + using ::parquet::schema::GroupNode; + using ::parquet::schema::NodePtr; + using ::parquet::schema::NodeVector; + using ::parquet::schema::PrimitiveNode; + + NodeVector fields; + fields.push_back(PrimitiveNode::Make(col_name, ::parquet::Repetition::REQUIRED, + Type::INT32)); + NodePtr schema = GroupNode::Make("schema", ::parquet::Repetition::REQUIRED, fields); + + auto descr = std::make_shared<::parquet::SchemaDescriptor>(); + descr->Init(schema); + return descr; +} + +TEST(EncodingPropertiesTest, BuilderRequiresPageType) { + auto builder = EncodingProperties::Builder(); + builder.ColumnPath("a"); + builder.PhysicalType(Type::INT32); + builder.CompressionCodec(::arrow::Compression::SNAPPY); + EXPECT_THROW(builder.Build(), std::invalid_argument); +} + +TEST(EncodingPropertiesTest, DictionaryPageSettersAndToMap) { + auto props = EncodingProperties::Builder() + .PageType(PageType::DICTIONARY_PAGE) + .PageEncoding(Encoding::PLAIN) + .Build(); + + props->set_column_path("schema.col"); + props->set_physical_type(Type::DOUBLE, std::nullopt); + props->set_compression_codec(::arrow::Compression::ZSTD); + + EXPECT_NO_THROW(props->validate()); + + auto m = props->ToPropertiesMap(); + ASSERT_EQ(m.at("column_path"), std::string("schema.col")); + ASSERT_EQ(m.at("physical_type"), std::string("DOUBLE")); + ASSERT_EQ(m.at("compression_codec"), std::string("ZSTD")); + ASSERT_EQ(m.at("page_type"), std::string("DICTIONARY_PAGE")); + ASSERT_EQ(m.at("page_encoding"), std::string("PLAIN")); +} + +TEST(EncodingPropertiesTest, BuilderDataPageV1ValidationSuccessAndMap) { + auto props = EncodingProperties::Builder() + .ColumnPath("col") + .PhysicalType(Type::INT32) + .CompressionCodec(::arrow::Compression::SNAPPY) + .PageType(PageType::DATA_PAGE) + .PageEncoding(Encoding::PLAIN) + .DataPageNumValues(123) + .PageV1DefinitionLevelEncoding(Encoding::RLE) + .PageV1RepetitionLevelEncoding(Encoding::RLE) + .DataPageMaxDefinitionLevel(1) + .DataPageMaxRepetitionLevel(0) + .Build(); + + EXPECT_NO_THROW(props->validate()); + + auto m = props->ToPropertiesMap(); + ASSERT_EQ(m.at("column_path"), std::string("col")); + ASSERT_EQ(m.at("physical_type"), std::string("INT32")); + ASSERT_EQ(m.at("compression_codec"), std::string("SNAPPY")); + ASSERT_EQ(m.at("page_type"), std::string("DATA_PAGE_V1")); + ASSERT_EQ(m.at("page_encoding"), std::string("PLAIN")); + ASSERT_EQ(m.at("data_page_num_values"), std::to_string(123)); + ASSERT_EQ(m.at("data_page_max_definition_level"), std::to_string(1)); + ASSERT_EQ(m.at("data_page_max_repetition_level"), std::to_string(0)); + ASSERT_EQ(m.at("page_v1_definition_level_encoding"), std::string("RLE")); + ASSERT_EQ(m.at("page_v1_repetition_level_encoding"), std::string("RLE")); +} + +TEST(EncodingPropertiesTest, BuilderDataPageV2ValidationSuccessAndMap) { + auto props = EncodingProperties::Builder() + .ColumnPath("col") + .PhysicalType(Type::BYTE_ARRAY) + .CompressionCodec(::arrow::Compression::ZSTD) + .PageType(PageType::DATA_PAGE_V2) + .PageEncoding(Encoding::DELTA_LENGTH_BYTE_ARRAY) + .DataPageNumValues(42) + .PageV2DefinitionLevelsByteLength(8) + .PageV2RepetitionLevelsByteLength(4) + .PageV2NumNulls(5) + .PageV2IsCompressed(true) + .DataPageMaxDefinitionLevel(1) + .DataPageMaxRepetitionLevel(0) + .Build(); + + EXPECT_NO_THROW(props->validate()); + auto m = props->ToPropertiesMap(); + ASSERT_EQ(m.at("column_path"), std::string("col")); + ASSERT_EQ(m.at("physical_type"), std::string("BYTE_ARRAY")); + ASSERT_EQ(m.at("compression_codec"), std::string("ZSTD")); + ASSERT_EQ(m.at("page_type"), std::string("DATA_PAGE_V2")); + ASSERT_EQ(m.at("page_encoding"), std::string("DELTA_LENGTH_BYTE_ARRAY")); + ASSERT_EQ(m.at("data_page_num_values"), std::to_string(42)); + ASSERT_EQ(m.at("data_page_max_definition_level"), std::to_string(1)); + ASSERT_EQ(m.at("data_page_max_repetition_level"), std::to_string(0)); + ASSERT_EQ(m.at("page_v2_definition_levels_byte_length"), std::to_string(8)); + ASSERT_EQ(m.at("page_v2_repetition_levels_byte_length"), std::to_string(4)); + ASSERT_EQ(m.at("page_v2_num_nulls"), std::to_string(5)); + ASSERT_EQ(m.at("page_v2_is_compressed"), "true"); +} + +TEST(EncodingPropertiesTest, BuilderDataPageV2MissingFieldsValidationFails) { + auto props = EncodingProperties::Builder() + .ColumnPath("col") + .PhysicalType(Type::INT32) + .CompressionCodec(::arrow::Compression::GZIP) + .PageType(PageType::DATA_PAGE_V2) + .PageEncoding(Encoding::DELTA_BINARY_PACKED) + .DataPageNumValues(10) + .PageV2DefinitionLevelsByteLength(4) + .PageV2RepetitionLevelsByteLength(4) + // Intentionally omit PageV2NumNulls and PageV2IsCompressed + .Build(); + + EXPECT_THROW(props->validate(), std::invalid_argument); +} + +TEST(EncodingPropertiesTest, FixedLengthBytesWrongUsageThrows) { + auto props = EncodingProperties::Builder() + .ColumnPath("col") + .PhysicalType(Type::INT32) + .CompressionCodec(::arrow::Compression::SNAPPY) + .PageType(PageType::DATA_PAGE) + .PageEncoding(Encoding::PLAIN) + .DataPageNumValues(5) + .PageV1DefinitionLevelEncoding(Encoding::RLE) + .PageV1RepetitionLevelEncoding(Encoding::RLE) + .Build(); + + // Set a fixed length while physical type is not FIXED_LEN_BYTE_ARRAY + props->set_physical_type(Type::INT32, std::optional(16)); + EXPECT_THROW(props->validate(), std::invalid_argument); +} + +TEST(EncodingPropertiesTest, BuilderOptionalFixedLengthBytesAndToMap) { + auto props = EncodingProperties::Builder() + .PageType(PageType::DICTIONARY_PAGE) + .PageEncoding(Encoding::RLE_DICTIONARY) + .Build(); + props->set_column_path("fixed_col"); + props->set_physical_type(Type::FIXED_LEN_BYTE_ARRAY, std::optional(16)); + props->set_compression_codec(::arrow::Compression::LZ4); + + EXPECT_NO_THROW(props->validate()); + auto m = props->ToPropertiesMap(); + ASSERT_EQ(m.at("fixed_length_bytes"), std::to_string(16)); +} + +TEST(EncodingPropertiesTest, MissingPageEncodingThrows) { + auto props = EncodingProperties::Builder() + .PageType(PageType::DICTIONARY_PAGE) + .Build(); + props->set_column_path("schema.col"); + + EXPECT_THROW(props->validate(), std::invalid_argument); +} + +TEST(EncodingPropertiesTest, MakeFromMetadataDataPageV1) { + auto schema = MakeSingleInt32Schema(); + const ColumnDescriptor* descr = schema->Column(0); + + // WriterProperties with explicit compression for path "col" + auto path = parquet::schema::ColumnPath::FromDotString("col"); + parquet::WriterProperties::Builder wp_builder; + wp_builder.compression(path, ::arrow::Compression::SNAPPY); + auto writer_props = wp_builder.build(); + + // Build a V1 data page + auto buffer = ::parquet::AllocateBuffer(); + parquet::DataPageV1 page(buffer, /*num_values=*/7, Encoding::PLAIN, Encoding::RLE, + Encoding::RLE, /*uncompressed_size=*/0); + + auto props = EncodingProperties::MakeFromMetadata(descr, writer_props.get(), page); + // Should be valid and have all keys + EXPECT_NO_THROW(props->validate()); + auto m = props->ToPropertiesMap(); + ASSERT_EQ(m.at("column_path"), std::string("col")); + ASSERT_EQ(m.at("physical_type"), std::string("INT32")); + ASSERT_EQ(m.at("compression_codec"), std::string("SNAPPY")); + ASSERT_EQ(m.at("page_type"), std::string("DATA_PAGE_V1")); + ASSERT_EQ(m.at("page_encoding"), std::string("PLAIN")); + ASSERT_EQ(m.at("data_page_num_values"), std::to_string(7)); + ASSERT_EQ(m.at("page_v1_definition_level_encoding"), std::string("RLE")); + ASSERT_EQ(m.at("page_v1_repetition_level_encoding"), std::string("RLE")); + ASSERT_EQ(m.at("data_page_max_definition_level"), std::to_string(0)); + ASSERT_EQ(m.at("data_page_max_repetition_level"), std::to_string(0)); +} + +TEST(EncodingPropertiesTest, MakeFromMetadataDictionaryPage) { + auto schema = MakeSingleInt32Schema(); + const ColumnDescriptor* descr = schema->Column(0); + + auto path = parquet::schema::ColumnPath::FromDotString("col"); + parquet::WriterProperties::Builder wp_builder; + wp_builder.compression(path, ::arrow::Compression::GZIP); + auto writer_props = wp_builder.build(); + + auto buffer = ::parquet::AllocateBuffer(); + parquet::DictionaryPage page(buffer, /*num_values=*/4, Encoding::RLE_DICTIONARY); + + auto props = EncodingProperties::MakeFromMetadata(descr, writer_props.get(), page); + EXPECT_NO_THROW(props->validate()); + auto m = props->ToPropertiesMap(); + ASSERT_EQ(m.at("column_path"), std::string("col")); + ASSERT_EQ(m.at("physical_type"), std::string("INT32")); + ASSERT_EQ(m.at("compression_codec"), std::string("GZIP")); + ASSERT_EQ(m.at("page_type"), std::string("DICTIONARY_PAGE")); + ASSERT_EQ(m.at("page_encoding"), std::string("RLE_DICTIONARY")); +} + +TEST(EncodingPropertiesTest, MakeFromMetadataDataPageV2ValidationAndMap) { + auto schema = MakeSingleInt32Schema(); + const ColumnDescriptor* descr = schema->Column(0); + + auto path = parquet::schema::ColumnPath::FromDotString("col"); + parquet::WriterProperties::Builder wp_builder; + wp_builder.compression(path, ::arrow::Compression::ZSTD); + auto writer_props = wp_builder.build(); + + auto buffer = ::parquet::AllocateBuffer(); + parquet::DataPageV2 page(buffer, /*num_values=*/5, /*num_nulls=*/2, /*num_rows=*/5, + Encoding::DELTA_BYTE_ARRAY, + /*definition_levels_byte_length=*/3, + /*repetition_levels_byte_length=*/2, + /*uncompressed_size=*/0, /*is_compressed=*/true); + + auto props = EncodingProperties::MakeFromMetadata(descr, writer_props.get(), page); + EXPECT_NO_THROW(props->validate()); + + auto m = props->ToPropertiesMap(); + ASSERT_EQ(m.at("column_path"), std::string("col")); + ASSERT_EQ(m.at("physical_type"), std::string("INT32")); + ASSERT_EQ(m.at("compression_codec"), std::string("ZSTD")); + ASSERT_EQ(m.at("page_type"), std::string("DATA_PAGE_V2")); + ASSERT_EQ(m.at("page_encoding"), std::string("DELTA_BYTE_ARRAY")); + ASSERT_EQ(m.at("data_page_num_values"), std::to_string(5)); + ASSERT_EQ(m.at("data_page_max_definition_level"), std::to_string(0)); + ASSERT_EQ(m.at("data_page_max_repetition_level"), std::to_string(0)); + ASSERT_EQ(m.at("page_v2_definition_levels_byte_length"), std::to_string(3)); + ASSERT_EQ(m.at("page_v2_repetition_levels_byte_length"), std::to_string(2)); + ASSERT_EQ(m.at("page_v2_num_nulls"), std::to_string(2)); + ASSERT_EQ(m.at("page_v2_is_compressed"), "true"); +} + +TEST(EncodingPropertiesTest, MakeFromMetadataUnknownPageTypeThrows) { + auto schema = MakeSingleInt32Schema(); + const ColumnDescriptor* descr = schema->Column(0); + + auto path = parquet::schema::ColumnPath::FromDotString("col"); + parquet::WriterProperties::Builder wp_builder; + wp_builder.compression(path, ::arrow::Compression::SNAPPY); + auto writer_props = wp_builder.build(); + + auto buffer = ::parquet::AllocateBuffer(); + parquet::Page index_page(buffer, PageType::INDEX_PAGE); + + EXPECT_THROW(EncodingProperties::MakeFromMetadata(descr, writer_props.get(), index_page), + std::invalid_argument); +} + +TEST(EncodingPropertiesTest, MakeFromMetadataFixedLenByteArrayPropagatesLength) { + using ::parquet::schema::GroupNode; + using ::parquet::schema::NodePtr; + using ::parquet::schema::NodeVector; + using ::parquet::schema::PrimitiveNode; + + // Build a schema with a FIXED_LEN_BYTE_ARRAY(16) column named "col" + NodeVector fields; + fields.push_back(PrimitiveNode::Make("col", ::parquet::Repetition::REQUIRED, + Type::FIXED_LEN_BYTE_ARRAY, + ::parquet::ConvertedType::NONE, + /*type_length=*/16)); + NodePtr schema = GroupNode::Make("schema", ::parquet::Repetition::REQUIRED, fields); + auto descr = std::make_shared<::parquet::SchemaDescriptor>(); + descr->Init(schema); + + const ColumnDescriptor* col_descr = descr->Column(0); + + // WriterProperties for path "col" + auto path = parquet::schema::ColumnPath::FromDotString("col"); + parquet::WriterProperties::Builder wp_builder; + wp_builder.compression(path, ::arrow::Compression::SNAPPY); + auto writer_props = wp_builder.build(); + + // Use a DATA_PAGE_V1 with minimal valid settings + auto buffer = ::parquet::AllocateBuffer(); + parquet::DataPageV1 page(buffer, /*num_values=*/3, Encoding::PLAIN, Encoding::RLE, + Encoding::RLE, /*uncompressed_size=*/0); + + auto props = EncodingProperties::MakeFromMetadata(col_descr, writer_props.get(), page); + EXPECT_NO_THROW(props->validate()); + auto m = props->ToPropertiesMap(); + ASSERT_EQ(m.at("column_path"), std::string("col")); + ASSERT_EQ(m.at("physical_type"), std::string("FIXED_LEN_BYTE_ARRAY")); + ASSERT_EQ(m.at("compression_codec"), std::string("SNAPPY")); + ASSERT_EQ(m.at("page_type"), std::string("DATA_PAGE_V1")); + ASSERT_EQ(m.at("page_encoding"), std::string("PLAIN")); + ASSERT_EQ(m.at("fixed_length_bytes"), std::to_string(16)); +} + +TEST(EncodingPropertiesTest, DataPageV1MissingNumValuesThrows) { + auto props = EncodingProperties::Builder() + .ColumnPath("col") + .PhysicalType(Type::DOUBLE) + .CompressionCodec(::arrow::Compression::SNAPPY) + .PageType(PageType::DATA_PAGE) + .PageEncoding(Encoding::PLAIN) + .PageV1DefinitionLevelEncoding(Encoding::RLE) + .PageV1RepetitionLevelEncoding(Encoding::RLE) + .Build(); + + EXPECT_THROW(props->validate(), std::invalid_argument); +} + +TEST(EncodingPropertiesTest, SequentialFailuresForDataPageV1RequiredFields) { + // Start with minimal setup for a DATA_PAGE and add required fields step by step + auto builder = EncodingProperties::Builder() + .ColumnPath("col") + .PhysicalType(Type::INT32) + .CompressionCodec(::arrow::Compression::SNAPPY) + .PageType(PageType::DATA_PAGE) + .PageEncoding(Encoding::PLAIN); + + auto props = builder.Build(); + EXPECT_THROW(props->validate(), std::invalid_argument); // missing num_values + + builder.DataPageNumValues(10); + props = builder.Build(); + EXPECT_THROW(props->validate(), std::invalid_argument); // missing max def level + + builder.DataPageMaxDefinitionLevel(0); + props = builder.Build(); + EXPECT_THROW(props->validate(), std::invalid_argument); // missing max rep level + + builder.DataPageMaxRepetitionLevel(0); + props = builder.Build(); + EXPECT_THROW(props->validate(), std::invalid_argument); // missing V1 deflvl encoding + + builder.PageV1DefinitionLevelEncoding(Encoding::RLE); + props = builder.Build(); + EXPECT_THROW(props->validate(), std::invalid_argument); // missing V1 replvl encoding + + builder.PageV1RepetitionLevelEncoding(Encoding::RLE); + props = builder.Build(); + EXPECT_NO_THROW(props->validate()); +} + +TEST(EncodingPropertiesTest, SequentialFailuresForDataPageV2RequiredFields) { + // Start with minimal setup for a DATA_PAGE_V2 and add required fields step by step + auto builder = EncodingProperties::Builder() + .ColumnPath("col") + .PhysicalType(Type::INT32) + .CompressionCodec(::arrow::Compression::ZSTD) + .PageType(PageType::DATA_PAGE_V2) + .PageEncoding(Encoding::DELTA_BINARY_PACKED); + + auto props = builder.Build(); + EXPECT_THROW(props->validate(), std::invalid_argument); // missing num_values + + builder.DataPageNumValues(5); + props = builder.Build(); + EXPECT_THROW(props->validate(), std::invalid_argument); // missing max def level + + builder.DataPageMaxDefinitionLevel(0); + props = builder.Build(); + EXPECT_THROW(props->validate(), std::invalid_argument); // missing max rep level + + builder.DataPageMaxRepetitionLevel(0); + props = builder.Build(); + EXPECT_THROW(props->validate(), std::invalid_argument); // missing v2 num_nulls + + builder.PageV2NumNulls(2); + props = builder.Build(); + EXPECT_THROW(props->validate(), std::invalid_argument); // missing v2 deflvl bytes + + builder.PageV2DefinitionLevelsByteLength(3); + props = builder.Build(); + EXPECT_THROW(props->validate(), std::invalid_argument); // missing v2 replvl bytes + + builder.PageV2RepetitionLevelsByteLength(2); + props = builder.Build(); + EXPECT_THROW(props->validate(), std::invalid_argument); // missing v2 is_compressed + + builder.PageV2IsCompressed(true); + props = builder.Build(); + EXPECT_NO_THROW(props->validate()); +} + +TEST(EncodingPropertiesTest, SequentialFailuresForDictionaryPageRequiredFields) { + // Dictionary pages require: column path, physical type, compression codec, page type, + // and page encoding. We'll add them incrementally. + auto builder = EncodingProperties::Builder(); + + // Missing everything => Build should throw because page type is required at build time + EXPECT_THROW(builder.Build(), std::invalid_argument); + + builder.PageType(PageType::DICTIONARY_PAGE); + auto props = builder.Build(); + // Now validation fails due to missing encoding and column-level properties + EXPECT_THROW(props->validate(), std::invalid_argument); // missing encoding + + builder.PageEncoding(Encoding::RLE_DICTIONARY); + props = builder.Build(); + // Still missing column/path + EXPECT_THROW(props->validate(), std::invalid_argument); // missing column path + + props->set_column_path("schema.col"); + // Dictionary page doesn't require physical type or compression to validate + EXPECT_NO_THROW(props->validate()); +} + +} // namespace parquet::encryption::test diff --git a/cpp/src/parquet/encryption/encryption.cc b/cpp/src/parquet/encryption/encryption.cc index 52c809aa2f3b..6ee8a6db932f 100644 --- a/cpp/src/parquet/encryption/encryption.cc +++ b/cpp/src/parquet/encryption/encryption.cc @@ -17,19 +17,29 @@ #include "parquet/encryption/encryption.h" -#include - #include +#include +#include +#include #include #include "arrow/util/logging_internal.h" #include "arrow/util/utf8.h" -#include "parquet/encryption/encryption_internal.h" +#include "parquet/encryption/encryption_utils.h" using ::arrow::util::SecureString; namespace parquet { +namespace { + /// Helper method for validating JSON strings in app_context. + bool IsValidJson(const std::string& json_str) { + rapidjson::Document doc; + doc.Parse(json_str.c_str()); + return doc.IsObject() && !doc.HasParseError(); + } +} // Anonymous namespace + // any empty SecureString key is interpreted as if no key is given // this instance is used when a SecureString reference is returned static SecureString kNoKey = SecureString(); @@ -78,6 +88,12 @@ ColumnEncryptionProperties::Builder* ColumnEncryptionProperties::Builder::key_id return this; } +ColumnEncryptionProperties::Builder* ColumnEncryptionProperties::Builder::parquet_cipher( + ParquetCipher::type parquet_cipher) { + this->parquet_cipher_ = parquet_cipher; + return this; +} + FileDecryptionProperties::Builder* FileDecryptionProperties::Builder::column_keys( ColumnPathToDecryptionPropertiesMap column_decryption_properties) { if (column_decryption_properties.size() == 0) return this; @@ -127,6 +143,59 @@ FileDecryptionProperties::Builder* FileDecryptionProperties::Builder::aad_prefix return this; } +ExternalFileDecryptionProperties::Builder* ExternalFileDecryptionProperties::Builder::app_context( + std::string context) { + if (!app_context_.empty()) { + throw ParquetException("App context already set"); + } + if (context.empty()) { + return this; + } + + if (!IsValidJson(context)) { + throw ParquetException("App context is not a valid JSON string"); + } + app_context_ = std::move(context); + return this; +} + +ExternalFileDecryptionProperties::Builder* +ExternalFileDecryptionProperties::Builder::connection_config( + std::map> config) { + if (connection_config_.size() != 0) { + throw ParquetException("Connection config already set"); + } + + if (config.size() == 0) { + return this; + } + connection_config_ = std::move(config); + return this; +} + +std::shared_ptr +ExternalFileDecryptionProperties::Builder::build_external() { + return std::shared_ptr(new ExternalFileDecryptionProperties( + footer_key_, key_retriever_, check_plaintext_footer_integrity_, aad_prefix_, + aad_prefix_verifier_, column_decryption_properties_, plaintext_files_allowed_, + app_context_, connection_config_)); +} + +ExternalFileDecryptionProperties::ExternalFileDecryptionProperties( + ::arrow::util::SecureString footer_key, + std::shared_ptr key_retriever, + bool check_plaintext_footer_integrity, std::string aad_prefix, + std::shared_ptr aad_prefix_verifier, + ColumnPathToDecryptionPropertiesMap column_decryption_properties, + bool plaintext_files_allowed, + std::string app_context, + std::map> connection_config) + : FileDecryptionProperties(footer_key, key_retriever, check_plaintext_footer_integrity, + aad_prefix, aad_prefix_verifier, column_decryption_properties, + plaintext_files_allowed), + app_context_(app_context), + connection_config_(connection_config) {} + ColumnDecryptionProperties::Builder* ColumnDecryptionProperties::Builder::key( SecureString key) { if (key.empty()) return this; @@ -136,9 +205,15 @@ ColumnDecryptionProperties::Builder* ColumnDecryptionProperties::Builder::key( return this; } +ColumnDecryptionProperties::Builder* ColumnDecryptionProperties::Builder::parquet_cipher( + ParquetCipher::type parquet_cipher) { + parquet_cipher_ = parquet_cipher; + return this; +} + std::shared_ptr ColumnDecryptionProperties::Builder::build() { return std::shared_ptr( - new ColumnDecryptionProperties(column_path_, key_)); + new ColumnDecryptionProperties(column_path_, key_, parquet_cipher_)); } FileEncryptionProperties::Builder* FileEncryptionProperties::Builder::footer_key_metadata( @@ -179,15 +254,15 @@ FileEncryptionProperties::Builder::disable_aad_prefix_storage() { return this; } -ColumnEncryptionProperties::ColumnEncryptionProperties(bool encrypted, - std::string column_path, - SecureString key, - std::string key_metadata) +ColumnEncryptionProperties::ColumnEncryptionProperties( + bool encrypted, std::string column_path, SecureString key, std::string key_metadata, + std::optional parquet_cipher) : column_path_(std::move(column_path)), encrypted_(encrypted), encrypted_with_footer_key_(encrypted && key.empty()), key_(std::move(key)), - key_metadata_(std::move(key_metadata)) { + key_metadata_(std::move(key_metadata)), + parquet_cipher_(parquet_cipher) { DCHECK(!column_path_.empty()); if (!encrypted) { DCHECK(key_.empty() && key_metadata_.empty()); @@ -200,9 +275,9 @@ ColumnEncryptionProperties::ColumnEncryptionProperties(bool encrypted, } } -ColumnDecryptionProperties::ColumnDecryptionProperties(std::string column_path, - SecureString key) - : column_path_(std::move(column_path)), key_(std::move(key)) { +ColumnDecryptionProperties::ColumnDecryptionProperties( + std::string column_path, SecureString key, std::optional parquet_cipher) + : column_path_(std::move(column_path)), key_(std::move(key)), parquet_cipher_(parquet_cipher) { DCHECK(!column_path_.empty()); if (!key_.empty()) { @@ -310,4 +385,57 @@ FileEncryptionProperties::FileEncryptionProperties( } } +ExternalFileEncryptionProperties::Builder* ExternalFileEncryptionProperties::Builder::app_context( + std::string context) { + if (!app_context_.empty()) { + throw ParquetException("App context already set"); + } + + if (context.empty()) { + return this; + } + + if (!IsValidJson(context)) { + throw ParquetException("App context is not a valid JSON string"); + } + + app_context_ = std::move(context); + return this; +} + +ExternalFileEncryptionProperties::Builder* +ExternalFileEncryptionProperties::Builder::connection_config( + std::map> config) { + if (connection_config_.size() != 0) { + throw ParquetException("Connection config already set"); + } + + if (config.size() == 0) { + return this; + } + + connection_config_ = std::move(config); + return this; +} + +std::shared_ptr +ExternalFileEncryptionProperties::Builder::build_external() { + return std::shared_ptr(new ExternalFileEncryptionProperties( + parquet_cipher_, footer_key_, footer_key_metadata_, encrypted_footer_, aad_prefix_, + store_aad_prefix_in_file_, encrypted_columns_, app_context_, connection_config_)); +} + +ExternalFileEncryptionProperties::ExternalFileEncryptionProperties( + ParquetCipher::type cipher, ::arrow::util::SecureString footer_key, + std::string footer_key_metadata, bool encrypted_footer, + std::string aad_prefix, bool store_aad_prefix_in_file, + ColumnPathToEncryptionPropertiesMap encrypted_columns, + std::string app_context, + std::map> connection_config) + : FileEncryptionProperties(cipher, footer_key, footer_key_metadata, encrypted_footer, + aad_prefix, store_aad_prefix_in_file, encrypted_columns), + app_context_(app_context), + connection_config_(connection_config) {} + + } // namespace parquet diff --git a/cpp/src/parquet/encryption/encryption.h b/cpp/src/parquet/encryption/encryption.h index d822cc3c1845..140550acc23d 100644 --- a/cpp/src/parquet/encryption/encryption.h +++ b/cpp/src/parquet/encryption/encryption.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -138,9 +139,15 @@ class PARQUET_EXPORT ColumnEncryptionProperties { /// key_id will be converted to metadata (UTF-8 array). Builder* key_id(std::string key_id); + /// Set ParquetCipher type to use. + /// This field is declared as optional, present when per column encryption was used. If the + /// value is not set, then the ParquetCipher declared in the FileEncryptionProperties will be + /// used. + Builder* parquet_cipher(ParquetCipher::type parquet_cipher); + std::shared_ptr build() { - return std::shared_ptr( - new ColumnEncryptionProperties(encrypted_, column_path_, key_, key_metadata_)); + return std::shared_ptr(new ColumnEncryptionProperties( + encrypted_, column_path_, key_, key_metadata_, parquet_cipher_)); } private: @@ -148,6 +155,7 @@ class PARQUET_EXPORT ColumnEncryptionProperties { bool encrypted_; ::arrow::util::SecureString key_; std::string key_metadata_; + std::optional parquet_cipher_; Builder(std::string path, bool encrypted) : column_path_(std::move(path)), encrypted_(encrypted) {} @@ -159,15 +167,20 @@ class PARQUET_EXPORT ColumnEncryptionProperties { const ::arrow::util::SecureString& key() const { return key_; } const std::string& key_metadata() const { return key_metadata_; } + /// Check whether the optional has a value before using. + std::optional parquet_cipher() const { return parquet_cipher_; } + private: std::string column_path_; bool encrypted_; bool encrypted_with_footer_key_; ::arrow::util::SecureString key_; std::string key_metadata_; + std::optional parquet_cipher_; explicit ColumnEncryptionProperties(bool encrypted, std::string column_path, ::arrow::util::SecureString key, - std::string key_metadata); + std::string key_metadata, + std::optional parquet_cipher); }; class PARQUET_EXPORT ColumnDecryptionProperties { @@ -184,25 +197,36 @@ class PARQUET_EXPORT ColumnDecryptionProperties { /// key length must be either 16, 24 or 32 bytes. Builder* key(::arrow::util::SecureString key); + /// Set ParquetCipher type to use. + /// This field is declared as optional, present when per column encryption was used. If the + /// value is not set, then the ParquetCipher declared in the InternalFileDecryptor will be used. + Builder* parquet_cipher(ParquetCipher::type parquet_cipher); + std::shared_ptr build(); private: std::string column_path_; ::arrow::util::SecureString key_; + std::optional parquet_cipher_; }; const std::string& column_path() const { return column_path_; } const ::arrow::util::SecureString& key() const { return key_; } + /// Check whether the optional has a value before using. + std::optional parquet_cipher() const { return parquet_cipher_; } + private: std::string column_path_; ::arrow::util::SecureString key_; + std::optional parquet_cipher_; /// This class is only required for setting explicit column decryption keys - /// to override key retriever (or to provide keys when key metadata and/or /// key retriever are not available) explicit ColumnDecryptionProperties(std::string column_path, - ::arrow::util::SecureString key); + ::arrow::util::SecureString key, + std::optional parquet_cipher); }; class PARQUET_EXPORT AADPrefixVerifier { @@ -293,7 +317,7 @@ class PARQUET_EXPORT FileDecryptionProperties { aad_prefix_verifier_, column_decryption_properties_, plaintext_files_allowed_)); } - private: + protected: ::arrow::util::SecureString footer_key_; std::string aad_prefix_; std::shared_ptr aad_prefix_verifier_; @@ -333,6 +357,7 @@ class PARQUET_EXPORT FileDecryptionProperties { bool check_plaintext_footer_integrity_; bool plaintext_files_allowed_; + protected: FileDecryptionProperties( ::arrow::util::SecureString footer_key, std::shared_ptr key_retriever, @@ -340,8 +365,91 @@ class PARQUET_EXPORT FileDecryptionProperties { std::shared_ptr aad_prefix_verifier, ColumnPathToDecryptionPropertiesMap column_decryption_properties, bool plaintext_files_allowed); + + public: + virtual ~FileDecryptionProperties() = default; }; +class PARQUET_EXPORT ExternalFileDecryptionProperties : public FileDecryptionProperties { + public: + class PARQUET_EXPORT Builder : public FileDecryptionProperties::Builder { + public: + explicit Builder() : FileDecryptionProperties::Builder() {} + + Builder* app_context(std::string context); + + Builder* connection_config( + std::map> config); + + /// Forward all base class property methods to the base class Builder so we can return the + /// correct Builder type. + Builder* footer_key(::arrow::util::SecureString footer_key) { + FileDecryptionProperties::Builder::footer_key(footer_key); + return this; + } + + Builder* column_keys( + ColumnPathToDecryptionPropertiesMap column_decryption_properties) { + FileDecryptionProperties::Builder::column_keys(column_decryption_properties); + return this; + } + + Builder* key_retriever(std::shared_ptr key_retriever) { + FileDecryptionProperties::Builder::key_retriever(key_retriever); + return this; + } + + Builder* disable_footer_signature_verification() { + FileDecryptionProperties::Builder::disable_footer_signature_verification(); + return this; + } + + Builder* aad_prefix(std::string aad_prefix) { + FileDecryptionProperties::Builder::aad_prefix(aad_prefix); + return this; + } + + Builder* aad_prefix_verifier(std::shared_ptr aad_prefix_verifier) { + FileDecryptionProperties::Builder::aad_prefix_verifier(aad_prefix_verifier); + return this; + } + + Builder* plaintext_files_allowed() { + FileDecryptionProperties::Builder::plaintext_files_allowed(); + return this; + } + + std::shared_ptr build_external(); + + private: + std::string app_context_; + std::map> connection_config_; + }; + + const std::string& app_context() const { + return app_context_; + } + + const std::map>& + connection_config() const { + return connection_config_; + } + + private: + std::string app_context_; + std::map> connection_config_; + + ExternalFileDecryptionProperties( + ::arrow::util::SecureString footer_key, + std::shared_ptr key_retriever, + bool check_plaintext_footer_integrity, std::string aad_prefix, + std::shared_ptr aad_prefix_verifier, + ColumnPathToDecryptionPropertiesMap column_decryption_properties, + bool plaintext_files_allowed, + std::string app_context, + std::map> connection_config); + }; + class PARQUET_EXPORT FileEncryptionProperties { public: class PARQUET_EXPORT Builder { @@ -393,7 +501,7 @@ class PARQUET_EXPORT FileEncryptionProperties { aad_prefix_, store_aad_prefix_in_file_, encrypted_columns_)); } - private: + protected: ParquetCipher::type parquet_cipher_; bool encrypted_footer_; ::arrow::util::SecureString footer_key_; @@ -431,11 +539,99 @@ class PARQUET_EXPORT FileEncryptionProperties { bool store_aad_prefix_in_file_; ColumnPathToEncryptionPropertiesMap encrypted_columns_; + protected: FileEncryptionProperties(ParquetCipher::type cipher, ::arrow::util::SecureString footer_key, std::string footer_key_metadata, bool encrypted_footer, std::string aad_prefix, bool store_aad_prefix_in_file, ColumnPathToEncryptionPropertiesMap encrypted_columns); + public: + virtual ~FileEncryptionProperties() = default; }; +class PARQUET_EXPORT ExternalFileEncryptionProperties : public FileEncryptionProperties { + public: + + class PARQUET_EXPORT Builder : public FileEncryptionProperties::Builder { + public: + + explicit Builder(::arrow::util::SecureString footer_key) + : FileEncryptionProperties::Builder(footer_key) {} + + /// Valid JSON string with additional application context needed for security checks. + Builder* app_context(std::string context); + + /// Map of the encryption algorithms to the key/value map of the location of configuration files + /// needed by the external encryptors, including location of a dynamically-linked library, + /// or config files where the external encryptors can find urls, certificates, and parameters + /// needed to make a remote call. + Builder* connection_config( + std::map> config); + + /// Forward all base class property methods to the base class Builder so we can return the + /// correct Builder type. + Builder* set_plaintext_footer() { + FileEncryptionProperties::Builder::set_plaintext_footer(); + return this; + } + + Builder* algorithm(ParquetCipher::type parquet_cipher) { + FileEncryptionProperties::Builder::algorithm(parquet_cipher); + return this; + } + + Builder* footer_key_id(std::string key_id) { + FileEncryptionProperties::Builder::footer_key_id(key_id); + return this; + } + + Builder* footer_key_metadata(std::string footer_key_metadata) { + FileEncryptionProperties::Builder::footer_key_metadata(footer_key_metadata); + return this; + } + + Builder* aad_prefix(std::string aad_prefix) { + FileEncryptionProperties::Builder::aad_prefix(aad_prefix); + return this; + } + + Builder* disable_aad_prefix_storage() { + FileEncryptionProperties::Builder::disable_aad_prefix_storage(); + return this; + } + + Builder* encrypted_columns(ColumnPathToEncryptionPropertiesMap encrypted_columns) { + FileEncryptionProperties::Builder::encrypted_columns(encrypted_columns); + return this; + } + + std::shared_ptr build_external(); + + private: + std::string app_context_; + std::map> connection_config_; + }; + + const std::string& app_context() const { + return app_context_; + } + + const std::map>& + connection_config() const { + return connection_config_; + } + + private: + std::string app_context_; + std::map> connection_config_; + + ExternalFileEncryptionProperties(ParquetCipher::type cipher, + ::arrow::util::SecureString footer_key, + std::string footer_key_metadata, bool encrypted_footer, + std::string aad_prefix, bool store_aad_prefix_in_file, + ColumnPathToEncryptionPropertiesMap encrypted_columns, + std::string app_context, + std::map> connection_config); + }; + } // namespace parquet diff --git a/cpp/src/parquet/encryption/encryption_internal.h b/cpp/src/parquet/encryption/encryption_internal.h deleted file mode 100644 index 062527495659..000000000000 --- a/cpp/src/parquet/encryption/encryption_internal.h +++ /dev/null @@ -1,141 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include - -#include "arrow/util/span.h" -#include "parquet/properties.h" -#include "parquet/types.h" - -using parquet::ParquetCipher; - -namespace parquet::encryption { - -constexpr int32_t kGcmTagLength = 16; -constexpr int32_t kNonceLength = 12; - -// Module types -constexpr int8_t kFooter = 0; -constexpr int8_t kColumnMetaData = 1; -constexpr int8_t kDataPage = 2; -constexpr int8_t kDictionaryPage = 3; -constexpr int8_t kDataPageHeader = 4; -constexpr int8_t kDictionaryPageHeader = 5; -constexpr int8_t kColumnIndex = 6; -constexpr int8_t kOffsetIndex = 7; -constexpr int8_t kBloomFilterHeader = 8; -constexpr int8_t kBloomFilterBitset = 9; - -/// Performs AES encryption operations with GCM or CTR ciphers. -class PARQUET_EXPORT AesEncryptor { - public: - /// Can serve one key length only. Possible values: 16, 24, 32 bytes. - /// If write_length is true, prepend ciphertext length to the ciphertext - explicit AesEncryptor(ParquetCipher::type alg_id, int32_t key_len, bool metadata, - bool write_length = true); - - static std::unique_ptr Make(ParquetCipher::type alg_id, int32_t key_len, - bool metadata, bool write_length = true); - - ~AesEncryptor(); - - /// The size of the ciphertext, for this cipher and the specified plaintext length. - [[nodiscard]] int32_t CiphertextLength(int64_t plaintext_len) const; - - /// Encrypts plaintext with the key and aad. Key length is passed only for validation. - /// If different from value in constructor, exception will be thrown. - int32_t Encrypt(::arrow::util::span plaintext, - ::arrow::util::span key, - ::arrow::util::span aad, - ::arrow::util::span ciphertext); - - /// Encrypts plaintext footer, in order to compute footer signature (tag). - int32_t SignedFooterEncrypt(::arrow::util::span footer, - ::arrow::util::span key, - ::arrow::util::span aad, - ::arrow::util::span nonce, - ::arrow::util::span encrypted_footer); - - private: - // PIMPL Idiom - class AesEncryptorImpl; - std::unique_ptr impl_; -}; - -/// Performs AES decryption operations with GCM or CTR ciphers. -class PARQUET_EXPORT AesDecryptor { - public: - /// \brief Construct an AesDecryptor - /// - /// \param alg_id the encryption algorithm to use - /// \param key_len key length. Possible values: 16, 24, 32 bytes. - /// \param metadata if true then this is a metadata decryptor - /// \param contains_length if true, expect ciphertext length prepended to the ciphertext - explicit AesDecryptor(ParquetCipher::type alg_id, int32_t key_len, bool metadata, - bool contains_length = true); - - static std::unique_ptr Make(ParquetCipher::type alg_id, int32_t key_len, - bool metadata); - - ~AesDecryptor(); - - /// The size of the plaintext, for this cipher and the specified ciphertext length. - [[nodiscard]] int32_t PlaintextLength(int32_t ciphertext_len) const; - - /// The size of the ciphertext, for this cipher and the specified plaintext length. - [[nodiscard]] int32_t CiphertextLength(int32_t plaintext_len) const; - - /// Decrypts ciphertext with the key and aad. Key length is passed only for - /// validation. If different from value in constructor, exception will be thrown. - /// The caller is responsible for ensuring that the plaintext buffer is at least as - /// large as PlaintextLength(ciphertext_len). - int32_t Decrypt(::arrow::util::span ciphertext, - ::arrow::util::span key, - ::arrow::util::span aad, - ::arrow::util::span plaintext); - - private: - // PIMPL Idiom - class AesDecryptorImpl; - std::unique_ptr impl_; -}; - -std::string CreateModuleAad(const std::string& file_aad, int8_t module_type, - int16_t row_group_ordinal, int16_t column_ordinal, - int32_t page_ordinal); - -std::string CreateFooterAad(const std::string& aad_prefix_bytes); - -// Update last two bytes of page (or page header) module AAD -void QuickUpdatePageAad(int32_t new_page_ordinal, std::string* AAD); - -// Wraps OpenSSL RAND_bytes function -void RandBytes(unsigned char* buf, size_t num); - -// Ensure OpenSSL is initialized. -// -// This is only necessary in specific situations since OpenSSL otherwise -// initializes itself automatically. For example, under Valgrind, a memory -// leak will be reported if OpenSSL is initialized for the first time from -// a worker thread; calling this function from the main thread prevents this. -void EnsureBackendInitialized(); - -} // namespace parquet::encryption diff --git a/cpp/src/parquet/encryption/encryption_utils.cc b/cpp/src/parquet/encryption/encryption_utils.cc new file mode 100644 index 000000000000..66db130121e8 --- /dev/null +++ b/cpp/src/parquet/encryption/encryption_utils.cc @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "parquet/encryption/encryption_utils.h" +#include +#include +#include "parquet/encryption/openssl_internal.h" +#include "parquet/exception.h" + +namespace parquet::encryption { + +static std::string ShortToBytesLe(int16_t input) { + int8_t output[2]; + memset(output, 0, 2); + output[1] = static_cast(0xff & (input >> 8)); + output[0] = static_cast(0xff & (input)); + + return std::string(reinterpret_cast(output), 2); +} + +static void CheckPageOrdinal(int32_t page_ordinal) { + if (ARROW_PREDICT_FALSE(page_ordinal > std::numeric_limits::max())) { + throw ParquetException("Encrypted Parquet files can't have more than " + + std::to_string(std::numeric_limits::max()) + + " pages per chunk: got " + std::to_string(page_ordinal)); + } +} + +std::string CreateModuleAad(const std::string& file_aad, int8_t module_type, + int16_t row_group_ordinal, int16_t column_ordinal, + int32_t page_ordinal) { + CheckPageOrdinal(page_ordinal); + const int16_t page_ordinal_short = static_cast(page_ordinal); + int8_t type_ordinal_bytes[1]; + type_ordinal_bytes[0] = module_type; + std::string type_ordinal_bytes_str(reinterpret_cast(type_ordinal_bytes), 1); + if (kFooter == module_type) { + std::string result = file_aad + type_ordinal_bytes_str; + return result; + } + std::string row_group_ordinal_bytes = ShortToBytesLe(row_group_ordinal); + std::string column_ordinal_bytes = ShortToBytesLe(column_ordinal); + if (kDataPage != module_type && kDataPageHeader != module_type) { + std::ostringstream out; + out << file_aad << type_ordinal_bytes_str << row_group_ordinal_bytes + << column_ordinal_bytes; + return out.str(); + } + std::string page_ordinal_bytes = ShortToBytesLe(page_ordinal_short); + std::ostringstream out; + out << file_aad << type_ordinal_bytes_str << row_group_ordinal_bytes + << column_ordinal_bytes << page_ordinal_bytes; + return out.str(); +} + +std::string CreateFooterAad(const std::string& aad_prefix_bytes) { + return CreateModuleAad(aad_prefix_bytes, kFooter, static_cast(-1), + static_cast(-1), static_cast(-1)); +} + +// Update last two bytes with new page ordinal (instead of creating new page AAD from scratch) +void QuickUpdatePageAad(int32_t new_page_ordinal, std::string* AAD) { + CheckPageOrdinal(new_page_ordinal); + const std::string page_ordinal_bytes = + ShortToBytesLe(static_cast(new_page_ordinal)); + std::memcpy(AAD->data() + AAD->length() - 2, page_ordinal_bytes.data(), 2); +} + +void RandBytes(unsigned char* buf, size_t num) { + if (num > static_cast(std::numeric_limits::max())) { + std::stringstream ss; + ss << "Length " << num << " for RandBytes overflows int"; + throw ParquetException(ss.str()); + } + openssl::EnsureInitialized(); + int status = RAND_bytes(buf, static_cast(num)); + if (status != 1) { + const auto error_code = ERR_get_error(); + char buffer[256]; + ERR_error_string_n(error_code, buffer, sizeof(buffer)); + std::stringstream ss; + ss << "Failed to generate random bytes: " << buffer; + throw ParquetException(ss.str()); + } +} + +void EnsureBackendInitialized() { openssl::EnsureInitialized(); } + +} // namespace parquet::encryption diff --git a/cpp/src/parquet/encryption/encryption_utils.h b/cpp/src/parquet/encryption/encryption_utils.h new file mode 100644 index 000000000000..0108a99d708c --- /dev/null +++ b/cpp/src/parquet/encryption/encryption_utils.h @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +namespace parquet::encryption { + +constexpr int32_t kGcmTagLength = 16; +constexpr int32_t kNonceLength = 12; + +// Module types +constexpr int8_t kFooter = 0; +constexpr int8_t kColumnMetaData = 1; +constexpr int8_t kDataPage = 2; +constexpr int8_t kDictionaryPage = 3; +constexpr int8_t kDataPageHeader = 4; +constexpr int8_t kDictionaryPageHeader = 5; +constexpr int8_t kColumnIndex = 6; +constexpr int8_t kOffsetIndex = 7; +constexpr int8_t kBloomFilterHeader = 8; +constexpr int8_t kBloomFilterBitset = 9; + +std::string CreateModuleAad(const std::string& file_aad, int8_t module_type, + int16_t row_group_ordinal, int16_t column_ordinal, + int32_t page_ordinal); + +std::string CreateFooterAad(const std::string& aad_prefix_bytes); + +// Update last two bytes of page (or page header) module AAD +void QuickUpdatePageAad(int32_t new_page_ordinal, std::string* AAD); + +// Wraps OpenSSL RAND_bytes function +void RandBytes(unsigned char* buf, size_t num); + +// Ensure OpenSSL is initialized. +// +// This is only necessary in specific situations since OpenSSL otherwise +// initializes itself automatically. For example, under Valgrind, a memory +// leak will be reported if OpenSSL is initialized for the first time from +// a worker thread; calling this function from the main thread prevents this. +void EnsureBackendInitialized(); + +} // namespace parquet::encryption diff --git a/cpp/src/parquet/encryption/encryptor_interface.h b/cpp/src/parquet/encryption/encryptor_interface.h new file mode 100644 index 000000000000..ee31fa756351 --- /dev/null +++ b/cpp/src/parquet/encryption/encryptor_interface.h @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "parquet/platform.h" +#include "parquet/encryption/encoding_properties.h" + +namespace parquet::encryption { + +class PARQUET_EXPORT EncryptorInterface { + public: + virtual ~EncryptorInterface() = default; + + /// Signal whether the encryptor can calculate a valid ciphertext length before performing + /// encryption or not. If false, a proper sized buffer cannot be allocated before calling the + /// Encrypt method, and Arrow must use this encryptor's EncryptWithManagedBuffer method + /// instead of Encrypt. + [[nodiscard]] virtual bool CanCalculateCiphertextLength() const = 0; + + /// Calculate the size of the ciphertext for a given plaintext length. + [[nodiscard]] virtual int32_t CiphertextLength(int64_t plaintext_len) const = 0; + + /// Encrypt the plaintext and leave the results in the ciphertext buffer. + /// Most implementations will require the key and aad to be provided, but it is up to + /// each encryptor whether to use them or not. + virtual int32_t Encrypt(::arrow::util::span plaintext, + ::arrow::util::span key, + ::arrow::util::span aad, + ::arrow::util::span ciphertext) = 0; + + /// Encrypt the plaintext and leave the results in the ciphertext buffer. + /// The buffer will be resized to the appropriate size by the encryptor during encryption. + /// This method is used when the encryptor cannot calculate the ciphertext length before + /// encryption. + virtual int32_t EncryptWithManagedBuffer(::arrow::util::span plaintext, + ::arrow::ResizableBuffer* ciphertext) = 0; + + // Some Encryptors may need to understand the page encoding before the encryption process. + // This method will be called from ColumnWriter before invoking the Encrypt method. + virtual void UpdateEncodingProperties(std::unique_ptr encoding_properties) {}; + + /// After the column_writer writes a dictionary or a data page, this method will be called + /// so that each encryptor can provide any encryptor-specific column metadata that should be + /// stored in the Parquet file. The keys and values are added to the column metadata, any + /// conflicting key and value pairs are overwritten. There is no need to clear the metadata + /// after the call. + virtual std::shared_ptr GetKeyValueMetadata(int8_t module_type) { + return nullptr; + } + + /// Encrypt footer metadata for signature verification purposes only. + /// This method is used specifically for footer signature verification in encrypted + /// Parquet files with plaintext footers. It encrypts the footer metadata using + /// the provided key, AAD, and nonce to generate an authentication tag that can + /// be compared against a stored signature. + virtual int32_t SignedFooterEncrypt(::arrow::util::span footer, + ::arrow::util::span key, + ::arrow::util::span aad, + ::arrow::util::span nonce, + ::arrow::util::span encrypted_footer) = 0; +}; + +} // namespace parquet::encryption diff --git a/cpp/src/parquet/encryption/external/dbpa_enum_utils.cc b/cpp/src/parquet/encryption/external/dbpa_enum_utils.cc new file mode 100644 index 000000000000..95fdc15e297d --- /dev/null +++ b/cpp/src/parquet/encryption/external/dbpa_enum_utils.cc @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "parquet/encryption/external/dbpa_enum_utils.h" + +#include +#include + +namespace parquet::encryption::external { + +// Static map definitions for the class members + +// ------------------------------------------------------------ +// parquet::Type -> dbps::external::Type +// Parquet types are coming from parquet/types.h +// ------------------------------------------------------------ +const std::unordered_map DBPAEnumUtils::parquet_to_external_type_map = { + {parquet::Type::BOOLEAN, dbps::external::Type::BOOLEAN}, + {parquet::Type::INT32, dbps::external::Type::INT32}, + {parquet::Type::INT64, dbps::external::Type::INT64}, + {parquet::Type::INT96, dbps::external::Type::INT96}, + {parquet::Type::FLOAT, dbps::external::Type::FLOAT}, + {parquet::Type::DOUBLE, dbps::external::Type::DOUBLE}, + {parquet::Type::BYTE_ARRAY, dbps::external::Type::BYTE_ARRAY}, + {parquet::Type::FIXED_LEN_BYTE_ARRAY, dbps::external::Type::FIXED_LEN_BYTE_ARRAY}, + {parquet::Type::UNDEFINED, dbps::external::Type::UNDEFINED} +}; + +// ------------------------------------------------------------ +// arrow::Compression -> dbps::external::CompressionCodec +// values for arrow::Compression are coming from arrow/type_fwd.h +// Note: Some Arrow compression types don't have direct DBPS equivalents +// ------------------------------------------------------------ +const std::unordered_map<::arrow::Compression::type, dbps::external::CompressionCodec::type> DBPAEnumUtils::arrow_to_external_compression_map = { + {::arrow::Compression::UNCOMPRESSED, dbps::external::CompressionCodec::UNCOMPRESSED}, + {::arrow::Compression::SNAPPY, dbps::external::CompressionCodec::SNAPPY}, + {::arrow::Compression::GZIP, dbps::external::CompressionCodec::GZIP}, + {::arrow::Compression::LZO, dbps::external::CompressionCodec::LZO}, + {::arrow::Compression::BROTLI, dbps::external::CompressionCodec::BROTLI}, + {::arrow::Compression::LZ4, dbps::external::CompressionCodec::LZ4}, + {::arrow::Compression::ZSTD, dbps::external::CompressionCodec::ZSTD}, + {::arrow::Compression::LZ4_FRAME, dbps::external::CompressionCodec::LZ4_FRAME}, + {::arrow::Compression::BZ2, dbps::external::CompressionCodec::BZ2}, + {::arrow::Compression::LZ4_HADOOP, dbps::external::CompressionCodec::LZ4_HADOOP} +}; + +// ------------------------------------------------------------ +// function which returns parquet::Type::type to dbps::external::Type::type +// ------------------------------------------------------------ +dbps::external::Type::type DBPAEnumUtils::ParquetTypeToDBPA(parquet::Type::type parquet_type) { + + // Look up the mapping in the static map + auto it = parquet_to_external_type_map.find(parquet_type); + if (it != parquet_to_external_type_map.end()) { + return it->second; + } + + throw std::invalid_argument("Invalid parquet::Type value"); +} + +// ------------------------------------------------------------ +// function which returns arrow::Compression::type to dbps::external::CompressionCodec::type +// ------------------------------------------------------------ +dbps::external::CompressionCodec::type DBPAEnumUtils::ArrowCompressionToDBPA(::arrow::Compression::type arrow_compression) { + // Look up the mapping in the static map + auto it = arrow_to_external_compression_map.find(arrow_compression); + if (it != arrow_to_external_compression_map.end()) { + return it->second; + } + + throw std::invalid_argument("Invalid arrow::Compression value"); +} + +} // namespace parquet::encryption::external diff --git a/cpp/src/parquet/encryption/external/dbpa_enum_utils.h b/cpp/src/parquet/encryption/external/dbpa_enum_utils.h new file mode 100644 index 000000000000..4e2831b8ce32 --- /dev/null +++ b/cpp/src/parquet/encryption/external/dbpa_enum_utils.h @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include + +#include "parquet/types.h" +#include "arrow/type_fwd.h" // For arrow::Compression + +namespace parquet::encryption::external { + +/** + * Utility class for translating between Parquet/Arrow enums and dbps::external enums. + * + * This class provides methods to convert between: + * - parquet::Type and dbps::external::Type + * - arrow::Compression and dbps::external::CompressionCodec + */ +class DBPAEnumUtils { +public: + // Static maps for type conversions + static const std::unordered_map parquet_to_external_type_map; + static const std::unordered_map<::arrow::Compression::type, dbps::external::CompressionCodec::type> arrow_to_external_compression_map; + + /** + * Convert parquet::Type to dbps::external::Type + * + * @param parquet_type The parquet type to convert + * @return The corresponding dbps::external::Type + */ + static dbps::external::Type::type ParquetTypeToDBPA(parquet::Type::type parquet_type); + + /** + * Convert arrow::Compression to dbps::external::CompressionCodec + * + * @param arrow_compression The Arrow compression type to convert + * @return The corresponding dbps::external::CompressionCodec + * @throws std::invalid_argument if the Arrow compression type cannot be mapped + */ + static dbps::external::CompressionCodec::type ArrowCompressionToDBPA(::arrow::Compression::type arrow_compression); +}; + +} // namespace parquet::encryption::external diff --git a/cpp/src/parquet/encryption/external/dbpa_enum_utils_test.cc b/cpp/src/parquet/encryption/external/dbpa_enum_utils_test.cc new file mode 100644 index 000000000000..344cf12796aa --- /dev/null +++ b/cpp/src/parquet/encryption/external/dbpa_enum_utils_test.cc @@ -0,0 +1,163 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include "parquet/platform.h" +#include "parquet/types.h" +#include "arrow/util/type_fwd.h" +#include +#include "parquet/encryption/external/dbpa_enum_utils.h" + +#include + +using magic_enum::enum_count; + +namespace parquet::encryption::external { + +class DBPAUtilsTest : public ::testing::Test { + protected: + void SetUp() override { + // Calculate enum sizes once during test initialization + + // We use "Magic Enum" to check the sizes of the enums. + // https://github.com/Neargye/magic_enum + // (the additional library is needed as reflection for enums is not available in C++) + parquet_type_enum_size_ = magic_enum::enum_count(); + arrow_compression_enum_size_ = magic_enum::enum_count<::arrow::Compression::type>(); + } + + // Enum sizes calculated during test initialization + std::size_t parquet_type_enum_size_; + std::size_t arrow_compression_enum_size_ ; +}; + +TEST_F(DBPAUtilsTest, ParquetTypeToExternal) { + // Test all valid parquet types + EXPECT_EQ(DBPAEnumUtils::ParquetTypeToDBPA(parquet::Type::BOOLEAN), + dbps::external::Type::BOOLEAN); + EXPECT_EQ(DBPAEnumUtils::ParquetTypeToDBPA(parquet::Type::INT32), + dbps::external::Type::INT32); + EXPECT_EQ(DBPAEnumUtils::ParquetTypeToDBPA(parquet::Type::INT64), + dbps::external::Type::INT64); + EXPECT_EQ(DBPAEnumUtils::ParquetTypeToDBPA(parquet::Type::INT96), + dbps::external::Type::INT96); + EXPECT_EQ(DBPAEnumUtils::ParquetTypeToDBPA(parquet::Type::FLOAT), + dbps::external::Type::FLOAT); + EXPECT_EQ(DBPAEnumUtils::ParquetTypeToDBPA(parquet::Type::DOUBLE), + dbps::external::Type::DOUBLE); + EXPECT_EQ(DBPAEnumUtils::ParquetTypeToDBPA(parquet::Type::BYTE_ARRAY), + dbps::external::Type::BYTE_ARRAY); + EXPECT_EQ(DBPAEnumUtils::ParquetTypeToDBPA(parquet::Type::FIXED_LEN_BYTE_ARRAY), + dbps::external::Type::FIXED_LEN_BYTE_ARRAY); +} + +TEST_F(DBPAUtilsTest, ArrowCompressionToExternal) { + // Test all valid arrow compression types that have mappings + EXPECT_EQ(DBPAEnumUtils::ArrowCompressionToDBPA(::arrow::Compression::UNCOMPRESSED), + dbps::external::CompressionCodec::UNCOMPRESSED); + EXPECT_EQ(DBPAEnumUtils::ArrowCompressionToDBPA(::arrow::Compression::SNAPPY), + dbps::external::CompressionCodec::SNAPPY); + EXPECT_EQ(DBPAEnumUtils::ArrowCompressionToDBPA(::arrow::Compression::GZIP), + dbps::external::CompressionCodec::GZIP); + EXPECT_EQ(DBPAEnumUtils::ArrowCompressionToDBPA(::arrow::Compression::LZO), + dbps::external::CompressionCodec::LZO); + EXPECT_EQ(DBPAEnumUtils::ArrowCompressionToDBPA(::arrow::Compression::BROTLI), + dbps::external::CompressionCodec::BROTLI); + EXPECT_EQ(DBPAEnumUtils::ArrowCompressionToDBPA(::arrow::Compression::LZ4), + dbps::external::CompressionCodec::LZ4); + EXPECT_EQ(DBPAEnumUtils::ArrowCompressionToDBPA(::arrow::Compression::ZSTD), + dbps::external::CompressionCodec::ZSTD); + EXPECT_EQ(DBPAEnumUtils::ArrowCompressionToDBPA(::arrow::Compression::LZ4_FRAME), + dbps::external::CompressionCodec::LZ4_FRAME); + EXPECT_EQ(DBPAEnumUtils::ArrowCompressionToDBPA(::arrow::Compression::BZ2), + dbps::external::CompressionCodec::BZ2); + EXPECT_EQ(DBPAEnumUtils::ArrowCompressionToDBPA(::arrow::Compression::LZ4_HADOOP), + dbps::external::CompressionCodec::LZ4_HADOOP); +} + +TEST_F(DBPAUtilsTest, AllValidTypeMappings) { + // Test that all valid parquet types can be converted to external types + // Using the actual enum values from parquet::Type (excluding UNDEFINED) + std::vector valid_parquet_types = { + parquet::Type::BOOLEAN, + parquet::Type::INT32, + parquet::Type::INT64, + parquet::Type::INT96, + parquet::Type::FLOAT, + parquet::Type::DOUBLE, + parquet::Type::BYTE_ARRAY, + parquet::Type::FIXED_LEN_BYTE_ARRAY, + parquet::Type::UNDEFINED + }; + + //ensure that the map is complete. + ASSERT_EQ(valid_parquet_types.size(), parquet_type_enum_size_); + + for (auto parquet_type : valid_parquet_types) { + EXPECT_NO_THROW(DBPAEnumUtils::ParquetTypeToDBPA(parquet_type)); + } +} + +TEST_F(DBPAUtilsTest, AllValidCompressionMappings) { + // Test that all valid arrow compression types that have mappings work + // Using the actual enum values from arrow::Compression that are supported + std::vector<::arrow::Compression::type> valid_arrow_compressions = { + ::arrow::Compression::UNCOMPRESSED, + ::arrow::Compression::SNAPPY, + ::arrow::Compression::GZIP, + ::arrow::Compression::LZO, + ::arrow::Compression::BROTLI, + ::arrow::Compression::LZ4, + ::arrow::Compression::ZSTD, + ::arrow::Compression::LZ4_FRAME, + ::arrow::Compression::BZ2, + ::arrow::Compression::LZ4_HADOOP + }; + + ASSERT_EQ(valid_arrow_compressions.size(), arrow_compression_enum_size_); + + for (auto arrow_compression : valid_arrow_compressions) { + EXPECT_NO_THROW(DBPAEnumUtils::ArrowCompressionToDBPA(arrow_compression)); + } +} + +TEST_F(DBPAUtilsTest, MapSizeAssertions) { + // Test the actual map sizes by accessing the enums and the public static maps directly + // This provides a direct way to verify map completeness + + // Parquet::Type::type assertions + EXPECT_EQ(parquet_type_enum_size_, 9) + << "Expected 9 parquet type mappings (excluding UNDEFINED)"; + + EXPECT_EQ(parquet_type_enum_size_, DBPAEnumUtils::parquet_to_external_type_map.size()) + << "Expected 9 parquet type mappings (excluding UNDEFINED)"; + + EXPECT_EQ(DBPAEnumUtils::parquet_to_external_type_map.size(), 9) + << "Expected 9 parquet type mappings (excluding UNDEFINED)"; + + // Arrow::Compression::type assertions + EXPECT_EQ(DBPAEnumUtils::arrow_to_external_compression_map.size(), 10) + << "Expected 10 arrow compression mappings"; + + EXPECT_EQ(arrow_compression_enum_size_, 10) + << "Expected 10 arrow compression mappings"; + + EXPECT_EQ(arrow_compression_enum_size_, DBPAEnumUtils::arrow_to_external_compression_map.size()) + << "Expected 10 arrow compression mappings"; +} +} // namespace parquet::encryption::external diff --git a/cpp/src/parquet/encryption/external/dbpa_executor.cc b/cpp/src/parquet/encryption/external/dbpa_executor.cc new file mode 100644 index 000000000000..1f7f665f115d --- /dev/null +++ b/cpp/src/parquet/encryption/external/dbpa_executor.cc @@ -0,0 +1,196 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "parquet/encryption/external/dbpa_executor.h" +#include + +#include +#include +#include +#include + +#include "arrow/util/logging.h" + +namespace parquet::encryption::external { + +/** + * Utility function to execute a wrapped operation with timeout using + * pure C++ futures + * @tparam Func The function type to execute + * @tparam Args The argument types + * @param operation_name Name of the operation for error reporting + * @param timeout_milliseconds Timeout in milliseconds + * @param func The function to execute + * @param args The arguments to pass to the function + * @return The result of the function execution + */ +template +auto ExecuteWithTimeout(const std::string& operation_name, + int64_t timeout_milliseconds, + Func&& func, + Args&&... args) -> decltype(func(args...)) { + + // Get the return type of the function that we're executing + using ReturnType = decltype(func(args...)); + + ARROW_LOG(DEBUG) << "[DBPAExecutor] Starting " << operation_name << " operation with timeout " + << timeout_milliseconds << " milliseconds"; + + auto start_time = std::chrono::steady_clock::now(); + + // Create a future to run the operation asynchronously + ARROW_LOG(DEBUG) << "[DBPAExecutor] Creating async future for " << operation_name; + auto future = std::async(std::launch::async, [&]() -> ReturnType { + ARROW_LOG(DEBUG) << "[DBPAExecutor] Async task started for " << operation_name; + + // Execute without inner exception logging to avoid duplicate logs. + if constexpr (std::is_void_v) { + func(args...); + } else { + return func(args...); + } + }); + + ARROW_LOG(DEBUG) << "[DBPAExecutor] Future created, waiting for " << operation_name + << " with timeout " << timeout_milliseconds << " milliseconds"; + + // Wait for the function to complete or timeout. + auto status = future.wait_for(std::chrono::milliseconds(timeout_milliseconds)); + + auto end_time = std::chrono::steady_clock::now(); + auto duration = std::chrono::duration_cast(end_time - start_time); + + // if we timed out, throw a DBPAExecutorTimeoutException + if (status == std::future_status::timeout) { + ARROW_LOG(ERROR) << "[DBPAExecutor] TIMEOUT: " << operation_name << " exceeded " + << timeout_milliseconds << " milliseconds (actual: " << duration.count() << "ms)"; + throw DBPAExecutorTimeoutException(operation_name, timeout_milliseconds); + } + + ARROW_LOG(DEBUG) << "[DBPAExecutor] Future completed for " << operation_name + << " in " << duration.count() << "ms, retrieving result..."; + + try { + // If any exceptions are thrown in the body of the function, + // they will be re-thrown by future.get() (original exception is thrown unchanged, no wrapping)) + if constexpr (std::is_void_v) { + future.get(); + ARROW_LOG(DEBUG) << "[DBPAExecutor] SUCCESS: " << operation_name << " completed successfully"; + return; + } else { + auto result = future.get(); + ARROW_LOG(DEBUG) << "[DBPAExecutor] SUCCESS: " << operation_name << " completed successfully"; + return result; + } + } + catch (const std::exception& e) { + ARROW_LOG(ERROR) << "[DBPAExecutor] EXCEPTION: " << operation_name << " failed with: " << e.what(); + throw; // Re-throw original exception + } + catch (...) { + ARROW_LOG(ERROR) << "[DBPAExecutor] UNKNOWN EXCEPTION: " << operation_name << " failed with unknown exception"; + throw; // Re-throw original exception + } +} + +DBPAExecutor::DBPAExecutor(std::unique_ptr agent, + int64_t init_timeout, + int64_t encrypt_timeout, + int64_t decrypt_timeout) + : wrapped_agent_(std::move(agent)), + init_timeout_milliseconds_(init_timeout), + encrypt_timeout_milliseconds_(encrypt_timeout), + decrypt_timeout_milliseconds_(decrypt_timeout) { + + // Ensure the wrapped agent is not null + if (!wrapped_agent_) { + ARROW_LOG(ERROR) << "[DBPAExecutor] ERROR: Cannot create executor with null agent"; + throw std::invalid_argument("DBPAExecutor: Cannot create executor with null agent"); + } + + ARROW_LOG(DEBUG) << "[DBPAExecutor] Constructor called with timeouts - init: " << init_timeout + << "ms, encrypt: " << encrypt_timeout << "ms, decrypt: " << decrypt_timeout << "ms"; + + // Validate timeout values + if (init_timeout_milliseconds_ <= 0 || encrypt_timeout_milliseconds_ <= 0 || decrypt_timeout_milliseconds_ <= 0) { + ARROW_LOG(ERROR) << "[DBPAExecutor] ERROR: Invalid timeout values - init: " << init_timeout_milliseconds_ + << ", encrypt: " << encrypt_timeout_milliseconds_ + << ", decrypt: " << decrypt_timeout_milliseconds_; + throw std::invalid_argument("DBPAExecutor: All timeout values must be positive"); + } + + ARROW_LOG(DEBUG) << "[DBPAExecutor] Constructor completed successfully"; +} + +void DBPAExecutor::init( + std::string column_name, + std::map connection_config, + std::string app_context, + std::string column_key_id, + Type::type data_type, + std::optional datatype_length, + CompressionCodec::type compression_type, + std::optional> column_encryption_metadata) { + + ARROW_LOG(DEBUG) << "[DBPAExecutor] init() called for column: " << column_name + << ", key_id: " << column_key_id; + + ExecuteWithTimeout("init", init_timeout_milliseconds_, + [this](std::string col_name, + std::map conn_config, + std::string app_ctx, + std::string col_key_id, + Type::type dt, + std::optional dt_len, + CompressionCodec::type comp_type, + std::optional> col_enc_metadata) { + wrapped_agent_->init(std::move(col_name), std::move(conn_config), + std::move(app_ctx), std::move(col_key_id), + dt, dt_len, comp_type, std::move(col_enc_metadata)); + }, + std::move(column_name), std::move(connection_config), + std::move(app_context), std::move(column_key_id), + data_type, datatype_length, compression_type, std::move(column_encryption_metadata)); +} + +std::unique_ptr DBPAExecutor::Encrypt( + span plaintext, + std::map encoding_attributes) { + + ARROW_LOG(DEBUG) << "[DBPAExecutor] Encrypt() called with " << plaintext.size() << " bytes"; + + return ExecuteWithTimeout("encrypt", encrypt_timeout_milliseconds_, + [this](span pt, std::map attrs) { + return wrapped_agent_->Encrypt(pt, std::move(attrs)); + }, + plaintext, std::move(encoding_attributes)); +} + +std::unique_ptr DBPAExecutor::Decrypt( + span ciphertext, + std::map encoding_attributes) { + + ARROW_LOG(DEBUG) << "[DBPAExecutor] Decrypt() called with " << ciphertext.size() << " bytes"; + + return ExecuteWithTimeout("decrypt", decrypt_timeout_milliseconds_, + [this](span ct, std::map attrs) { + return wrapped_agent_->Decrypt(ct, std::move(attrs)); + }, + ciphertext, std::move(encoding_attributes)); +} + +} // namespace parquet::encryption::external diff --git a/cpp/src/parquet/encryption/external/dbpa_executor.h b/cpp/src/parquet/encryption/external/dbpa_executor.h new file mode 100644 index 000000000000..2e890bcf2ad7 --- /dev/null +++ b/cpp/src/parquet/encryption/external/dbpa_executor.h @@ -0,0 +1,122 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +template +using span = tcb::span; + +namespace parquet::encryption::external { + +using dbps::external::DataBatchProtectionAgentInterface; +using dbps::external::EncryptionResult; +using dbps::external::DecryptionResult; +using dbps::external::Type; +using dbps::external::CompressionCodec; + +class DBPAExecutorTimeoutException; + +/** + * DBPAExecutor - A decorator for DataBatchProtectionAgentInterface with timeout support + * Original exceptions from wrapped agents are preserved and re-thrown unchanged. + */ +class DBPAExecutor : public DataBatchProtectionAgentInterface { + public: + /** + * Constructor that takes ownership of the wrapped agent with configurable timeouts + * @param agent The DataBatchProtectionAgentInterface instance to wrap + * @param init_timeout Timeout for init operations in milliseconds (default: 10000) + * @param encrypt_timeout Timeout for encrypt operations in milliseconds (default: 30000) + * @param decrypt_timeout Timeout for decrypt operations in milliseconds (default: 30000) + */ + explicit DBPAExecutor(std::unique_ptr agent, + int64_t init_timeout = 10000, + int64_t encrypt_timeout = 30000, + int64_t decrypt_timeout = 30000); + + /** + * Destructor + */ + ~DBPAExecutor() override = default; + + /** + * Initialize the agent with configuration parameters + * Executes with timeout - original exceptions preserved + * @throws DBPAExecutorTimeoutException if operation times out + * @throws Original exceptions from wrapped agent (unchanged!) + */ + void init( + std::string column_name, + std::map connection_config, + std::string app_context, + std::string column_key_id, + Type::type data_type, + std::optional datatype_length, + CompressionCodec::type compression_type, + std::optional> column_encryption_metadata) override; + + /** + * Encrypt the provided plaintext + * Executes with timeout - original exceptions preserved + * @param plaintext The data to encrypt + * @return Unique pointer to EncryptionResult + * @throws DBPAExecutorTimeoutException if operation times out + * @throws Original exceptions from wrapped agent (unchanged!) + */ + std::unique_ptr Encrypt( + span plaintext, + std::map encoding_attributes) override; + + /** + * Decrypt the provided ciphertext + * Executes with timeout - original exceptions preserved + * @param ciphertext The data to decrypt + * @return Unique pointer to DecryptionResult + * @throws DBPAExecutorTimeoutException if operation times out + * @throws Original exceptions from wrapped agent (unchanged!) + */ + std::unique_ptr Decrypt( + span ciphertext, + std::map encoding_attributes) override; + + private: + std::unique_ptr wrapped_agent_; + int64_t init_timeout_milliseconds_; + int64_t encrypt_timeout_milliseconds_; + int64_t decrypt_timeout_milliseconds_; +}; // class DBPAExecutor + +/** + * Exception thrown when a DBPA operation times out + */ + class DBPAExecutorTimeoutException : public std::runtime_error { + public: + explicit DBPAExecutorTimeoutException(const std::string& operation, int64_t timeout_milliseconds) + : std::runtime_error("DBPAExecutor: " + operation + " operation timed out after " + + std::to_string(timeout_milliseconds) + " milliseconds") {} + }; + +} // namespace parquet::encryption::external diff --git a/cpp/src/parquet/encryption/external/dbpa_executor_test.cc b/cpp/src/parquet/encryption/external/dbpa_executor_test.cc new file mode 100644 index 000000000000..4cecd9fdbfd5 --- /dev/null +++ b/cpp/src/parquet/encryption/external/dbpa_executor_test.cc @@ -0,0 +1,500 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "parquet/encryption/external/dbpa_executor.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace parquet::encryption::external { + +// Concrete implementations for testing +class TestEncryptionResult : public EncryptionResult { +public: + TestEncryptionResult(std::vector data, bool success = true, + std::string error_msg = "", + std::map error_fields = {}, + std::optional> metadata = std::nullopt) + : ciphertext_data_(std::move(data)), success_(success), + error_message_(std::move(error_msg)), error_fields_(std::move(error_fields)), + metadata_(std::move(metadata)) {} + + span ciphertext() const override { + return span(ciphertext_data_.data(), ciphertext_data_.size()); + } + + std::size_t size() const override { return ciphertext_data_.size(); } + bool success() const override { return success_; } + const std::optional> encryption_metadata() const override { + return metadata_; + } + const std::string& error_message() const override { return error_message_; } + const std::map& error_fields() const override { return error_fields_; } + +private: + std::vector ciphertext_data_; + bool success_; + std::string error_message_; + std::map error_fields_; + std::optional> metadata_; +}; + +class TestDecryptionResult : public DecryptionResult { +public: + TestDecryptionResult(std::vector data, bool success = true, + std::string error_msg = "", + std::map error_fields = {}) + : plaintext_data_(std::move(data)), success_(success), + error_message_(std::move(error_msg)), error_fields_(std::move(error_fields)) {} + + span plaintext() const override { + return span(plaintext_data_.data(), plaintext_data_.size()); + } + + std::size_t size() const override { return plaintext_data_.size(); } + bool success() const override { return success_; } + const std::string& error_message() const override { return error_message_; } + const std::map& error_fields() const override { return error_fields_; } + +private: + std::vector plaintext_data_; + bool success_; + std::string error_message_; + std::map error_fields_; +}; + +// Mock agent that tracks method calls and parameters for verification +class MockDBPAAgent : public DataBatchProtectionAgentInterface { +public: + // Track call counts + int init_call_count_ = 0; + int encrypt_call_count_ = 0; + int decrypt_call_count_ = 0; + + // Track parameters from last calls + std::string last_init_column_name_; + std::map last_init_connection_config_; + std::string last_init_app_context_; + std::string last_init_column_key_id_; + Type::type last_init_data_type_; + CompressionCodec::type last_init_compression_type_; + std::optional last_init_datatype_length_; + std::optional> last_init_column_encryption_metadata_; + std::map last_encrypt_encoding_attrs_; + std::map last_decrypt_encoding_attrs_; + + std::vector last_encrypt_plaintext_; + std::vector last_decrypt_ciphertext_; + + // Mock results + std::unique_ptr mock_encrypt_result_; + std::unique_ptr mock_decrypt_result_; + + // Control behavior + bool should_throw_on_init_ = false; + bool should_throw_on_encrypt_ = false; + bool should_throw_on_decrypt_ = false; + std::string throw_message_ = "Mock agent error"; + + void init(std::string column_name, + std::map connection_config, + std::string app_context, + std::string column_key_id, + Type::type data_type, + std::optional datatype_length, + CompressionCodec::type compression_type, + std::optional> column_encryption_metadata) override { + init_call_count_++; + last_init_column_name_ = column_name; + last_init_connection_config_ = connection_config; + last_init_app_context_ = app_context; + last_init_column_key_id_ = column_key_id; + last_init_data_type_ = data_type; + last_init_compression_type_ = compression_type; + last_init_datatype_length_ = datatype_length; + last_init_column_encryption_metadata_ = std::move(column_encryption_metadata); + + if (should_throw_on_init_) { + throw std::runtime_error(throw_message_); + } + } + + std::unique_ptr Encrypt(span plaintext, + std::map encoding_attributes) override { + encrypt_call_count_++; + last_encrypt_plaintext_.assign(plaintext.begin(), plaintext.end()); + last_encrypt_encoding_attrs_ = std::move(encoding_attributes); + + if (should_throw_on_encrypt_) { + throw std::runtime_error(throw_message_); + } + + // Return a copy of the mock result if available + if (mock_encrypt_result_) { + // Create a new result with the same data + return std::make_unique(*mock_encrypt_result_); + } + return nullptr; + } + + std::unique_ptr Decrypt(span ciphertext, + std::map encoding_attributes) override { + decrypt_call_count_++; + last_decrypt_ciphertext_.assign(ciphertext.begin(), ciphertext.end()); + last_decrypt_encoding_attrs_ = std::move(encoding_attributes); + + if (should_throw_on_decrypt_) { + throw std::runtime_error(throw_message_); + } + + // Return a copy of the mock result if available + if (mock_decrypt_result_) { + // Create a new result with the same data + return std::make_unique(*mock_decrypt_result_); + } + return nullptr; + } + + // Helper methods for test setup + void ResetCallCounts() { + init_call_count_ = 0; + encrypt_call_count_ = 0; + decrypt_call_count_ = 0; + } + + void SetMockEncryptResult(std::unique_ptr result) { + mock_encrypt_result_ = std::move(result); + } + + void SetMockDecryptResult(std::unique_ptr result) { + mock_decrypt_result_ = std::move(result); + } +}; + +class DBPAExecutorTest : public ::testing::Test { + protected: + void SetUp() override { + // Create a mock agent to wrap + mock_agent_ = std::make_unique(); + mock_agent_ptr_ = mock_agent_.get(); // Keep raw pointer for verification + + // Create the executor that wraps the mock agent with custom timeouts + executor_ = std::make_unique(std::move(mock_agent_), + 1000, // init timeout: 1 second + 2000, // encrypt timeout: 2 seconds + 2000); // decrypt timeout: 2 seconds + } + + std::unique_ptr executor_; + std::unique_ptr mock_agent_; + MockDBPAAgent* mock_agent_ptr_; // Raw pointer for verification +}; + +TEST_F(DBPAExecutorTest, ConstructorWithNullAgentThrows) { + EXPECT_THROW(DBPAExecutor(nullptr), std::invalid_argument); +} + +TEST_F(DBPAExecutorTest, ConstructorWithInvalidTimeoutsThrows) { + auto test_agent = std::make_unique(); + + // Test negative timeout + EXPECT_THROW(DBPAExecutor(std::move(test_agent), -1, 1000, 1000), std::invalid_argument); + + // Test zero timeout + test_agent = std::make_unique(); + EXPECT_THROW(DBPAExecutor(std::move(test_agent), 0, 1000, 1000), std::invalid_argument); +} + +TEST_F(DBPAExecutorTest, InitForwardsToWrappedAgent) { + std::string column_name = "test_column"; + std::map connection_config = {{"key", "value"}, {"server", "localhost"}}; + std::string app_context = "test_context"; + std::string column_key_id = "test_key_id"; + Type::type data_type = Type::type::INT32; + CompressionCodec::type compression_type = CompressionCodec::type::UNCOMPRESSED; + + // Reset call count before test + mock_agent_ptr_->ResetCallCounts(); + + // Call init through executor + std::optional> column_encryption_metadata = std::map{{"metaKey", "metaValue"}}; + EXPECT_NO_THROW(executor_->init(column_name, connection_config, app_context, + column_key_id, data_type, std::nullopt, compression_type, column_encryption_metadata)); + + // Verify the mock agent was called exactly once + EXPECT_EQ(mock_agent_ptr_->init_call_count_, 1); + + // Verify all parameters were forwarded correctly + EXPECT_EQ(mock_agent_ptr_->last_init_column_name_, column_name); + EXPECT_EQ(mock_agent_ptr_->last_init_connection_config_, connection_config); + EXPECT_EQ(mock_agent_ptr_->last_init_app_context_, app_context); + EXPECT_EQ(mock_agent_ptr_->last_init_column_key_id_, column_key_id); + EXPECT_EQ(mock_agent_ptr_->last_init_data_type_, data_type); + EXPECT_EQ(mock_agent_ptr_->last_init_compression_type_, compression_type); + ASSERT_TRUE(mock_agent_ptr_->last_init_column_encryption_metadata_.has_value()); + EXPECT_EQ(mock_agent_ptr_->last_init_column_encryption_metadata_.value().at("metaKey"), "metaValue"); +} + +TEST_F(DBPAExecutorTest, EncryptForwardsToWrappedAgent) { + // Initialize the executor first + executor_->init("test_column", {}, "test_context", "test_key_id", + Type::type::INT32, std::nullopt, CompressionCodec::type::UNCOMPRESSED, std::nullopt); + + // Create test data + std::vector plaintext = {1, 2, 3, 4, 5}; + span plaintext_span(plaintext); + + // Reset call count before test + mock_agent_ptr_->ResetCallCounts(); + + // Encrypt should not throw and should return a result + std::map attrs = {{"format", "plain"}}; + auto result = executor_->Encrypt(plaintext_span, attrs); + + // Verify the mock agent was called exactly once + EXPECT_EQ(mock_agent_ptr_->encrypt_call_count_, 1); + + // Verify the plaintext was forwarded correctly + EXPECT_EQ(mock_agent_ptr_->last_encrypt_plaintext_, plaintext); + EXPECT_EQ(mock_agent_ptr_->last_encrypt_encoding_attrs_, attrs); + + // Note: result might be nullptr since we didn't set up a mock result + // The important thing is that the call was forwarded +} + +TEST_F(DBPAExecutorTest, DecryptForwardsToWrappedAgent) { + // Initialize the executor first + executor_->init("test_column", {}, "test_context", "test_key_id", + Type::type::INT32, std::nullopt, CompressionCodec::type::UNCOMPRESSED, std::nullopt); + + // Create test data + std::vector ciphertext = {5, 4, 3, 2, 1}; + span ciphertext_span(ciphertext); + + // Reset call count before test + mock_agent_ptr_->ResetCallCounts(); + + // Decrypt should not throw and should return a result + std::map attrs = {{"format", "plain"}}; + auto result = executor_->Decrypt(ciphertext_span, attrs); + + // Verify the mock agent was called exactly once + EXPECT_EQ(mock_agent_ptr_->decrypt_call_count_, 1); + + // Verify the ciphertext was forwarded correctly + EXPECT_EQ(mock_agent_ptr_->last_decrypt_ciphertext_, ciphertext); + EXPECT_EQ(mock_agent_ptr_->last_decrypt_encoding_attrs_, attrs); + + // Note: result might be nullptr since we didn't set up a mock result + // The important thing is that the call was forwarded +} + +TEST_F(DBPAExecutorTest, InitForwardsDatatypeLength) { + mock_agent_ptr_->ResetCallCounts(); + EXPECT_NO_THROW(executor_->init("col", {}, "ctx", "kid", + Type::type::INT32, 16, CompressionCodec::type::UNCOMPRESSED, std::nullopt)); + EXPECT_EQ(mock_agent_ptr_->init_call_count_, 1); + ASSERT_TRUE(mock_agent_ptr_->last_init_datatype_length_.has_value()); + EXPECT_EQ(mock_agent_ptr_->last_init_datatype_length_.value(), 16); +} + +// Test that multiple calls are properly forwarded +TEST_F(DBPAExecutorTest, MultipleCallsAreProperlyForwarded) { + // Reset call counts + mock_agent_ptr_->ResetCallCounts(); + + // Make multiple init calls with different parameters + executor_->init("column1", {{"key1", "value1"}}, "context1", "key1", + Type::type::INT32, std::nullopt, CompressionCodec::type::UNCOMPRESSED, std::nullopt); + + // Verify both calls were made + EXPECT_EQ(mock_agent_ptr_->init_call_count_, 1); + + // Verify the last call parameters (second call) + EXPECT_EQ(mock_agent_ptr_->last_init_column_name_, "column1"); + EXPECT_EQ(mock_agent_ptr_->last_init_connection_config_["key1"], "value1"); + EXPECT_EQ(mock_agent_ptr_->last_init_app_context_, "context1"); + EXPECT_EQ(mock_agent_ptr_->last_init_column_key_id_, "key1"); + EXPECT_EQ(mock_agent_ptr_->last_init_data_type_, Type::type::INT32); + EXPECT_EQ(mock_agent_ptr_->last_init_compression_type_, CompressionCodec::type::UNCOMPRESSED); + + // Make multiple encrypt calls + std::vector data1 = {1, 2, 3}; + std::vector data2 = {4, 5, 6, 7}; + span span1(data1); + span span2(data2); + + executor_->Encrypt(span1, {}); + executor_->Encrypt(span2, {}); + + // Verify both encrypt calls were made + EXPECT_EQ(mock_agent_ptr_->encrypt_call_count_, 2); + + // Verify the last encrypt call parameters + EXPECT_EQ(mock_agent_ptr_->last_encrypt_plaintext_, data2); + + // Make multiple decrypt calls + std::vector cipher1 = {8, 9, 10}; + std::vector cipher2 = {11, 12, 13, 14, 15}; + span cipher_span1(cipher1); + span cipher_span2(cipher2); + + executor_->Decrypt(cipher_span1, {}); + executor_->Decrypt(cipher_span2, {}); + + // Verify both decrypt calls were made + EXPECT_EQ(mock_agent_ptr_->decrypt_call_count_, 2); + + // Verify the last decrypt call parameters + EXPECT_EQ(mock_agent_ptr_->last_decrypt_ciphertext_, cipher2); +} + +// Test timeout functionality +TEST_F(DBPAExecutorTest, TimeoutExceptionThrownOnSlowOperation) { + // Create a mock agent that simulates slow operations + class SlowMockAgent : public DataBatchProtectionAgentInterface { + public: + void init(std::string, std::map, std::string, + std::string, Type::type, std::optional, CompressionCodec::type, + std::optional>) override { + // Simulate slow operation that takes 200ms + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + std::unique_ptr Encrypt(span, std::map) override { + // Simulate slow operation that takes 150ms + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + return nullptr; + } + + std::unique_ptr Decrypt(span, std::map) override { + // Simulate slow operation that takes 120ms + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + return nullptr; + } + }; + + // Create an executor with very short timeouts + auto slow_agent = std::make_unique(); + auto timeout_executor = std::make_unique(std::move(slow_agent), + 50, // init timeout: 50ms (will timeout) + 55, // encrypt timeout: 50ms (will timeout) + 60); // decrypt timeout: 50ms (will timeout) + + // Test init timeout - should throw DBPAExecutorTimeoutException + try { + timeout_executor->init("test_column", {}, "test_context", "test_key_id", + Type::type::INT32, std::nullopt, CompressionCodec::type::UNCOMPRESSED, std::nullopt); + FAIL() << "Expected DBPAExecutorTimeoutException to be thrown"; + } catch (const DBPAExecutorTimeoutException& e) { + // Verify the timeout exception contains expected information + std::string error_msg = e.what(); + EXPECT_TRUE(error_msg.find("init") != std::string::npos); + EXPECT_TRUE(error_msg.find("50") != std::string::npos); + EXPECT_TRUE(error_msg.find("milliseconds") != std::string::npos); + } catch (...) { + FAIL() << "Expected DBPAExecutorTimeoutException, but got different exception type"; + } + + // Test encrypt timeout - should throw DBPAExecutorTimeoutException + std::vector data = {1, 2, 3, 4, 5}; + span data_span(data); + + try { + timeout_executor->Encrypt(data_span, {}); + FAIL() << "Expected DBPAExecutorTimeoutException to be thrown"; + } catch (const DBPAExecutorTimeoutException& e) { + // Verify the timeout exception contains expected information + std::string error_msg = e.what(); + EXPECT_TRUE(error_msg.find("encrypt") != std::string::npos); + EXPECT_TRUE(error_msg.find("55") != std::string::npos); + EXPECT_TRUE(error_msg.find("milliseconds") != std::string::npos); + } catch (...) { + FAIL() << "Expected DBPAExecutorTimeoutException, but got different exception type"; + } + + // Test decrypt timeout - should throw DBPAExecutorTimeoutException + try { + timeout_executor->Decrypt(data_span, {}); + FAIL() << "Expected DBPAExecutorTimeoutException to be thrown"; + } catch (const DBPAExecutorTimeoutException& e) { + // Verify the timeout exception contains expected information + std::string error_msg = e.what(); + EXPECT_TRUE(error_msg.find("decrypt") != std::string::npos); + EXPECT_TRUE(error_msg.find("60") != std::string::npos); + EXPECT_TRUE(error_msg.find("milliseconds") != std::string::npos); + } catch (...) { + FAIL() << "Expected DBPAExecutorTimeoutException, but got different exception type"; + } +} + +// Test that original exceptions are preserved (not wrapped) +TEST_F(DBPAExecutorTest, OriginalExceptionsArePreserved) { + // Configure mock agent to throw exceptions + mock_agent_ptr_->should_throw_on_init_ = true; + mock_agent_ptr_->throw_message_ = "Mock init error"; + + // Test that init exception is preserved + try { + executor_->init("test_column", {}, "test_context", "test_key_id", + Type::type::INT32, std::nullopt, CompressionCodec::type::UNCOMPRESSED, std::nullopt); + FAIL() << "Expected std::runtime_error to be thrown"; + } catch (const std::runtime_error& e) { + EXPECT_STREQ(e.what(), "Mock init error"); + } catch (...) { + FAIL() << "Unexpected exception type"; + } + + // Reset and test encrypt exception + mock_agent_ptr_->should_throw_on_init_ = false; + mock_agent_ptr_->should_throw_on_encrypt_ = true; + mock_agent_ptr_->throw_message_ = "Mock encrypt error"; + + std::vector data = {1, 2, 3, 4, 5}; + span data_span(data); + + try { + executor_->Encrypt(data_span, {}); + FAIL() << "Expected std::runtime_error to be thrown"; + } catch (const std::runtime_error& e) { + EXPECT_STREQ(e.what(), "Mock encrypt error"); + } catch (...) { + FAIL() << "Unexpected exception type"; + } + + // Reset and test decrypt exception + mock_agent_ptr_->should_throw_on_encrypt_ = false; + mock_agent_ptr_->should_throw_on_decrypt_ = true; + mock_agent_ptr_->throw_message_ = "Mock decrypt error"; + + try { + executor_->Decrypt(data_span, {}); + FAIL() << "Expected std::runtime_error to be thrown"; + } catch (const std::runtime_error& e) { + EXPECT_STREQ(e.what(), "Mock decrypt error"); + } catch (...) { + FAIL() << "Unexpected exception type"; + } +} + +} // namespace parquet::encryption::external diff --git a/cpp/src/parquet/encryption/external/dbpa_library_wrapper.cc b/cpp/src/parquet/encryption/external/dbpa_library_wrapper.cc new file mode 100644 index 000000000000..34e904e0bc10 --- /dev/null +++ b/cpp/src/parquet/encryption/external/dbpa_library_wrapper.cc @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "parquet/encryption/external/dbpa_library_wrapper.h" +#include + +#include +#include + +#include + +#include "arrow/util/io_util.h" +#include "arrow/util/logging.h" + +namespace parquet::encryption::external { + +// Default implementation for handle closing function +void DefaultSharedLibraryClosingFn(void* library_handle) { + auto status = arrow::internal::CloseDynamicLibrary(library_handle); + if (!status.ok()) { + ARROW_LOG(WARNING) << "Error closing library: " << status.message(); + } +} + +DBPALibraryWrapper::DBPALibraryWrapper( + std::unique_ptr agent, + void* library_handle, + std::function handle_closing_fn) + : wrapped_agent_(std::move(agent)), + library_handle_(library_handle), + handle_closing_fn_(std::move(handle_closing_fn)) { + // Ensure the wrapped agent is not null + if (!wrapped_agent_) { + throw std::invalid_argument("DBPAWrapper: Cannot create wrapper with null agent"); + } + if (!library_handle_) { + throw std::invalid_argument("DBPAWrapper: Cannot create wrapper with null library handle"); + } + if (!handle_closing_fn_) { + throw std::invalid_argument("DBPAWrapper: Cannot create wrapper with null handle closing function"); + } +} + +// DBPALibraryWrapper destructor +// This is the main reason for the decorator/wrapper. +// This will (a) destroy the wrapped agent, and (b) close the shared library. +// While the wrapped_agent_ would automatically be destroyed when this object is destroyed +// we need to explicitly destroy **before** we are able to close the shared library. +// Doing it in a different order, may cause issues, as by unloading the library may cause the class +// definition to be unloaded before the destructor completes, and that is likely to cause issues +// (such as a segfault). +DBPALibraryWrapper::~DBPALibraryWrapper() { + // Explicitly destroy the wrapped agent first + if (wrapped_agent_) { + DataBatchProtectionAgentInterface* wrapped_agent = wrapped_agent_.release(); + delete wrapped_agent; + } + + // Now we can close the shared library using the provided function + handle_closing_fn_(library_handle_); + library_handle_ = nullptr; +} //DBPALibraryWrapper::~DBPALibraryWrapper() +} // namespace parquet::encryption::external diff --git a/cpp/src/parquet/encryption/external/dbpa_library_wrapper.h b/cpp/src/parquet/encryption/external/dbpa_library_wrapper.h new file mode 100644 index 000000000000..c233a884608a --- /dev/null +++ b/cpp/src/parquet/encryption/external/dbpa_library_wrapper.h @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include + +template +using span = tcb::span; + +namespace parquet::encryption::external { + +using dbps::external::DataBatchProtectionAgentInterface; +using dbps::external::EncryptionResult; +using dbps::external::DecryptionResult; +using dbps::external::Type; +using dbps::external::CompressionCodec; + +// Default implementation for shared library closing function +// This is passed into the constructor of DBPALibraryWrapper, +// and is used as the default function to close the shared library. +void DefaultSharedLibraryClosingFn(void* library_handle); + +// Decorator/Wrapper class for the DataBatchProtectionAgentInterface +// Its main purpose is to close the shared library when Arrow is about to destroy +// an instance of an DBPAgent +// +// In the constructor we allow to pass a function that will be used to close the shared library. +// This simplifies testing, as we can use a mock function to avoid actually closing the shared library. +class DBPALibraryWrapper : public DataBatchProtectionAgentInterface { + private: + std::unique_ptr wrapped_agent_; + void* library_handle_; + std::function handle_closing_fn_; + + public: + // Constructor that takes ownership of the wrapped agent + explicit DBPALibraryWrapper( + std::unique_ptr agent, + void* library_handle, + std::function handle_closing_fn = &DefaultSharedLibraryClosingFn); + + // Destructor + // This is the main reason for the decorator/wrapper. + // This will (a) destroy the wrapped agent, and (b) close the shared library. + // While the wrapped_agent_ would automatically be destroyed when this object is destroyed + // we need to explicitly destroy **before** we are able to close the shared library. + // Doing it in a different order, may cause issues, as by unloading the library may cause the class + // definition to be unloaded before the destructor completes, and that is likely to cause issues + // (such as a segfault). + ~DBPALibraryWrapper(); + + // Decorator implementation of init method + inline void init( + std::string column_name, + std::map connection_config, + std::string app_context, + std::string column_key_id, + Type::type data_type, + std::optional datatype_length, + CompressionCodec::type compression_type, + std::optional> column_encryption_metadata) override { + wrapped_agent_->init(std::move(column_name), std::move(connection_config), + std::move(app_context), std::move(column_key_id), + data_type, datatype_length, compression_type, std::move(column_encryption_metadata)); + } + + // Decorator implementation of Encrypt method - inlined for performance + inline std::unique_ptr Encrypt( + span plaintext, + std::map encoding_attributes) override { + return wrapped_agent_->Encrypt(plaintext, std::move(encoding_attributes)); + } + + // Decorator implementation of Decrypt method - inlined for performance + inline std::unique_ptr Decrypt( + span ciphertext, + std::map encoding_attributes) override { + return wrapped_agent_->Decrypt(ciphertext, std::move(encoding_attributes)); + } +}; + +} // namespace parquet::encryption::external diff --git a/cpp/src/parquet/encryption/external/dbpa_library_wrapper_test.cc b/cpp/src/parquet/encryption/external/dbpa_library_wrapper_test.cc new file mode 100644 index 000000000000..2404ed7dab64 --- /dev/null +++ b/cpp/src/parquet/encryption/external/dbpa_library_wrapper_test.cc @@ -0,0 +1,1072 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "parquet/encryption/external/dbpa_library_wrapper.h" + +#include "parquet/test_util.h" + +template +using span = tcb::span; + +namespace parquet::encryption::external::test { + +using dbps::external::EncryptionResult; +using dbps::external::DecryptionResult; +using dbps::external::Type; +using dbps::external::CompressionCodec; + +// Simple implementation of EncryptionResult for testing +class TestEncryptionResult : public EncryptionResult { +public: + TestEncryptionResult(std::vector data, bool success = true, + std::string error_msg = "", + std::map error_fields = {}, + std::optional> metadata = std::nullopt) + : ciphertext_data_(std::move(data)), success_(success), + error_message_(std::move(error_msg)), error_fields_(std::move(error_fields)), + metadata_(std::move(metadata)) {} + + span ciphertext() const override { + return span(ciphertext_data_.data(), ciphertext_data_.size()); + } + + std::size_t size() const override { return ciphertext_data_.size(); } + bool success() const override { return success_; } + const std::optional> encryption_metadata() const override { + return metadata_; + } + const std::string& error_message() const override { return error_message_; } + const std::map& error_fields() const override { return error_fields_; } + +private: + std::vector ciphertext_data_; + bool success_; + std::string error_message_; + std::map error_fields_; + std::optional> metadata_; +}; + +// Simple implementation of DecryptionResult for testing +class TestDecryptionResult : public DecryptionResult { +public: + TestDecryptionResult(std::vector data, bool success = true, + std::string error_msg = "", + std::map error_fields = {}) + : plaintext_data_(std::move(data)), success_(success), + error_message_(std::move(error_msg)), error_fields_(std::move(error_fields)) {} + + span plaintext() const override { + return span(plaintext_data_.data(), plaintext_data_.size()); + } + + std::size_t size() const override { return plaintext_data_.size(); } + bool success() const override { return success_; } + const std::string& error_message() const override { return error_message_; } + const std::map& error_fields() const override { return error_fields_; } + +private: + std::vector plaintext_data_; + bool success_; + std::string error_message_; + std::map error_fields_; +}; + +// Companion object to track the order of destruction events +class DestructionOrderTracker { + public: + DestructionOrderTracker() : sequence_counter_(0) {} + + // Record an event with a sequence number + void RecordEvent(const std::string& event_name) { + events_.emplace_back(event_name, ++sequence_counter_); + } + + // Get the sequence number for a specific event + int GetEventSequence(const std::string& event_name) const { + for (const auto& event : events_) { + if (event.first == event_name) { + return event.second; + } + } + return -1; // Event not found + } + + // Verify that first_event occurred before second_event + bool VerifyOrder(const std::string& first_event, const std::string& second_event) const { + int first_seq = GetEventSequence(first_event); + int second_seq = GetEventSequence(second_event); + + if (first_seq == -1 || second_seq == -1) { + return false; // One or both events not recorded + } + + return first_seq < second_seq; + } + + // Get all recorded events in order + const std::vector>& GetEvents() const { + return events_; + } + + // Clear all recorded events + void Clear() { + events_.clear(); + sequence_counter_ = 0; + } + + // Check if an event was recorded + bool WasEventRecorded(const std::string& event_name) const { + return GetEventSequence(event_name) != -1; + } + + private: + std::vector> events_; + int sequence_counter_; +}; //DestructionOrderTracker + +// Companion object to hold mock state that persists after mock instance destruction +class MockCompanionDBPA { + public: + MockCompanionDBPA(std::shared_ptr order_tracker = nullptr) + : encrypt_called_(false), + decrypt_called_(false), + init_called_(false), + destructor_called_(false), + encrypt_count_(0), + decrypt_count_(0), + init_count_(0), + order_tracker_(order_tracker ? order_tracker : std::make_shared()) {} + + // Test helper methods + bool WasEncryptCalled() const { return encrypt_called_; } + bool WasDecryptCalled() const { return decrypt_called_; } + bool WasInitCalled() const { return init_called_; } + bool WasDestructorCalled() const { return destructor_called_; } + int GetEncryptCount() const { return encrypt_count_; } + int GetDecryptCount() const { return decrypt_count_; } + int GetInitCount() const { return init_count_; } + const std::vector& GetEncryptPlaintext() const { return encrypt_plaintext_; } + const std::vector& GetDecryptCiphertext() const { return decrypt_ciphertext_; } + size_t GetEncryptCiphertextSize() const { return encrypt_ciphertext_size_; } + std::shared_ptr GetOrderTracker() const { return order_tracker_; } + const std::map& GetLastEncryptEncodingAttrs() const { + return last_encrypt_encoding_attrs_; + } + const std::map& GetLastDecryptEncodingAttrs() const { + return last_decrypt_encoding_attrs_; + } + void SetNextEncryptResultMetadata(std::optional> metadata) { + next_encrypt_result_metadata_ = std::move(metadata); + } + std::optional> ConsumeNextEncryptResultMetadata() { + auto tmp = std::move(next_encrypt_result_metadata_); + next_encrypt_result_metadata_.reset(); + return tmp; + } + + // Init tracking methods + const std::string& GetInitColumnName() const { return init_column_name_; } + const std::map& GetInitConnectionConfig() const { return init_connection_config_; } + const std::string& GetInitAppContext() const { return init_app_context_; } + const std::string& GetInitColumnKeyId() const { return init_column_key_id_; } + Type::type GetInitDataType() const { return init_data_type_; } + CompressionCodec::type GetInitCompressionType() const { return init_compression_type_; } + std::optional GetInitDatatypeLength() const { return init_datatype_length_; } + const std::optional>& GetInitColumnEncryptionMetadata() const { return init_column_encryption_metadata_; } + + // State update methods (called by the mock instance) + void SetEncryptCalled(bool called) { encrypt_called_ = called; } + void SetDecryptCalled(bool called) { decrypt_called_ = called; } + void SetInitCalled(bool called) { init_called_ = called; } + void SetDestructorCalled(bool called) { + destructor_called_ = called; + if (called) { + order_tracker_->RecordEvent("agent_destructor"); + } + } + void IncrementEncryptCount() { encrypt_count_++; } + void IncrementDecryptCount() { decrypt_count_++; } + void IncrementInitCount() { init_count_++; } + void SetEncryptPlaintext(const std::vector& plaintext) { encrypt_plaintext_ = plaintext; } + void SetDecryptCiphertext(const std::vector& ciphertext) { decrypt_ciphertext_ = ciphertext; } + void SetEncryptCiphertextSize(size_t size) { encrypt_ciphertext_size_ = size; } + void SetLastEncryptEncodingAttrs(std::map attrs) { + last_encrypt_encoding_attrs_ = std::move(attrs); + } + void SetLastDecryptEncodingAttrs(std::map attrs) { + last_decrypt_encoding_attrs_ = std::move(attrs); + } + + // Init parameter tracking + void SetInitParameters( + std::string column_name, + std::map connection_config, + std::string app_context, + std::string column_key_id, + Type::type data_type, + CompressionCodec::type compression_type, + std::optional datatype_length = std::nullopt, + std::optional> column_encryption_metadata = std::nullopt) { + init_column_name_ = std::move(column_name); + init_connection_config_ = std::move(connection_config); + init_app_context_ = std::move(app_context); + init_column_key_id_ = std::move(column_key_id); + init_data_type_ = data_type; + init_compression_type_ = compression_type; + init_datatype_length_ = datatype_length; + init_column_encryption_metadata_ = std::move(column_encryption_metadata); + } + + private: + bool encrypt_called_; + bool decrypt_called_; + bool init_called_; + bool destructor_called_; + int encrypt_count_; + int decrypt_count_; + int init_count_; + std::vector encrypt_plaintext_; + std::vector decrypt_ciphertext_; + size_t encrypt_ciphertext_size_; + std::shared_ptr order_tracker_; + + // Init parameters + std::string init_column_name_; + std::map init_connection_config_; + std::string init_app_context_; + std::string init_column_key_id_; + Type::type init_data_type_; + CompressionCodec::type init_compression_type_; + std::optional init_datatype_length_; + std::optional> init_column_encryption_metadata_; + std::map last_encrypt_encoding_attrs_; + std::map last_decrypt_encoding_attrs_; + std::optional> next_encrypt_result_metadata_; +}; //MockCompanionDBPA + +// Companion object to track shared library handle management operations +class SharedLibHandleManagementCompanion { + public: + SharedLibHandleManagementCompanion(std::shared_ptr order_tracker = nullptr) + : handle_close_called_(false), + handle_close_count_(0), + last_closed_handle_(nullptr), + order_tracker_(order_tracker ? order_tracker : std::make_shared()) {} + + // Test helper methods + bool WasHandleCloseCalled() const { return handle_close_called_; } + int GetHandleCloseCount() const { return handle_close_count_; } + void* GetLastClosedHandle() const { return last_closed_handle_; } + std::shared_ptr GetOrderTracker() const { return order_tracker_; } + + // State update methods + void SetHandleCloseCalled(bool called) { handle_close_called_ = called; } + void IncrementHandleCloseCount() { handle_close_count_++; } + void SetLastClosedHandle(void* handle) { last_closed_handle_ = handle; } + + // Create a closure that captures this companion object + // and returns a function that can be used to close the shared library handle + std::function CreateHandleClosingFunction() { + return [this](void* library_handle) { + this->SetHandleCloseCalled(true); + this->IncrementHandleCloseCount(); + this->SetLastClosedHandle(library_handle); + this->order_tracker_->RecordEvent("handle_close"); + }; + } + + private: + bool handle_close_called_; + int handle_close_count_; + void* last_closed_handle_; + std::shared_ptr order_tracker_; +}; //SharedLibHandleManagementCompanion + +// Mock implementation of DataBatchProtectionAgentInterface for testing delegation +class MockDataBatchProtectionAgent : public DataBatchProtectionAgentInterface { + public: + explicit MockDataBatchProtectionAgent(std::shared_ptr companion = nullptr) + : companion_(companion ? companion : std::make_shared()) {} + + ~MockDataBatchProtectionAgent() override { + companion_->SetDestructorCalled(true); + } + + void init( + std::string column_name, + std::map connection_config, + std::string app_context, + std::string column_key_id, + Type::type data_type, + std::optional datatype_length, + CompressionCodec::type compression_type, + std::optional> column_encryption_metadata) override { + companion_->SetInitCalled(true); + companion_->IncrementInitCount(); + companion_->SetInitParameters( + std::move(column_name), + std::move(connection_config), + std::move(app_context), + std::move(column_key_id), + data_type, + compression_type, + datatype_length, + std::move(column_encryption_metadata)); + } + + std::unique_ptr Encrypt( + span plaintext, + std::map encoding_attributes) override { + companion_->SetEncryptCalled(true); + companion_->IncrementEncryptCount(); + companion_->SetEncryptPlaintext(std::vector(plaintext.begin(), plaintext.end())); + companion_->SetEncryptCiphertextSize(plaintext.size()); + companion_->SetLastEncryptEncodingAttrs(std::move(encoding_attributes)); + + // Create a simple mock encryption result + std::vector ciphertext_data(plaintext.begin(), plaintext.end()); + auto result_metadata = companion_->ConsumeNextEncryptResultMetadata(); + return std::make_unique(std::move(ciphertext_data), true, "", + std::map{}, + std::move(result_metadata)); + } + + std::unique_ptr Decrypt( + span ciphertext, + std::map encoding_attributes) override { + companion_->SetDecryptCalled(true); + companion_->IncrementDecryptCount(); + companion_->SetDecryptCiphertext(std::vector(ciphertext.begin(), ciphertext.end())); + companion_->SetLastDecryptEncodingAttrs(std::move(encoding_attributes)); + + // Create a simple mock decryption result + std::vector plaintext_data(ciphertext.begin(), ciphertext.end()); + return std::make_unique(std::move(plaintext_data)); + } + + // Getter for the companion object + std::shared_ptr GetCompanion() const { return companion_; } + + private: + std::shared_ptr companion_; +}; + +// Test fixture for DBPALibraryWrapper tests +class DBPALibraryWrapperTest : public ::testing::Test { + protected: + void SetUp() override { + // Create test data + test_plaintext_ = "Hello, World!"; + test_ciphertext_.resize(test_plaintext_.size()); + + // Create shared destruction order tracker + destruction_order_tracker_ = std::make_shared(); + + // Create companion objects with shared order tracker + mock_companion_ = std::make_shared(destruction_order_tracker_); + handle_companion_ = std::make_shared(destruction_order_tracker_); + + // Create mock agent + mock_agent_ = std::make_unique(mock_companion_); + mock_agent_ptr_ = mock_agent_.get(); + } + + void TearDown() override { + // mock_companion_ and handle_companion_ remain valid for assertions even after mock_agent_ is destroyed + mock_agent_.reset(); + } + + // Helper method to create a wrapper with mock agent and handle management tracking + std::unique_ptr CreateWrapper() { + return CreateWrapperWithAgent(std::move(mock_agent_)); + } + + // Helper method to create a wrapper with custom agent and handle management tracking + std::unique_ptr CreateWrapperWithAgent( + std::unique_ptr agent) { + void* dummy_handle = reinterpret_cast(0x12345678); + + // Use the existing handle companion from the test fixture + return std::make_unique( + std::move(agent), dummy_handle, handle_companion_->CreateHandleClosingFunction()); + } + + // Helper method to create wrapper with custom handle closing function + std::unique_ptr CreateWrapperWithCustomClosing( + std::function handle_closing_fn) { + void* dummy_handle = reinterpret_cast(0x12345678); + + return std::make_unique( + std::move(mock_agent_), dummy_handle, handle_closing_fn); + } + + std::string test_plaintext_; + std::vector test_ciphertext_; + std::shared_ptr destruction_order_tracker_; + std::shared_ptr mock_companion_; + std::shared_ptr handle_companion_; + std::unique_ptr mock_agent_; + MockDataBatchProtectionAgent* mock_agent_ptr_; +}; + +// ============================================================================ +// CONSTRUCTOR TESTS +// ============================================================================ + +TEST_F(DBPALibraryWrapperTest, ConstructorValidParameters) { + auto mock_agent = std::make_unique(); + void* dummy_handle = reinterpret_cast(0x12345678); + + EXPECT_NO_THROW({ + DBPALibraryWrapper wrapper(std::move(mock_agent), dummy_handle, handle_companion_->CreateHandleClosingFunction()); + }); +} + +TEST_F(DBPALibraryWrapperTest, ConstructorValidParametersWithDefaultClosing) { + auto mock_agent = std::make_unique(); + void* dummy_handle = reinterpret_cast(0x12345678); + + EXPECT_NO_THROW({ + DBPALibraryWrapper wrapper(std::move(mock_agent), dummy_handle, handle_companion_->CreateHandleClosingFunction()); + }); +} + +TEST_F(DBPALibraryWrapperTest, ConstructorNullAgent) { + void* dummy_handle = reinterpret_cast(0x12345678); + + // Test with custom function + EXPECT_THROW({ + DBPALibraryWrapper wrapper(nullptr, dummy_handle, handle_companion_->CreateHandleClosingFunction()); + }, std::invalid_argument); +} + +TEST_F(DBPALibraryWrapperTest, ConstructorNullLibraryHandle) { + auto mock_agent = std::make_unique(); + void* dummy_handle = reinterpret_cast(0x12345678); + + // Test with custom function + EXPECT_THROW({ + DBPALibraryWrapper wrapper(std::move(mock_agent), dummy_handle, nullptr); + }, std::invalid_argument); +} + +// ============================================================================ +// HANDLE CLOSING FUNCTION TESTS +// ============================================================================ + +TEST_F(DBPALibraryWrapperTest, HandleClosingFunctionCalled) { + void* dummy_handle = reinterpret_cast(0x12345678); + + // Create wrapper in a scope to trigger destructor + { + auto wrapper = CreateWrapper(); + + // Verify handle closing hasn't been called yet + EXPECT_FALSE(handle_companion_->WasHandleCloseCalled()); + EXPECT_EQ(handle_companion_->GetHandleCloseCount(), 0); + } + + // After wrapper destruction, handle closing should have been called + EXPECT_TRUE(handle_companion_->WasHandleCloseCalled()); + EXPECT_EQ(handle_companion_->GetHandleCloseCount(), 1); + EXPECT_EQ(handle_companion_->GetLastClosedHandle(), dummy_handle); +} + +TEST_F(DBPALibraryWrapperTest, CustomHandleClosingFunction) { + bool custom_function_called = false; + void* custom_last_handle = nullptr; + + auto custom_closing_fn = [&custom_function_called, &custom_last_handle](void* handle) { + custom_function_called = true; + custom_last_handle = handle; + }; + + void* dummy_handle = reinterpret_cast(0x87654321); + + // Create wrapper with custom closing function + { + auto mock_agent = std::make_unique(); + DBPALibraryWrapper wrapper(std::move(mock_agent), dummy_handle, custom_closing_fn); + } + + // Verify custom function was called + EXPECT_TRUE(custom_function_called); + EXPECT_EQ(custom_last_handle, dummy_handle); + + // Verify our handle companion wasn't called + EXPECT_FALSE(handle_companion_->WasHandleCloseCalled()); + EXPECT_EQ(handle_companion_->GetHandleCloseCount(), 0); +} + +// ============================================================================ +// DELEGATION FUNCTIONALITY TESTS +// ============================================================================ + +TEST_F(DBPALibraryWrapperTest, InitDelegation) { + auto wrapper = CreateWrapper(); + + // Test data for init parameters + std::string column_name = "test_column"; + std::map connection_config = { + {"host", "localhost"}, + {"port", "5432"}, + {"database", "testdb"} + }; + std::string app_context = "test_app_context"; + std::string column_key_id = "test_key_id"; + Type::type data_type = Type::INT32; + CompressionCodec::type compression_type = CompressionCodec::SNAPPY; + std::optional> column_encryption_metadata = std::map{{"metaKey", "metaValue"}}; + + // Call init through wrapper + wrapper->init(column_name, connection_config, app_context, column_key_id, data_type, std::nullopt, compression_type, column_encryption_metadata); + + // Verify the mock agent was called + EXPECT_TRUE(mock_companion_->WasInitCalled()); + EXPECT_EQ(mock_companion_->GetInitCount(), 1); + + // Verify the correct parameters were passed to the mock + EXPECT_EQ(mock_companion_->GetInitColumnName(), column_name); + EXPECT_EQ(mock_companion_->GetInitConnectionConfig(), connection_config); + EXPECT_EQ(mock_companion_->GetInitAppContext(), app_context); + EXPECT_EQ(mock_companion_->GetInitColumnKeyId(), column_key_id); + EXPECT_EQ(mock_companion_->GetInitDataType(), data_type); + EXPECT_EQ(mock_companion_->GetInitCompressionType(), compression_type); + ASSERT_TRUE(mock_companion_->GetInitColumnEncryptionMetadata().has_value()); + EXPECT_EQ(mock_companion_->GetInitColumnEncryptionMetadata().value().at("metaKey"), "metaValue"); +} + +TEST_F(DBPALibraryWrapperTest, InitDelegationWithDatatypeLength) { + auto wrapper = CreateWrapper(); + + std::string column_name = "fixed_len_col"; + std::map connection_config = {}; + std::string app_context = "ctx"; + std::string column_key_id = "kid"; + Type::type data_type = Type::FIXED_LEN_BYTE_ARRAY; + CompressionCodec::type compression_type = CompressionCodec::UNCOMPRESSED; + + wrapper->init(column_name, connection_config, app_context, column_key_id, data_type, 16, compression_type, std::nullopt); + + EXPECT_TRUE(mock_companion_->WasInitCalled()); + ASSERT_TRUE(mock_companion_->GetInitDatatypeLength().has_value()); + EXPECT_EQ(mock_companion_->GetInitDatatypeLength().value(), 16); + EXPECT_FALSE(mock_companion_->GetInitColumnEncryptionMetadata().has_value()); +} + +TEST_F(DBPALibraryWrapperTest, InitDelegationWithEmptyParameters) { + auto wrapper = CreateWrapper(); + + // Test init with empty parameters + std::string empty_column_name = ""; + std::map empty_connection_config = {}; + std::string empty_app_context = ""; + std::string empty_column_key_id = ""; + Type::type data_type = Type::BYTE_ARRAY; + CompressionCodec::type compression_type = CompressionCodec::UNCOMPRESSED; + + // Call init through wrapper + wrapper->init(empty_column_name, empty_connection_config, empty_app_context, empty_column_key_id, data_type, std::nullopt, compression_type, std::nullopt); + + // Verify the mock agent was called + EXPECT_TRUE(mock_companion_->WasInitCalled()); + EXPECT_EQ(mock_companion_->GetInitCount(), 1); + + // Verify the empty parameters were passed correctly + EXPECT_EQ(mock_companion_->GetInitColumnName(), empty_column_name); + EXPECT_EQ(mock_companion_->GetInitConnectionConfig(), empty_connection_config); + EXPECT_EQ(mock_companion_->GetInitAppContext(), empty_app_context); + EXPECT_EQ(mock_companion_->GetInitColumnKeyId(), empty_column_key_id); + EXPECT_EQ(mock_companion_->GetInitDataType(), data_type); + EXPECT_EQ(mock_companion_->GetInitCompressionType(), compression_type); + EXPECT_FALSE(mock_companion_->GetInitColumnEncryptionMetadata().has_value()); +} + +TEST_F(DBPALibraryWrapperTest, EncryptDelegation) { + auto wrapper = CreateWrapper(); + + // Convert test data to spans + span plaintext_span( + reinterpret_cast(test_plaintext_.data()), + test_plaintext_.size()); + span ciphertext_span(test_ciphertext_.data(), test_ciphertext_.size()); + + // Call encrypt through wrapper + auto result = wrapper->Encrypt(plaintext_span, {}); + + // Verify the mock agent was called + EXPECT_TRUE(mock_companion_->WasEncryptCalled()); + EXPECT_EQ(mock_companion_->GetEncryptCount(), 1); + + // Verify the correct plaintext was passed to the mock + auto mock_plaintext = mock_companion_->GetEncryptPlaintext(); + std::string mock_plaintext_str(mock_plaintext.begin(), mock_plaintext.end()); + EXPECT_EQ(mock_plaintext_str, test_plaintext_); + + // Verify the correct ciphertext size was passed + EXPECT_EQ(mock_companion_->GetEncryptCiphertextSize(), test_ciphertext_.size()); + + // Verify result is not null + EXPECT_NE(result, nullptr); +} + +TEST_F(DBPALibraryWrapperTest, EncryptDecryptEncodingAttributesDelegation) { + auto wrapper = CreateWrapper(); + + // Encrypt + std::map expected_result_metadata = {{"encryption_algorithm_version", "1"}, + {"test_kid", "kid_123"}}; + mock_companion_->SetNextEncryptResultMetadata(expected_result_metadata); + std::map enc_attrs = {{"format", "plain"}, {"scale", "0"}}; + span plaintext_span( + reinterpret_cast(test_plaintext_.data()), + test_plaintext_.size()); + + auto enc_result = wrapper->Encrypt(plaintext_span, enc_attrs); + EXPECT_NE(enc_result, nullptr); + EXPECT_EQ(mock_companion_->GetLastEncryptEncodingAttrs(), enc_attrs); + ASSERT_TRUE(enc_result->encryption_metadata().has_value()); + EXPECT_EQ(enc_result->encryption_metadata().value(), expected_result_metadata); + + // Decrypt + auto ct_span = enc_result->ciphertext(); + std::map dec_attrs = {{"format", "plain"}}; + auto dec_result = wrapper->Decrypt(ct_span, dec_attrs); + EXPECT_NE(dec_result, nullptr); + EXPECT_EQ(mock_companion_->GetLastDecryptEncodingAttrs(), dec_attrs); +} + +TEST_F(DBPALibraryWrapperTest, DecryptDelegation) { + auto wrapper = CreateWrapper(); + + // Convert test data to spans + span ciphertext_span( + reinterpret_cast(test_plaintext_.data()), + test_plaintext_.size()); + + // Call decrypt through wrapper + auto result = wrapper->Decrypt(ciphertext_span, {}); + + // Verify the mock agent was called + EXPECT_TRUE(mock_companion_->WasDecryptCalled()); + EXPECT_EQ(mock_companion_->GetDecryptCount(), 1); + + // Verify the correct ciphertext was passed to the mock + auto mock_ciphertext = mock_companion_->GetDecryptCiphertext(); + std::string mock_ciphertext_str(mock_ciphertext.begin(), mock_ciphertext.end()); + EXPECT_EQ(mock_ciphertext_str, test_plaintext_); + + // Verify result is not null + EXPECT_NE(result, nullptr); +} + +TEST_F(DBPALibraryWrapperTest, MultipleEncryptDelegations) { + auto wrapper = CreateWrapper(); + + // Perform multiple encrypt operations + for (int i = 0; i < 5; ++i) { + std::string plaintext = "Test " + std::to_string(i); + std::vector ciphertext(plaintext.size()); + + span plaintext_span( + reinterpret_cast(plaintext.data()), + plaintext.size()); + + auto result = wrapper->Encrypt(plaintext_span, {}); + EXPECT_NE(result, nullptr); + } + + // Verify the mock agent was called the correct number of times + EXPECT_TRUE(mock_companion_->WasEncryptCalled()); + EXPECT_EQ(mock_companion_->GetEncryptCount(), 5); +} + +TEST_F(DBPALibraryWrapperTest, MultipleDecryptDelegations) { + auto wrapper = CreateWrapper(); + + // Perform multiple decrypt operations + for (int i = 0; i < 3; ++i) { + std::string ciphertext = "Test " + std::to_string(i); + + span ciphertext_span( + reinterpret_cast(ciphertext.data()), + ciphertext.size()); + + auto result = wrapper->Decrypt(ciphertext_span, {}); + EXPECT_NE(result, nullptr); + } + + // Verify the mock agent was called the correct number of times + EXPECT_TRUE(mock_companion_->WasDecryptCalled()); + EXPECT_EQ(mock_companion_->GetDecryptCount(), 3); +} + +TEST_F(DBPALibraryWrapperTest, MixedOperationsDelegation) { + auto wrapper = CreateWrapper(); + + // Perform mixed encrypt and decrypt operations + std::vector test_data = {"Hello", "World", "Test", "Data"}; + auto call_count = static_cast(test_data.size()); + + for (const auto& data : test_data) { + // Encrypt + span plaintext_span( + reinterpret_cast(data.data()), + data.size()); + + auto encrypt_result = wrapper->Encrypt(plaintext_span, {}); + EXPECT_NE(encrypt_result, nullptr); + + // Decrypt using the ciphertext from the encryption result + auto ciphertext_span = encrypt_result->ciphertext(); + auto decrypt_result = wrapper->Decrypt(ciphertext_span, {}); + EXPECT_NE(decrypt_result, nullptr); + } + + // Verify both operations were called + EXPECT_TRUE(mock_companion_->WasEncryptCalled()); + EXPECT_TRUE(mock_companion_->WasDecryptCalled()); + EXPECT_EQ(mock_companion_->GetEncryptCount(), call_count); + EXPECT_EQ(mock_companion_->GetDecryptCount(), call_count); +} + +TEST_F(DBPALibraryWrapperTest, InitWithEncryptDecryptOperations) { + auto wrapper = CreateWrapper(); + + // First, initialize the wrapper + std::string column_name = "test_column"; + std::map connection_config = { + {"host", "localhost"}, + {"port", "5432"} + }; + std::string app_context = "test_app"; + std::string column_key_id = "test_key"; + Type::type data_type = Type::INT32; + CompressionCodec::type compression_type = CompressionCodec::SNAPPY; + + wrapper->init(column_name, connection_config, app_context, column_key_id, data_type, std::nullopt, compression_type, std::nullopt); + + // Verify init was called + EXPECT_TRUE(mock_companion_->WasInitCalled()); + EXPECT_EQ(mock_companion_->GetInitCount(), 1); + + // Then perform encrypt/decrypt operations + std::string test_data = "Test data after init"; + span plaintext_span( + reinterpret_cast(test_data.data()), + test_data.size()); + + auto encrypt_result = wrapper->Encrypt(plaintext_span, {}); + EXPECT_NE(encrypt_result, nullptr); + + auto ciphertext_span = encrypt_result->ciphertext(); + auto decrypt_result = wrapper->Decrypt(ciphertext_span, {}); + EXPECT_NE(decrypt_result, nullptr); + + // Verify all operations were called + EXPECT_TRUE(mock_companion_->WasEncryptCalled()); + EXPECT_TRUE(mock_companion_->WasDecryptCalled()); + EXPECT_EQ(mock_companion_->GetEncryptCount(), 1); + EXPECT_EQ(mock_companion_->GetDecryptCount(), 1); + + // Verify init parameters are still accessible + EXPECT_EQ(mock_companion_->GetInitColumnName(), column_name); + EXPECT_EQ(mock_companion_->GetInitConnectionConfig(), connection_config); + EXPECT_EQ(mock_companion_->GetInitAppContext(), app_context); + EXPECT_EQ(mock_companion_->GetInitColumnKeyId(), column_key_id); + EXPECT_EQ(mock_companion_->GetInitDataType(), data_type); + EXPECT_EQ(mock_companion_->GetInitCompressionType(), compression_type); + EXPECT_FALSE(mock_companion_->GetInitColumnEncryptionMetadata().has_value()); +} + +TEST_F(DBPALibraryWrapperTest, DelegationWithEmptyData) { + auto wrapper = CreateWrapper(); + + // Test encryption with empty data + std::vector empty_plaintext; + + span plaintext_span(empty_plaintext); + + auto encrypt_result = wrapper->Encrypt(plaintext_span, {}); + EXPECT_NE(encrypt_result, nullptr); + EXPECT_TRUE(mock_companion_->WasEncryptCalled()); + + // Test decryption with empty data + auto ciphertext_span = encrypt_result->ciphertext(); + auto decrypt_result = wrapper->Decrypt(ciphertext_span, {}); + EXPECT_NE(decrypt_result, nullptr); + EXPECT_TRUE(mock_companion_->WasDecryptCalled()); +} + +TEST_F(DBPALibraryWrapperTest, DelegationWithNullData) { + auto wrapper = CreateWrapper(); + + // Test encryption with null data pointers but valid spans + // This tests that the wrapper properly delegates even with null data + span null_plaintext_span(nullptr, size_t{0}); + + auto encrypt_result = wrapper->Encrypt(null_plaintext_span, {}); + EXPECT_NE(encrypt_result, nullptr); + EXPECT_TRUE(mock_companion_->WasEncryptCalled()); + + // Test decryption with null data pointer but valid span + span null_decrypt_span(nullptr, size_t{0}); + auto decrypt_result = wrapper->Decrypt(null_decrypt_span, {}); + EXPECT_NE(decrypt_result, nullptr); + EXPECT_TRUE(mock_companion_->WasDecryptCalled()); + + // Verify the mock agent received the correct data (empty vectors) + auto mock_plaintext = mock_companion_->GetEncryptPlaintext(); + auto mock_ciphertext = mock_companion_->GetDecryptCiphertext(); + EXPECT_EQ(mock_plaintext.size(), 0); + EXPECT_EQ(mock_ciphertext.size(), 0); +} + +// ============================================================================ +// DESTRUCTOR FUNCTIONALITY TESTS +// ============================================================================ + +TEST_F(DBPALibraryWrapperTest, DestructorBasicBehavior) { + void* dummy_handle = reinterpret_cast(0x12345678); + + // Create wrapper in a scope to test destructor + { + auto wrapper = CreateWrapper(); + + // Perform some operations to ensure the wrapper is used + std::vector plaintext = {1, 2, 3, 4, 5}; + + span plaintext_span(plaintext.data(), plaintext.size()); + + auto result = wrapper->Encrypt(plaintext_span, {}); + EXPECT_NE(result, nullptr); + + // Verify handle closing hasn't been called yet + EXPECT_FALSE(handle_companion_->WasHandleCloseCalled()); + EXPECT_EQ(handle_companion_->GetHandleCloseCount(), 0); + } + + // At this point, the wrapper should have been destroyed and handle closed + EXPECT_TRUE(handle_companion_->WasHandleCloseCalled()); + EXPECT_EQ(handle_companion_->GetHandleCloseCount(), 1); + EXPECT_EQ(handle_companion_->GetLastClosedHandle(), dummy_handle); +} + +TEST_F(DBPALibraryWrapperTest, DestructorWithMultipleOperations) { + void* dummy_handle = reinterpret_cast(0x12345678); + + // Create wrapper in a scope to test destructor + { + auto wrapper = CreateWrapper(); + + // Perform multiple operations + for (int i = 0; i < 10; ++i) { + std::string plaintext = "Test " + std::to_string(i); + std::vector ciphertext(plaintext.size()); + + span plaintext_span( + reinterpret_cast(plaintext.data()), + plaintext.size()); + + auto encrypt_result = wrapper->Encrypt(plaintext_span, {}); + EXPECT_NE(encrypt_result, nullptr); + + auto ciphertext_span = encrypt_result->ciphertext(); + auto decrypt_result = wrapper->Decrypt(ciphertext_span, {}); + EXPECT_NE(decrypt_result, nullptr); + } + + // Verify operations completed but handle not closed yet + EXPECT_FALSE(handle_companion_->WasHandleCloseCalled()); + } + + // Verify the wrapper was destroyed properly and handle was closed + EXPECT_TRUE(handle_companion_->WasHandleCloseCalled()); + EXPECT_EQ(handle_companion_->GetHandleCloseCount(), 1); + EXPECT_EQ(handle_companion_->GetLastClosedHandle(), dummy_handle); +} + +TEST_F(DBPALibraryWrapperTest, DestructorOrderVerification) { + // Clear any previous events from the shared order tracker + destruction_order_tracker_->Clear(); + + // Create a custom mock agent that tracks destruction order + auto custom_companion = std::make_shared(destruction_order_tracker_); + auto custom_agent = std::make_unique(custom_companion); + + void* dummy_handle = reinterpret_cast(0x12345678); + + // Create wrapper in a scope + { + auto wrapper = CreateWrapperWithAgent(std::move(custom_agent)); + + // Perform some operations + std::vector plaintext = {1, 2, 3}; + + span plaintext_span(plaintext.data(), plaintext.size()); + + auto result = wrapper->Encrypt(plaintext_span, {}); + EXPECT_NE(result, nullptr); + + // Verify neither destructor nor handle closing has been called yet + EXPECT_FALSE(custom_companion->WasDestructorCalled()); + EXPECT_FALSE(handle_companion_->WasHandleCloseCalled()); + EXPECT_FALSE(destruction_order_tracker_->WasEventRecorded("handle_close")); + EXPECT_FALSE(destruction_order_tracker_->WasEventRecorded("agent_destructor")); + } + + // Verify both the custom agent was destroyed and handle was closed + EXPECT_TRUE(custom_companion->WasDestructorCalled()); + EXPECT_TRUE(handle_companion_->WasHandleCloseCalled()); + EXPECT_EQ(handle_companion_->GetHandleCloseCount(), 1); + EXPECT_EQ(handle_companion_->GetLastClosedHandle(), dummy_handle); + + // Verify the order of destruction: handle_close should be called BEFORE agent_destructor + EXPECT_TRUE(destruction_order_tracker_->WasEventRecorded("agent_destructor")); + EXPECT_TRUE(destruction_order_tracker_->WasEventRecorded("handle_close")); + EXPECT_TRUE(destruction_order_tracker_->VerifyOrder("agent_destructor", "handle_close")); +} + +TEST_F(DBPALibraryWrapperTest, DestructionOrderTrackerFunctionality) { + // Test the destruction order tracker functionality independently + auto tracker = std::make_shared(); + + // Record events in a specific order + tracker->RecordEvent("first"); + tracker->RecordEvent("second"); + tracker->RecordEvent("third"); + + // Verify order tracking + EXPECT_TRUE(tracker->VerifyOrder("first", "second")); + EXPECT_TRUE(tracker->VerifyOrder("second", "third")); + EXPECT_TRUE(tracker->VerifyOrder("first", "third")); + + // Verify reverse order is false + EXPECT_FALSE(tracker->VerifyOrder("second", "first")); + EXPECT_FALSE(tracker->VerifyOrder("third", "second")); + EXPECT_FALSE(tracker->VerifyOrder("third", "first")); + + // Verify sequence numbers + EXPECT_EQ(tracker->GetEventSequence("first"), 1); + EXPECT_EQ(tracker->GetEventSequence("second"), 2); + EXPECT_EQ(tracker->GetEventSequence("third"), 3); + + // Verify event recording + EXPECT_TRUE(tracker->WasEventRecorded("first")); + EXPECT_TRUE(tracker->WasEventRecorded("second")); + EXPECT_TRUE(tracker->WasEventRecorded("third")); + EXPECT_FALSE(tracker->WasEventRecorded("nonexistent")); +} + +// ============================================================================ +// INTERFACE COMPLIANCE TESTS +// ============================================================================ + +TEST_F(DBPALibraryWrapperTest, InterfaceCompliancePolymorphic) { + auto wrapper = CreateWrapper(); + + // Verify the wrapper can be used polymorphically + DataBatchProtectionAgentInterface* interface_ptr = wrapper.get(); + EXPECT_NE(interface_ptr, nullptr); + + // Test polymorphic init call + std::string column_name = "polymorphic_column"; + std::map connection_config = {{"test", "value"}}; + std::string app_context = "polymorphic_context"; + std::string column_key_id = "polymorphic_key"; + Type::type data_type = Type::INT64; + CompressionCodec::type compression_type = CompressionCodec::GZIP; + + interface_ptr->init(column_name, connection_config, app_context, column_key_id, data_type, std::nullopt, compression_type, std::nullopt); + + // Verify init was called through the interface + EXPECT_TRUE(mock_companion_->WasInitCalled()); + EXPECT_EQ(mock_companion_->GetInitCount(), 1); + EXPECT_EQ(mock_companion_->GetInitColumnName(), column_name); + EXPECT_EQ(mock_companion_->GetInitDataType(), data_type); + EXPECT_EQ(mock_companion_->GetInitCompressionType(), compression_type); + EXPECT_FALSE(mock_companion_->GetInitColumnEncryptionMetadata().has_value()); + + // Test polymorphic encrypt/decrypt calls + std::vector plaintext = {1, 2, 3}; + + span plaintext_span(plaintext.data(), plaintext.size()); + + auto encrypt_result = interface_ptr->Encrypt(plaintext_span, {}); + EXPECT_NE(encrypt_result, nullptr); + + auto ciphertext_span = encrypt_result->ciphertext(); + auto decrypt_result = interface_ptr->Decrypt(ciphertext_span, {}); + EXPECT_NE(decrypt_result, nullptr); + + // Verify the mock agent was called through the interface + EXPECT_TRUE(mock_companion_->WasEncryptCalled()); + EXPECT_TRUE(mock_companion_->WasDecryptCalled()); +} + +// ============================================================================ +// EDGE CASE TESTS +// ============================================================================ + +TEST_F(DBPALibraryWrapperTest, EdgeCaseZeroSizeSpans) { + auto wrapper = CreateWrapper(); + + // Test with zero-size spans + std::vector empty_data; + + span empty_plaintext_span(empty_data); + + auto encrypt_result = wrapper->Encrypt(empty_plaintext_span, {}); + EXPECT_NE(encrypt_result, nullptr); + + auto ciphertext_span = encrypt_result->ciphertext(); + auto decrypt_result = wrapper->Decrypt(ciphertext_span, {}); + EXPECT_NE(decrypt_result, nullptr); + + EXPECT_TRUE(mock_companion_->WasEncryptCalled()); + EXPECT_TRUE(mock_companion_->WasDecryptCalled()); +} + +TEST_F(DBPALibraryWrapperTest, EdgeCaseSingleByteData) { + auto wrapper = CreateWrapper(); + + // Test with single byte data + std::vector single_byte = {0x42}; + + span plaintext_span(single_byte.data(), single_byte.size()); + + auto encrypt_result = wrapper->Encrypt(plaintext_span, {}); + EXPECT_NE(encrypt_result, nullptr); + + auto ciphertext_span = encrypt_result->ciphertext(); + auto decrypt_result = wrapper->Decrypt(ciphertext_span, {}); + EXPECT_NE(decrypt_result, nullptr); + + EXPECT_TRUE(mock_companion_->WasEncryptCalled()); + EXPECT_TRUE(mock_companion_->WasDecryptCalled()); +} + +} // namespace parquet::encryption::external::test \ No newline at end of file diff --git a/cpp/src/parquet/encryption/external/dbpa_test_agent.cc b/cpp/src/parquet/encryption/external/dbpa_test_agent.cc new file mode 100644 index 000000000000..a14f8ab198a2 --- /dev/null +++ b/cpp/src/parquet/encryption/external/dbpa_test_agent.cc @@ -0,0 +1,151 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include +#include + +#include "parquet/exception.h" +#include "parquet/encryption/external/dbpa_test_agent.h" +#include + +template +using span = tcb::span; +using dbps::external::EncryptionResult; +using dbps::external::DecryptionResult; + +namespace parquet::encryption::external { + +// Concrete implementation of EncryptionResult for testing +class TestEncryptionResult : public EncryptionResult { +public: + TestEncryptionResult(std::vector data, bool success = true, + std::string error_msg = "", + std::map error_fields = {}, + std::optional> metadata = std::nullopt) + : ciphertext_data_(std::move(data)), success_(success), + error_message_(std::move(error_msg)), error_fields_(std::move(error_fields)), + metadata_(std::move(metadata)) {} + + span ciphertext() const override { + return span(ciphertext_data_.data(), ciphertext_data_.size()); + } + + std::size_t size() const override { return ciphertext_data_.size(); } + bool success() const override { return success_; } + const std::string& error_message() const override { return error_message_; } + const std::map& error_fields() const override { return error_fields_; } + const std::optional> encryption_metadata() const override { + if (metadata_.has_value()) { + return metadata_; + } + return std::map{{"test_key1", "test_value1"}, {"test_key2", "test_value2"}}; + } + +private: + std::vector ciphertext_data_; + bool success_; + std::string error_message_; + std::map error_fields_; + std::optional> metadata_; +}; + +// Concrete implementation of DecryptionResult for testing +class TestDecryptionResult : public DecryptionResult { +public: + TestDecryptionResult(std::vector data, bool success = true, + std::string error_msg = "", + std::map error_fields = {}) + : plaintext_data_(std::move(data)), success_(success), + error_message_(std::move(error_msg)), error_fields_(std::move(error_fields)) {} + + span plaintext() const override { + return span(plaintext_data_.data(), plaintext_data_.size()); + } + + std::size_t size() const override { return plaintext_data_.size(); } + bool success() const override { return success_; } + const std::string& error_message() const override { return error_message_; } + const std::map& error_fields() const override { return error_fields_; } + +private: + std::vector plaintext_data_; + bool success_; + std::string error_message_; + std::map error_fields_; +}; + +DBPATestAgent::DBPATestAgent() { +} + +std::unique_ptr DBPATestAgent::Encrypt( + span plaintext, + std::map) { + + // Simple XOR encryption for testing purposes + // In a real implementation, this would use proper encryption + std::vector ciphertext_data(plaintext.size()); + + const size_t key_len = key_.size(); + for (size_t i = 0; i < plaintext.size(); ++i) { + ciphertext_data[i] = plaintext[i] ^ static_cast(key_[i % key_len]); + } + + // For tests, optionally force a conflicting metadata value on subsequent calls + auto it = connection_config_.find("dbpa_test_force_conflicting_metadata"); + bool force_conflict = (it != connection_config_.end() && it->second == "1"); + encrypt_calls_++; + if (force_conflict && encrypt_calls_ >= 2) { + // Return a different value for test_key1 to trigger conflict in writer + std::map md {{"test_key1", "test_value1_conflict"}, {"test_key2", "test_value2"}}; + return std::make_unique(std::move(ciphertext_data), true, "", std::map{}, md); + } + + return std::make_unique(std::move(ciphertext_data)); +} + +std::unique_ptr DBPATestAgent::Decrypt( + span ciphertext, + std::map) { + + // Simple XOR decryption for testing purposes + // In a real implementation, this would perform actual decryption + std::vector plaintext_data(ciphertext.size()); + + const size_t key_len = key_.size(); + for (size_t i = 0; i < ciphertext.size(); ++i) { + plaintext_data[i] = ciphertext[i] ^ static_cast(key_[i % key_len]); + } + + return std::make_unique(std::move(plaintext_data)); +} + +DBPATestAgent::~DBPATestAgent() { +} + +// Export function for creating new instances from shared library +extern "C" { + DataBatchProtectionAgentInterface* create_new_instance() { + return new parquet::encryption::external::DBPATestAgent(); + } +} + +} // namespace parquet::encryption::external diff --git a/cpp/src/parquet/encryption/external/dbpa_test_agent.h b/cpp/src/parquet/encryption/external/dbpa_test_agent.h new file mode 100644 index 000000000000..da8af599ff9c --- /dev/null +++ b/cpp/src/parquet/encryption/external/dbpa_test_agent.h @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include + +template +using span = tcb::span; + +using dbps::external::DataBatchProtectionAgentInterface; +using dbps::external::EncryptionResult; +using dbps::external::DecryptionResult; +using dbps::external::Type; +using dbps::external::CompressionCodec; + +namespace parquet::encryption::external { + +// Implementation of the DataBatchProtectionAgentInterface for testing purposes. +// It is used to test library wrapper/loading code. +// Will never be used in production. +class DBPATestAgent : public DataBatchProtectionAgentInterface { + public: + explicit DBPATestAgent(); + + void init( + std::string column_name, + std::map connection_config, + std::string app_context, + std::string column_key_id, + Type::type data_type, + std::optional datatype_length, + CompressionCodec::type compression_type, + std::optional> column_encryption_metadata) override { + + if (column_key_id.empty()) { + throw std::invalid_argument("column_key_id cannot be empty"); + } + // Store the key id so we can use it for simple test XOR encryption/decryption + key_ = std::move(column_key_id); + connection_config_ = std::move(connection_config); + } + + std::unique_ptr Encrypt( + span plaintext, + std::map encoding_attributes) override; + + std::unique_ptr Decrypt( + span ciphertext, + std::map encoding_attributes) override; + + ~DBPATestAgent(); + + private: + // Used as a simple XOR key for test encryption/decryption + std::string key_; + // Stored connection configuration from init(); used to toggle test behaviors + std::map connection_config_; + // Count Encrypt() calls to allow staged behavior in tests + size_t encrypt_calls_ = 0; +}; + +} // namespace parquet::encryption::external diff --git a/cpp/src/parquet/encryption/external/loadable_encryptor_utils.cc b/cpp/src/parquet/encryption/external/loadable_encryptor_utils.cc new file mode 100644 index 000000000000..cf743c78c1ea --- /dev/null +++ b/cpp/src/parquet/encryption/external/loadable_encryptor_utils.cc @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "parquet/encryption/external/loadable_encryptor_utils.h" +#include +#include "parquet/encryption/external/dbpa_library_wrapper.h" + +#include "arrow/util/io_util.h" //utils for loading shared libraries +#include "arrow/result.h" +#include "arrow/util/logging.h" + +#include +#include +#include +#include + +using ::arrow::Result; + +namespace parquet::encryption::external { + +// Function pointer type for creating encryptor instances +// This needs to match the return type of the create_new_instance function in the shared library. +typedef DataBatchProtectionAgentInterface* (*create_encryptor_t)(); + +std::unique_ptr LoadableEncryptorUtils::CreateInstance(void* library_handle) { + auto symbol_result = arrow::internal::GetSymbol(library_handle, "create_new_instance"); + if (!symbol_result.ok()) { + ARROW_LOG(ERROR) << "Cannot load symbol 'create_new_instance()': " << symbol_result.status().message(); + auto status = arrow::internal::CloseDynamicLibrary(library_handle); + + throw std::runtime_error("Failed to load symbol 'create_new_instance()': " + symbol_result.status().message()); + } + + //create_instance_fn is a function pointer to the create_new_instance function in the shared library. + create_encryptor_t create_instance_fn = reinterpret_cast(symbol_result.ValueOrDie()); + + // at this point, we have the create_instance function pointer (from the shared library) + // so we can create a new instance of the DataBatchProtectionAgentInterface + DataBatchProtectionAgentInterface* instance = create_instance_fn(); + + if (instance == nullptr) { + ARROW_LOG(ERROR) << "Cannot create instance of DataBatchProtectionAgentInterface"; + auto status = arrow::internal::CloseDynamicLibrary(library_handle); + throw std::runtime_error("Failed to create instance of DataBatchProtectionAgentInterface"); + } + + auto instance_ptr = std::unique_ptr(instance); + + return instance_ptr; +} // CreateInstance() + +std::unique_ptr LoadableEncryptorUtils::LoadFromLibrary(const std::string& library_path) { + + if (library_path.empty()) { + throw std::invalid_argument("LoadableEncryptorUtils::LoadFromLibrary: No library path provided"); + } + + auto library_handle_result = arrow::internal::LoadDynamicLibrary(library_path.c_str());; + if (!library_handle_result.ok()) { + throw std::runtime_error("Failed to load library: " + library_handle_result.status().message()); + } + + void* library_handle = library_handle_result.ValueOrDie(); + auto agent_instance = CreateInstance(library_handle); + + //wrap the agent in a DBPALibraryWrapper + auto wrapped_agent = std::make_unique( + std::move(agent_instance), + library_handle); + + return wrapped_agent; +} + +} // namespace parquet::encryption::external diff --git a/cpp/src/parquet/encryption/external/loadable_encryptor_utils.h b/cpp/src/parquet/encryption/external/loadable_encryptor_utils.h new file mode 100644 index 000000000000..6556da61798c --- /dev/null +++ b/cpp/src/parquet/encryption/external/loadable_encryptor_utils.h @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "parquet/platform.h" +#include + +using dbps::external::DataBatchProtectionAgentInterface; + +namespace parquet::encryption::external { + +class PARQUET_EXPORT LoadableEncryptorUtils { + public: + //Will load the shared library and instantiate the DataBatchProtectionAgentInterface + // provided by the shared library. The instance will be wrapped in a DBPALibraryWrapper. + static std::unique_ptr LoadFromLibrary(const std::string& library_path); + + private: + static std::unique_ptr CreateInstance(void* library_handle); +}; + +} // namespace parquet::encryption::external diff --git a/cpp/src/parquet/encryption/external/loadable_encryptor_utils_test.cc b/cpp/src/parquet/encryption/external/loadable_encryptor_utils_test.cc new file mode 100644 index 000000000000..2035c2a131cd --- /dev/null +++ b/cpp/src/parquet/encryption/external/loadable_encryptor_utils_test.cc @@ -0,0 +1,125 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "parquet/encryption/external/loadable_encryptor_utils.h" +#include +#include "parquet/encryption/external/dbpa_library_wrapper.h" +#include "parquet/encryption/external/test_utils.h" + + +namespace parquet::encryption::external::test { + +// Test fixture for LoadableEncryptorUtils tests +class LoadableEncryptorUtilsTest : public ::testing::Test { + public: + std::string library_path_; + + protected: + void SetUp() override { + // Get the path to the DBPATestAgent shared library + // This assumes the library is built + library_path_ = TestUtils::GetTestLibraryPath(); + } +}; + +// ============================================================================ +// SUCCESS TESTS +// ============================================================================ + +TEST_F(LoadableEncryptorUtilsTest, LoadValidLibrary) { + // Test loading the library + std::unique_ptr agent; + + try { + agent = LoadableEncryptorUtils::LoadFromLibrary(library_path_); + ASSERT_NE(agent, nullptr) << "Agent should be successfully loaded"; + } catch (const std::runtime_error& e) { + // Library doesn't exist or failed to load - this is expected in some build environments + GTEST_SKIP() << "Library not available: " << e.what(); + } +} + +TEST_F(LoadableEncryptorUtilsTest, MultipleLoads) { + // Load multiple agents + std::unique_ptr agent1, agent2, agent3; + + try { + agent1 = LoadableEncryptorUtils::LoadFromLibrary(library_path_); + agent2 = LoadableEncryptorUtils::LoadFromLibrary(library_path_); + agent3 = LoadableEncryptorUtils::LoadFromLibrary(library_path_); + + ASSERT_NE(agent1, nullptr) << "First agent should be successfully loaded"; + ASSERT_NE(agent2, nullptr) << "Second agent should be successfully loaded"; + ASSERT_NE(agent3, nullptr) << "Third agent should be successfully loaded"; + + // Verify that all instances are different from each other + ASSERT_NE(agent1.get(), agent2.get()) << "First and second agents should be different instances"; + ASSERT_NE(agent1.get(), agent3.get()) << "First and third agents should be different instances"; + ASSERT_NE(agent2.get(), agent3.get()) << "Second and third agents should be different instances"; + + } catch (const std::runtime_error& e) { + // Library doesn't exist or failed to load - this is expected in some build environments + GTEST_SKIP() << "Library not available: " << e.what(); + } +} + +TEST_F(LoadableEncryptorUtilsTest, ReturnsDBPALibraryWrapper) { + // Test that LoadFromLibrary returns an instance of DBPALibraryWrapper + std::unique_ptr agent; + + try { + agent = LoadableEncryptorUtils::LoadFromLibrary(library_path_); + ASSERT_NE(agent, nullptr) << "Agent should be successfully loaded"; + + // Verify that the returned instance is of type DBPALibraryWrapper + DBPALibraryWrapper* wrapper = dynamic_cast(agent.get()); + EXPECT_NE(wrapper, nullptr) << "Returned instance should be of type DBPALibraryWrapper"; + } catch (const std::runtime_error& e) { + // Library doesn't exist or failed to load - this is expected in some build environments + GTEST_SKIP() << "Library not available: " << e.what(); + } +} + +// ============================================================================ +// ERROR HANDLING TESTS +// ============================================================================ + +TEST_F(LoadableEncryptorUtilsTest, EmptyLibraryPath) { + EXPECT_THROW({ + LoadableEncryptorUtils::LoadFromLibrary(""); + }, std::invalid_argument); +} + +TEST_F(LoadableEncryptorUtilsTest, NonexistentLibrary) { + EXPECT_THROW({ + LoadableEncryptorUtils::LoadFromLibrary("./nonexistent_library.so"); + }, std::runtime_error); +} + +TEST_F(LoadableEncryptorUtilsTest, InvalidLibraryPath) { + EXPECT_THROW({ + LoadableEncryptorUtils::LoadFromLibrary("/invalid/path/to/library.so"); + }, std::runtime_error); +} + +} // namespace parquet::encryption::external::test diff --git a/cpp/src/parquet/encryption/external/test_utils.cc b/cpp/src/parquet/encryption/external/test_utils.cc new file mode 100644 index 000000000000..0186f3474de7 --- /dev/null +++ b/cpp/src/parquet/encryption/external/test_utils.cc @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "parquet/encryption/external/test_utils.h" + +#include +#include +#include + +#ifdef __APPLE__ +#include +#elif defined(__linux__) +#include +#include +#elif defined(_WIN32) +#include +#endif + +namespace parquet::encryption::external::test { + +std::string TestUtils::GetExecutableDirectory() { +#ifdef __APPLE__ + char path[PATH_MAX]; + uint32_t size = sizeof(path); + if (_NSGetExecutablePath(path, &size) == 0) { + return std::filesystem::path(path).parent_path().string(); + } +#elif defined(__linux__) + char path[PATH_MAX]; + ssize_t len = readlink("/proc/self/exe", path, sizeof(path) - 1); + if (len != -1) { + path[len] = '\0'; + return std::filesystem::path(path).parent_path().string(); + } +#elif defined(_WIN32) + char path[MAX_PATH]; + if (GetModuleFileNameA(NULL, path, MAX_PATH) != 0) { + return std::filesystem::path(path).parent_path().string(); + } +#endif + // Fallback to current working directory if we can't determine executable path + return std::filesystem::current_path().string(); +} + +std::string TestUtils::GetTestLibraryPath() { + // Check for environment variable to override the executable directory + const char* cwd_override = std::getenv("PARQUET_TEST_LIBRARY_CWD"); + std::string base_path; + + if (cwd_override && cwd_override[0]) { + base_path = std::string(cwd_override); + } else { + // Get the directory where the executable is located + base_path = GetExecutableDirectory(); + } + + std::vector possible_filenames = { + "libDBPATestAgent.so", + "libDBPATestAgent.dylib", + "DBPATestAgent.dll" + }; + + std::vector possible_directories = { + GetExecutableDirectory() + "/", + base_path + "/", + "./", + "" + }; + + for (const auto& filename : possible_filenames) { + for (const auto& directory : possible_directories) { + std::string path = directory + filename; + if (std::filesystem::exists(path)) { + return path; + } + } + } + + throw std::runtime_error("Could not find library"); +} + +} // namespace parquet::encryption::external::test diff --git a/cpp/src/parquet/encryption/external/test_utils.h b/cpp/src/parquet/encryption/external/test_utils.h new file mode 100644 index 000000000000..381760afad04 --- /dev/null +++ b/cpp/src/parquet/encryption/external/test_utils.h @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +namespace parquet::encryption::external::test { + +/** + * Utility class for test-related helper functions. + */ +class TestUtils { + public: + /** + * Get the directory where the executable is located. + * Used to determine the path to test libraries (*.so, *.dylib, *.dll). + * + * @return The directory path where the executable is located + */ + static std::string GetExecutableDirectory(); + + /** + * Get the path to the test library (DBPATestAgent). + * Searches for the library in various possible locations and filenames. + * + * @return The full path to the test library + * @throws std::runtime_error if the library cannot be found + */ + static std::string GetTestLibraryPath(); +}; + +} // namespace parquet::encryption::external::test diff --git a/cpp/src/parquet/encryption/external_dbpa_encryption.cc b/cpp/src/parquet/encryption/external_dbpa_encryption.cc new file mode 100644 index 000000000000..bf35d649efd4 --- /dev/null +++ b/cpp/src/parquet/encryption/external_dbpa_encryption.cc @@ -0,0 +1,669 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/util/key_value_metadata.h" +#include "arrow/util/logging.h" +#include "parquet/encryption/external_dbpa_encryption.h" +#include "parquet/encryption/key_metadata.h" +#include "parquet/encryption/encryption_utils.h" +#include "parquet/encryption/external/loadable_encryptor_utils.h" +#include "parquet/encryption/external/dbpa_enum_utils.h" +#include "parquet/encryption/external/dbpa_executor.h" +#include "parquet/encryption/external_dbpa_encryption_utils.h" +#include "parquet/exception.h" +#include "parquet/types.h" + +using parquet::encryption::external::LoadableEncryptorUtils; +using parquet::encryption::external::DBPAEnumUtils; +using parquet::encryption::external::DBPAExecutor; + +using dbps::external::EncryptionResult; +using dbps::external::DecryptionResult; + +namespace parquet::encryption { + +namespace { +// Utility function to load and initialize a DataBatchProtectionAgentInterface instance +// Shared between the encryptor and decryptor. +std::unique_ptr LoadAndInitializeAgent( + const std::string& column_name, + const std::map& connection_config, + const std::string& app_context, + const std::string& key_id, + Type::type data_type, + Compression::type compression_type, + std::optional datatype_length, + std::shared_ptr key_value_metadata) { + + // Load a new DataBatchProtectionAgentInterface instance from the shared library + const std::string SHARED_LIBRARY_PATH_KEY = "agent_library_path"; + const std::string INIT_TIMEOUT_KEY = "agent_init_timeout_ms"; + const std::string ENCRYPT_TIMEOUT_KEY = "agent_encrypt_timeout_ms"; + const std::string DECRYPT_TIMEOUT_KEY = "agent_decrypt_timeout_ms"; + + // Step 1: Get path to the shared library + auto it = connection_config.find(SHARED_LIBRARY_PATH_KEY); + if (it == connection_config.end()) { + auto const msg = "Required configuration key '" + SHARED_LIBRARY_PATH_KEY + "' not found in connection_config"; + ARROW_LOG(ERROR) << msg; + throw ParquetException(msg); + } + auto library_path = it->second; + ARROW_LOG(DEBUG) << "Loading agent from library: library_path = " << library_path; + + // Step 2: Load an instance of the DataBatchProtectionAgentInterface + auto agent_instance = LoadableEncryptorUtils::LoadFromLibrary(library_path); + if (!agent_instance) { + ARROW_LOG(ERROR) << "Failed to create instance of DataBatchProtectionAgentInterface"; + throw ParquetException("Failed to create instance of DataBatchProtectionAgentInterface"); + } + + //Step 3: Wrap the agent in a DBPAExecutor. + //operations will timeout, exceptions will be re-thrown. + + ARROW_LOG(DEBUG) << "Wrapping Agent in DBPAExecutor"; + + // Assign default values to the timeouts. + int64_t init_timeout_ms = 10*1000; //10 seconds + int64_t encrypt_timeout_ms = 30*1000; //30 seconds + int64_t decrypt_timeout_ms = 30*1000; //30 seconds. + // Override the default values if they are present in the connection_config. + try { + if (connection_config.find(INIT_TIMEOUT_KEY) != connection_config.end()) { + init_timeout_ms = std::stoi(connection_config.at(INIT_TIMEOUT_KEY)); + } + if (connection_config.find(ENCRYPT_TIMEOUT_KEY) != connection_config.end()) { + encrypt_timeout_ms = std::stoi(connection_config.at(ENCRYPT_TIMEOUT_KEY)); + } + if (connection_config.find(DECRYPT_TIMEOUT_KEY) != connection_config.end()) { + decrypt_timeout_ms = std::stoi(connection_config.at(DECRYPT_TIMEOUT_KEY)); + } + } catch (const std::exception& e) { + ARROW_LOG(ERROR) << "Failed to parse timeout values from connection_config: " << e.what(); + throw ParquetException("Failed to parse timeout values from connection_config"); + } + + ARROW_LOG(DEBUG) << "init_timeout_ms = " << init_timeout_ms; + ARROW_LOG(DEBUG) << "encrypt_timeout_ms = " << encrypt_timeout_ms; + ARROW_LOG(DEBUG) << "decrypt_timeout_ms = " << decrypt_timeout_ms; + + auto executor_wrapped_agent = std::make_unique( + /*agent*/ std::move(agent_instance), + /*init_timeout_ms*/ init_timeout_ms, + /*encrypt_timeout_ms*/ encrypt_timeout_ms, + /*decrypt_timeout_ms*/ decrypt_timeout_ms + ); + + // Step 4: Initialize the agent. + ARROW_LOG(DEBUG) << "Initializing agent instance"; + + // Convert KeyValueMetadata (only provided by the decryptor) into a std::map + std::optional> column_encryption_metadata = + ExternalDBPAUtils::KeyValueMetadataToStringMap(key_value_metadata); + + executor_wrapped_agent->init( + /*column_name*/ column_name, + /*connection_config*/ connection_config, + /*app_context*/ app_context, + /*column_key_id*/ key_id, + /*data_type*/ DBPAEnumUtils::ParquetTypeToDBPA(data_type), + /*datatype_length*/ datatype_length, + /*compression_type*/ DBPAEnumUtils::ArrowCompressionToDBPA(compression_type), + /*column_encryption_metadata*/ std::move(column_encryption_metadata) + ); + + ARROW_LOG(DEBUG) << "Successfully initialized agent instance"; + + return executor_wrapped_agent; +} //LoadAndInitializeAgent() + +// Local helper to map encoding properties' page_type to encryption module type +// Returns std::nullopt if page_type is unsupported +std::optional GetModuleTypeFromEncodingProperties( + const EncodingProperties& encoding_properties) { + auto page_type = encoding_properties.GetPageType(); + if (page_type == parquet::PageType::DICTIONARY_PAGE) { + return encryption::kDictionaryPage; + } + if (page_type == parquet::PageType::DATA_PAGE || page_type == parquet::PageType::DATA_PAGE_V2) { + return encryption::kDataPage; + } + return std::nullopt; +} //GetModuleTypeFromEncodingProperties() + +} // namespace + +// Update the encryptor-level metadata accumulator based on encoding attributes and +// EncryptionResult-provided metadata. If no metadata is available or page_type is +// unsupported/absent, function performs no-op. +void UpdateEncryptorMetadata( + std::map>& metadata_by_module, + const EncodingProperties& encoding_properties, + const dbps::external::EncryptionResult& result) { + try { + auto module_type_opt = GetModuleTypeFromEncodingProperties(encoding_properties); + if (!module_type_opt.has_value()) { + return; + } + auto column_encryption_metadata_opt = result.encryption_metadata(); + if (!column_encryption_metadata_opt.has_value()) { + return; + } + auto& module_metadata = metadata_by_module[module_type_opt.value()]; + for (const auto& kv : column_encryption_metadata_opt.value()) { + module_metadata[kv.first] = kv.second; + } + } catch (const std::exception& e) { + throw ParquetException("UpdateEncryptorMetadata failed: " + std::string(e.what())); + } +} //UpdateEncryptorMetadata() + +std::optional> ExternalDBPAUtils::KeyValueMetadataToStringMap( + const std::shared_ptr& key_value_metadata) { +if (key_value_metadata == nullptr) { + return std::nullopt; +} +std::map metadata_map; +const auto& keys = key_value_metadata->keys(); +const auto& values = key_value_metadata->values(); +const auto count = std::min(keys.size(), values.size()); +for (size_t i = 0; i < count; ++i) { + metadata_map.emplace(keys[i], values[i]); +} +if (metadata_map.empty()) { + return std::nullopt; +} +return metadata_map; +} + +//this is a private constructor, invoked from Make() +//at this point, the agent_instance is assumed to be initialized. +ExternalDBPAEncryptorAdapter::ExternalDBPAEncryptorAdapter( + ParquetCipher::type algorithm, std::string column_name, std::string key_id, + Type::type data_type, Compression::type compression_type, Encoding::type encoding_type, + std::optional datatype_length, std::string app_context, std::map connection_config, + std::unique_ptr agent_instance) + : algorithm_(algorithm), column_name_(column_name), key_id_(key_id), + data_type_(data_type), compression_type_(compression_type), + encoding_type_(encoding_type), datatype_length_(datatype_length), app_context_(app_context), + connection_config_(connection_config), + agent_instance_(std::move(agent_instance)) { + + if (algorithm != ParquetCipher::EXTERNAL_DBPA_V1) { + throw ParquetException("ExternalDBPAEncryptorAdapter -- Only algorithm ExternalDBPA_V1 is supported"); + } +} + +std::unique_ptr ExternalDBPAEncryptorAdapter::Make( + ParquetCipher::type algorithm, std::string column_name, std::string key_id, + Type::type data_type, Compression::type compression_type, Encoding::type encoding_type, + std::string app_context, std::map connection_config, + std::optional datatype_length) { + + // Ensure DBPA logging threshold is configured before any logs here + EnsureDbpaLoggingConfigured(); + + + if (::arrow::util::ArrowLog::IsLevelEnabled(::arrow::util::ArrowLogLevel::ARROW_DEBUG)) { + ARROW_LOG(DEBUG) << "ExternalDBPAEncryptorAdapter::Make() -- Make()"; + ARROW_LOG(DEBUG) << " algorithm = " << algorithm; + ARROW_LOG(DEBUG) << " column_name = " << column_name; + ARROW_LOG(DEBUG) << " key_id = " << key_id; + ARROW_LOG(DEBUG) << " data_type = " << data_type; + ARROW_LOG(DEBUG) << " compression_type = " << compression_type; + ARROW_LOG(DEBUG) << " encoding_type = " << encoding_type; + ARROW_LOG(DEBUG) << " app_context = " << app_context; + ARROW_LOG(DEBUG) << " connection_config:"; + for (const auto& [key, value] : connection_config) { + ARROW_LOG(DEBUG) << " " << key << " = " << value; + } + } + + if (algorithm != ParquetCipher::EXTERNAL_DBPA_V1) { + throw ParquetException("ExternalDBPAEncryptorAdapter::Make() -- Only algorithm ExternalDBPA_V1 is supported"); + } + + ARROW_LOG(DEBUG) << "ExternalDBPAEncryptorAdapter::ExternalDBPAEncryptorAdapter() -- loading and initializing agent"; + // Load and initialize the agent using the utility function + auto agent_instance = LoadAndInitializeAgent( + column_name, connection_config, app_context, key_id, data_type, compression_type, datatype_length, + /*key_value_metadata*/ nullptr); + + //if we got to this point, the agent was initialized successfully + ARROW_LOG(DEBUG) << "ExternalDBPAEncryptorAdapter::ExternalDBPAEncryptorAdapter() -- creating ExternalDBPAEncryptorAdapter"; + + // create the instance of the ExternalDBPAEncryptorAdapter + auto result = std::unique_ptr( + new ExternalDBPAEncryptorAdapter( + /*algorithm*/ algorithm, + /*column_name*/ column_name, + /*key_id*/ key_id, + /*data_type*/ data_type, + /*compression_type*/ compression_type, + /*encoding_type*/ encoding_type, + /*datatype_length*/ datatype_length, + /*app_context*/ app_context, + /*connection_config*/ connection_config, + /*agent_instance*/ std::move(agent_instance)) + ); + + ARROW_LOG(DEBUG) << "ExternalDBPAEncryptorAdapter created successfully"; + + return result; + } + +int32_t ExternalDBPAEncryptorAdapter::CiphertextLength(int64_t plaintext_len) const { + throw ParquetException("ExternalDBPAEncryptorAdapter::CiphertextLength is not supported"); +} + +void ExternalDBPAEncryptorAdapter::UpdateEncodingProperties(std::unique_ptr encoding_properties) { + ARROW_LOG(DEBUG) << "ExternalDBPAEncryptorAdapter::UpdateEncodingProperties"; + + //fill-in values from the decryptor constructor. + encoding_properties->set_column_path(column_name_); + encoding_properties->set_physical_type(data_type_, datatype_length_); + encoding_properties->set_compression_codec(compression_type_); + + encoding_properties->validate(); + encoding_properties_ = std::move(encoding_properties); + encoding_properties_updated_ = true; +} + +std::shared_ptr ExternalDBPAEncryptorAdapter::GetKeyValueMetadata( + int8_t module_type) { + auto it = column_encryption_metadata_.find(module_type); + if (it == column_encryption_metadata_.end() || it->second.empty()) { + return nullptr; + } + + const auto& metadata_map = it->second; + std::unordered_map unordered_map(metadata_map.begin(), metadata_map.end()); + return ::arrow::key_value_metadata(unordered_map); +} //GetKeyValueMetadata() + +int32_t ExternalDBPAEncryptorAdapter::EncryptWithManagedBuffer( + ::arrow::util::span plaintext, ::arrow::ResizableBuffer* ciphertext) { + + if (!encoding_properties_updated_) { + ARROW_LOG(ERROR) << "ExternalDBPAEncryptorAdapter:: EncryptionParams not updated"; + throw ParquetException("ExternalDBPAEncryptorAdapter:: EncryptionParams not updated"); + } + + encoding_properties_updated_ = false; + + return InvokeExternalEncrypt(plaintext, ciphertext, encoding_properties_->ToPropertiesMap()); +} + +int32_t ExternalDBPAEncryptorAdapter::SignedFooterEncrypt( + ::arrow::util::span footer, ::arrow::util::span key, + ::arrow::util::span aad, ::arrow::util::span nonce, + ::arrow::util::span encrypted_footer) { + throw ParquetException("ExternalDBPAEncryptorAdapter::SignedFooterEncrypt is not supported"); +} + +int32_t ExternalDBPAEncryptorAdapter::InvokeExternalEncrypt( + ::arrow::util::span plaintext, + ::arrow::ResizableBuffer* ciphertext, + std::map encoding_attrs) { + + if (::arrow::util::ArrowLog::IsLevelEnabled(::arrow::util::ArrowLogLevel::ARROW_DEBUG)) { + ARROW_LOG(DEBUG) << "*-*-*- START: ExternalDBPAEncryptor::Encrypt *-*-*-"; + ARROW_LOG(DEBUG) << "Encryption Algorithm: [" << algorithm_ << "]"; + ARROW_LOG(DEBUG) << "Column Name: [" << column_name_ << "]"; + ARROW_LOG(DEBUG) << "Key ID: [" << key_id_ << "]"; + ARROW_LOG(DEBUG) << "Data Type: [" << data_type_ << "]"; + ARROW_LOG(DEBUG) << "Compression Type: [" << compression_type_ << "]"; + ARROW_LOG(DEBUG) << "Encoding Type: [" << encoding_type_ << "]"; + ARROW_LOG(DEBUG) << "App Context: [" << app_context_ << "]"; + ARROW_LOG(DEBUG) << "Connection Config:"; + for (const auto& [cfg_key, cfg_value] : connection_config_) { + ARROW_LOG(DEBUG) << " [" << cfg_key << "]: [" << cfg_value << "]"; + } + } + + ARROW_LOG(DEBUG) << "Calling agent_instance_->Encrypt..."; + std::unique_ptr result = agent_instance_->Encrypt(plaintext, std::move(encoding_attrs)); + + if (!result->success()) { + ARROW_LOG(ERROR) << "Encryption failed: " << result->error_message(); + throw ParquetException(result->error_message()); + } + + ARROW_LOG(DEBUG) << "Encryption successful"; + ARROW_LOG(DEBUG) << " result size: " << result->size() << " bytes"; + ARROW_LOG(DEBUG) << " result ciphertext size: " << result->ciphertext().size() << " bytes"; + + const auto ciphertext_size64 = result->ciphertext().size(); + if (ciphertext_size64 > static_cast(std::numeric_limits::max())) { + throw ParquetException("Ciphertext size exceeds int32_t max"); + } + const int32_t ciphertext_size = static_cast(ciphertext_size64); + auto status = ciphertext->Resize(ciphertext_size, false); + if (!status.ok()) { + ARROW_LOG(ERROR) << "Ciphertext buffer resize failed: " << status.ToString(); + throw ParquetException("Ciphertext buffer resize failed"); + } + + ARROW_LOG(DEBUG) << "Copying result to ciphertext buffer..."; + if (ciphertext_size > 0) { + std::memcpy(ciphertext->mutable_data(), result->ciphertext().data(), ciphertext_size); + } + ARROW_LOG(DEBUG) << "Encryption completed successfully"; + + // Accumulate any column_encryption_metadata returned by the result per module type + UpdateEncryptorMetadata( + /*metadata_by_module*/ column_encryption_metadata_, + /*encoding_properties*/ *encoding_properties_, + /*result*/ *result); + + return static_cast(result->size()); + } + +ExternalDBPAEncryptorAdapter* ExternalDBPAEncryptorAdapterFactory::GetEncryptor( + ParquetCipher::type algorithm, const ColumnChunkMetaDataBuilder* column_chunk_metadata, + ExternalFileEncryptionProperties* external_file_encryption_properties) { + if (column_chunk_metadata == nullptr) { + throw ParquetException("External DBPA encryption requires column chunk metadata"); + } + auto column_path = column_chunk_metadata->descr()->path(); + if (encryptor_cache_.find(column_path->ToDotString()) == encryptor_cache_.end()) { + auto connection_config = external_file_encryption_properties->connection_config(); + if (connection_config.find(algorithm) == connection_config.end()) { + throw ParquetException("External DBPA encryption requires its connection configuration"); + } + + auto column_encryption_properties = external_file_encryption_properties + ->column_encryption_properties(column_path->ToDotString()); + if (column_encryption_properties == nullptr) { + std::stringstream ss; + ss << "External DBPA encryption requires column encryption properties for column [" + << column_path->ToDotString() << "]"; + throw ParquetException(ss.str()); + } + + auto data_type = column_chunk_metadata->descr()->physical_type(); + std::optional datatype_length; + if (data_type == Type::FIXED_LEN_BYTE_ARRAY) { + datatype_length = column_chunk_metadata->descr()->type_length(); + } + auto compression_type = column_chunk_metadata->properties()->compression(column_path); + auto encoding_type = column_chunk_metadata->properties()->encoding(column_path); + auto app_context = external_file_encryption_properties->app_context(); + auto connection_config_for_algorithm = connection_config.at(algorithm); + + std::string key_id; + try { + auto key_metadata = KeyMetadata::Parse(column_encryption_properties->key_metadata()); + key_id = key_metadata.key_material().master_key_id(); + } catch (const ParquetException& e) { + // It is possible for the key metadata to only contain the key id itself, so if + // it cannot be parsed as valid JSON, send the key id as string for the ExternalDBPA + // to process. + key_id = column_encryption_properties->key_metadata(); + } + + encryptor_cache_[column_path->ToDotString()] = ExternalDBPAEncryptorAdapter::Make( + algorithm, column_path->ToDotString(), key_id, data_type, compression_type, + encoding_type, app_context, connection_config_for_algorithm, datatype_length); + } + + return encryptor_cache_[column_path->ToDotString()].get(); +} + +//private constructor, invoked from Make() +//at this point, the agent_instance is assumed to be initialized. +//TODO: consider cleaning up the signature of this private constructor. +// Most of the arguments are only needed by agent_instance, which is +// instantiated before this constructor is invoked. +ExternalDBPADecryptorAdapter::ExternalDBPADecryptorAdapter( + ParquetCipher::type algorithm, std::string column_name, std::string key_id, + Type::type data_type, Compression::type compression_type, + std::vector encoding_types, std::optional datatype_length, std::string app_context, + std::map connection_config, + std::unique_ptr agent_instance, + std::shared_ptr key_value_metadata) + : algorithm_(algorithm), column_name_(column_name), key_id_(key_id), + data_type_(data_type), compression_type_(compression_type), + encoding_types_(encoding_types), datatype_length_(datatype_length), app_context_(app_context), + connection_config_(connection_config), + agent_instance_(std::move(agent_instance)) { + + if (algorithm != ParquetCipher::EXTERNAL_DBPA_V1) { + throw ParquetException("ExternalDBPADecryptorAdapter -- Only algorithm ExternalDBPA_V1 is supported"); + } + if (key_value_metadata != nullptr) { + key_value_metadata_ = key_value_metadata->Copy(); + } +} + +std::unique_ptr ExternalDBPADecryptorAdapter::Make( + ParquetCipher::type algorithm, std::string column_name, std::string key_id, + Type::type data_type, Compression::type compression_type, + std::vector encoding_types, std::string app_context, + std::map connection_config, + std::optional datatype_length, + std::shared_ptr key_value_metadata) { + + if (algorithm != ParquetCipher::EXTERNAL_DBPA_V1) { + throw ParquetException("ExternalDBPADecryptorAdapter::Make() -- Only algorithm ExternalDBPA_V1 is supported"); + } + + // Ensure DBPA logging threshold is configured before any logs here + EnsureDbpaLoggingConfigured(); + + ARROW_LOG(DEBUG) << "ExternalDBPADecryptorAdapter::Make() -- Make()"; + ARROW_LOG(DEBUG) << " algorithm = " << algorithm; + ARROW_LOG(DEBUG) << " column_name = " << column_name; + ARROW_LOG(DEBUG) << " key_id = " << key_id; + ARROW_LOG(DEBUG) << " data_type = " << data_type; + ARROW_LOG(DEBUG) << " compression_type = " << compression_type; + { + std::stringstream ss; + ss << " encoding_types = ["; + for (const auto& encoding : encoding_types) { + ss << static_cast(encoding) << " "; + } + ss << "]"; + ARROW_LOG(DEBUG) << ss.str(); + } + ARROW_LOG(DEBUG) << " app_context = " << app_context; + ARROW_LOG(DEBUG) << " connection_config:"; + for (const auto& [key, value] : connection_config) { + ARROW_LOG(DEBUG) << " " << key << " = " << value; + } + ARROW_LOG(DEBUG) << " key_value_metadata:"; + if (key_value_metadata == nullptr) { + ARROW_LOG(DEBUG) << " key_value_metadata: not provided"; + } else { + ARROW_LOG(DEBUG) << " key_value_metadata: " << key_value_metadata->ToString(); + } + + ARROW_LOG(DEBUG) << "ExternalDBPADecryptorAdapter::ExternalDBPADecryptorAdapter() -- loading and initializing agent"; + // Load and initialize the agent using the utility function + auto agent_instance = LoadAndInitializeAgent( + column_name, connection_config, app_context, key_id, data_type, compression_type, datatype_length, + /*key_value_metadata*/ key_value_metadata); + + //if we got to this point, the agent was initialized successfully + + // create the instance of the ExternalDBPADecryptorAdapter + auto result = std::unique_ptr( + new ExternalDBPADecryptorAdapter( + /*algorithm*/ algorithm, + /*column_name*/ column_name, + /*key_id*/ key_id, + /*data_type*/ data_type, + /*compression_type*/ compression_type, + /*encoding_types*/ encoding_types, + /*datatype_length*/ datatype_length, + /*app_context*/ app_context, + /*connection_config*/ connection_config, + /*agent_instance*/ std::move(agent_instance), + /*key_value_metadata*/ key_value_metadata) + ); + ARROW_LOG(DEBUG) << "ExternalDBPADecryptorAdapter created successfully"; + + return result; + } + +int32_t ExternalDBPADecryptorAdapter::PlaintextLength(int32_t ciphertext_len) const { + throw ParquetException("ExternalDBPADecryptorAdapter::PlaintextLength is not supported"); +} + +int32_t ExternalDBPADecryptorAdapter::CiphertextLength(int32_t plaintext_len) const { + throw ParquetException("ExternalDBPADecryptorAdapter::CiphertextLength is not supported"); +} + +void ExternalDBPADecryptorAdapter::UpdateEncodingProperties(std::unique_ptr encoding_properties) { + ARROW_LOG(DEBUG) << "ExternalDBPADecryptorAdapter::UpdateEncodingProperties"; + + //fill-in values from the decryptor constructor. + encoding_properties->set_column_path(column_name_); + encoding_properties->set_physical_type(data_type_, datatype_length_); + encoding_properties->set_compression_codec(compression_type_); + + encoding_properties->validate(); + encoding_properties_ = std::move(encoding_properties); + encoding_properties_updated_ = true; +} + +int32_t ExternalDBPADecryptorAdapter::DecryptWithManagedBuffer( + ::arrow::util::span ciphertext, ::arrow::ResizableBuffer* plaintext) { + + if (!encoding_properties_updated_) { + ARROW_LOG(ERROR) << "ExternalDBPADecryptorAdapter:: DecryptionParams not updated"; + throw ParquetException("ExternalDBPADecryptorAdapter:: DecryptionParams not updated"); + } + + encoding_properties_updated_ = false; + + return InvokeExternalDecrypt(ciphertext, plaintext, encoding_properties_->ToPropertiesMap()); +} + +int32_t ExternalDBPADecryptorAdapter::InvokeExternalDecrypt( + ::arrow::util::span ciphertext, + ::arrow::ResizableBuffer* plaintext, + std::map encoding_attrs) { + + if (::arrow::util::ArrowLog::IsLevelEnabled(::arrow::util::ArrowLogLevel::ARROW_DEBUG)) { + ARROW_LOG(DEBUG) << "*-*-*- START: ExternalDBPADecryptor::Decrypt *-*-*-"; + ARROW_LOG(DEBUG) << "Decryption Algorithm: [" << algorithm_ << "]"; + ARROW_LOG(DEBUG) << "Column Name: [" << column_name_ << "]"; + ARROW_LOG(DEBUG) << "Key ID: [" << key_id_ << "]"; + ARROW_LOG(DEBUG) << "Data Type: [" << data_type_ << "]"; + ARROW_LOG(DEBUG) << "Compression Type: [" << compression_type_ << "]"; + { + std::stringstream ss; + ss << "Encoding Types: ["; + for (const auto& encoding_type : encoding_types_) { + ss << static_cast(encoding_type) << " "; + } + ss << "]"; + ARROW_LOG(DEBUG) << ss.str(); + } + ARROW_LOG(DEBUG) << "App Context: [" << app_context_ << "]"; + ARROW_LOG(DEBUG) << "Connection Config:"; + for (const auto& [key, value] : connection_config_) { + ARROW_LOG(DEBUG) << " [" << key << "]: [" << value << "]"; + } + } + + ARROW_LOG(DEBUG) << "Calling agent_instance_->Decrypt..."; + std::unique_ptr result = agent_instance_->Decrypt(ciphertext, std::move(encoding_attrs)); + + if (!result->success()) { + ARROW_LOG(ERROR) << "Decryption failed: " << result->error_message(); + throw ParquetException(result->error_message()); + } + + ARROW_LOG(DEBUG) << "Decryption successful"; + ARROW_LOG(DEBUG) << " result size: " << result->size() << " bytes"; + ARROW_LOG(DEBUG) << " result plaintext size: " << result->plaintext().size() << " bytes"; + + const auto plaintext_size64 = result->plaintext().size(); + if (plaintext_size64 > static_cast(std::numeric_limits::max())) { + throw ParquetException("Plaintext size exceeds int32_t max"); + } + const int32_t plaintext_size = static_cast(plaintext_size64); + auto status = plaintext->Resize(plaintext_size, false); + if (!status.ok()) { + ARROW_LOG(ERROR) << "Plaintext buffer resize failed: " << status.ToString(); + throw ParquetException("Plaintext buffer resize failed"); + } + + ARROW_LOG(DEBUG) << "Copying result to plaintext buffer..."; + if (plaintext_size > 0) { + std::memcpy(plaintext->mutable_data(), result->plaintext().data(), plaintext_size); + } + ARROW_LOG(DEBUG) << "Decryption completed successfully"; + + const auto total_size64 = result->size(); + if (total_size64 > static_cast(std::numeric_limits::max())) { + throw ParquetException("Result size exceeds int32_t max"); + } + return static_cast(total_size64); + } + +std::unique_ptr ExternalDBPADecryptorAdapterFactory::GetDecryptor( + ParquetCipher::type algorithm, const ColumnCryptoMetaData* crypto_metadata, + const ColumnChunkMetaData* column_chunk_metadata, + ExternalFileDecryptionProperties* external_file_decryption_properties) { + if (column_chunk_metadata == nullptr || crypto_metadata == nullptr) { + throw ParquetException("External DBPA decryption requires column chunk and crypto metadata"); + } + auto connection_config = external_file_decryption_properties->connection_config(); + if (connection_config.find(algorithm) == connection_config.end()) { + throw ParquetException("External DBPA decryption requires its connection configuration"); + } + auto column_path = column_chunk_metadata->descr()->path(); + auto data_type = column_chunk_metadata->descr()->physical_type(); + std::optional datatype_length; + if (data_type == Type::FIXED_LEN_BYTE_ARRAY) { + datatype_length = column_chunk_metadata->descr()->type_length(); + } + auto compression_type = column_chunk_metadata->compression(); + auto encoding_types = column_chunk_metadata->encodings(); + auto app_context = external_file_decryption_properties->app_context(); + auto connection_config_for_algorithm = connection_config.at(algorithm); + auto key_value_metadata = column_chunk_metadata->key_value_metadata(); + + std::string key_id; + try { + auto key_metadata = KeyMetadata::Parse(crypto_metadata->key_metadata()); + key_id = key_metadata.key_material().master_key_id(); + } catch (const ParquetException& e) { + // It is possible for the key metadata to only contain the key id itself, so if + // it cannot be parsed as valid JSON, send the key id as string for the ExternalDBPA + // to process. + key_id = crypto_metadata->key_metadata(); + } + + return ExternalDBPADecryptorAdapter::Make( + algorithm, column_path->ToDotString(), key_id, data_type, compression_type, + encoding_types, app_context, connection_config_for_algorithm, datatype_length, + key_value_metadata); + } + +} // namespace parquet::encryption \ No newline at end of file diff --git a/cpp/src/parquet/encryption/external_dbpa_encryption.h b/cpp/src/parquet/encryption/external_dbpa_encryption.h new file mode 100644 index 000000000000..edf88fd55c9c --- /dev/null +++ b/cpp/src/parquet/encryption/external_dbpa_encryption.h @@ -0,0 +1,238 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include + +#include "parquet/encryption/encryptor_interface.h" +#include "parquet/encryption/decryptor_interface.h" +#include "parquet/encryption/encoding_properties.h" +#include "parquet/metadata.h" +#include "parquet/types.h" + +using dbps::external::DataBatchProtectionAgentInterface; + +namespace parquet::encryption { + +/// Call an external Data Batch Protection Agent (DBPA) to encrypt data. +class ExternalDBPAEncryptorAdapter : public EncryptorInterface { + public: + static std::unique_ptr Make( + ParquetCipher::type algorithm, std::string column_name, + std::string key_id, Type::type data_type, Compression::type compression_type, + Encoding::type encoding_type, std::string app_context, + std::map connection_config, + std::optional datatype_length); + + ~ExternalDBPAEncryptorAdapter() = default; + + /// Signal whether the encryptor can calculate a valid ciphertext length before performing + /// encryption. + [[nodiscard]] bool CanCalculateCiphertextLength() const override { return false; } + + /// The size of the ciphertext, for this cipher and the specified plaintext length. + [[nodiscard]] int32_t CiphertextLength(int64_t plaintext_len) const override; + + /// Encryption not supported as we cannot calculate the ciphertext before encryption. + int32_t Encrypt(::arrow::util::span plaintext, + ::arrow::util::span key, + ::arrow::util::span aad, + ::arrow::util::span ciphertext) override { + std::stringstream ss; + ss << "Encrypt is not supported in ExternalDBPAEncryptorAdapter, "; + ss << "use EncryptWithManagedBuffer instead"; + throw ParquetException(ss.str()); + } + + /// Encrypt the plaintext and leave the results in the ciphertext buffer. + /// The buffer will be resized to the appropriate size by the agent during encryption. + int32_t EncryptWithManagedBuffer(::arrow::util::span plaintext, + ::arrow::ResizableBuffer* ciphertext) override; + + /// Encrypts plaintext footer, in order to compute footer signature (tag). + int32_t SignedFooterEncrypt(::arrow::util::span footer, + ::arrow::util::span key, + ::arrow::util::span aad, + ::arrow::util::span nonce, + ::arrow::util::span encrypted_footer) override; + + void UpdateEncodingProperties(std::unique_ptr encoding_properties) override; + + std::shared_ptr GetKeyValueMetadata(int8_t module_type) override; + + private: + //agent_instance is assumed to be initialized at the time of construction. + //no initialization nor checks to verify that it is initialized are performed. + + ExternalDBPAEncryptorAdapter( + ParquetCipher::type algorithm, std::string column_name, + std::string key_id, Type::type data_type, Compression::type compression_type, + Encoding::type encoding_type, std::optional datatype_length, std::string app_context, + std::map connection_config, + std::unique_ptr agent_instance); + + int32_t InvokeExternalEncrypt( + ::arrow::util::span plaintext, + ::arrow::ResizableBuffer* ciphertext, + std::map encoding_attrs); + + ParquetCipher::type algorithm_; + std::string column_name_; + std::string key_id_; + Type::type data_type_; + Compression::type compression_type_; + Encoding::type encoding_type_; + std::optional datatype_length_; + std::string app_context_; + std::map connection_config_; + + std::unique_ptr agent_instance_; + + std::unique_ptr encoding_properties_; + bool encoding_properties_updated_ = false; + + // Accumulated column encryption metadata per module type (e.g., data page, dictionary page) + // to be used later by GetKeyValueMetadata. + std::map> column_encryption_metadata_; +}; + +// Utilities for External DBPA adapters +class ExternalDBPAUtils { + public: + // Convert Arrow KeyValueMetadata to a std::map. + // Returns std::nullopt if the input is null or contains no pairs. + static std::optional> KeyValueMetadataToStringMap( + const std::shared_ptr& key_value_metadata); +}; + +// Update encryptor-level metadata accumulator based on encoding attributes and +// EncryptionResult-provided metadata. If no metadata is available or page_type is +// unsupported/absent, function performs no-op. +void UpdateEncryptorMetadata( + std::map>& metadata_by_module, + const EncodingProperties& encoding_properties, + const dbps::external::EncryptionResult& result); + +/// Factory for ExternalDBPAEncryptorAdapter instances. The cache exists while the write +/// operation is open, and is used to guarantee the lifetime of the encryptor. +class ExternalDBPAEncryptorAdapterFactory { + public: + ExternalDBPAEncryptorAdapter* GetEncryptor( + ParquetCipher::type algorithm, const ColumnChunkMetaDataBuilder* column_chunk_metadata, + ExternalFileEncryptionProperties* external_file_encryption_properties); + + private: + std::map> encryptor_cache_; +}; + +/// Call an external Data Batch Protection Agent (DBPA) to decrypt data. +/// connection configuration provided. +class ExternalDBPADecryptorAdapter : public DecryptorInterface { + public: + static std::unique_ptr Make( + ParquetCipher::type algorithm, std::string column_name, + std::string key_id, Type::type data_type, Compression::type compression_type, + std::vector encoding_types, std::string app_context, + std::map connection_config, + std::optional datatype_length, + std::shared_ptr key_value_metadata); + + ~ExternalDBPADecryptorAdapter() = default; + + /// Signal whether the decryptor can calculate a valid plaintext or ciphertext length before + /// performing decryption or not. If false, a proper sized buffer cannot be allocated before + /// calling the Decrypt method, and Arrow must use this decryptor's DecryptWithManagedBuffer + /// method instead of Decrypt. + [[nodiscard]] bool CanCalculateLengths() const override { return false; } + + /// The size of the plaintext, for this cipher and the specified ciphertext length. + [[nodiscard]] int32_t PlaintextLength(int32_t ciphertext_len) const override; + + /// The size of the ciphertext, for this cipher and the specified plaintext length. + [[nodiscard]] int32_t CiphertextLength(int32_t plaintext_len) const override; + + /// Decrypt is not supported as we cannot calculate the plaintext length before decryption. + int32_t Decrypt(::arrow::util::span ciphertext, + ::arrow::util::span key, + ::arrow::util::span aad, + ::arrow::util::span plaintext) override { + std::stringstream ss; + ss << "Decrypt is not supported in ExternalDBPADecryptorAdapter, "; + ss << "use DecryptWithManagedBuffer instead"; + throw ParquetException(ss.str()); + } + + /// Decrypt the ciphertext and leave the results in the plaintext buffer. + /// The buffer will be resized to the correct size during decryption. This method is used + /// when the decryptor cannot calculate the plaintext length before decryption. + int32_t DecryptWithManagedBuffer(::arrow::util::span ciphertext, + ::arrow::ResizableBuffer* plaintext) override; + + void UpdateEncodingProperties(std::unique_ptr encoding_properties) override; + + private: + //agent_instance is assumed to be initialized at the time of construction. + //no initialization nor checks to verify that it is initialized are performed. + ExternalDBPADecryptorAdapter( + ParquetCipher::type algorithm, std::string column_name, + std::string key_id, Type::type data_type, Compression::type compression_type, + std::vector encoding_types, std::optional datatype_length, + std::string app_context, std::map connection_config, + std::unique_ptr agent_instance, + std::shared_ptr key_value_metadata); + + int32_t InvokeExternalDecrypt( + ::arrow::util::span ciphertext, + ::arrow::ResizableBuffer* plaintext, + std::map encoding_attrs); + + ParquetCipher::type algorithm_; + std::string column_name_; + std::string key_id_; + Type::type data_type_; + Compression::type compression_type_; + // Set of all encodings used for this column. Comes directly from the column chunk metadata. + std::vector encoding_types_; + std::optional datatype_length_; + std::string app_context_; + std::map connection_config_; + + std::unique_ptr agent_instance_; + + std::unique_ptr encoding_properties_; + bool encoding_properties_updated_ = false; + + // Store the key value metadata from the column chunk metadata. + std::shared_ptr key_value_metadata_; +}; + +/// Factory for ExternalDBPADecryptorAdapter instances. No cache exists for decryptors. +class ExternalDBPADecryptorAdapterFactory { + public: + std::unique_ptr GetDecryptor( + ParquetCipher::type algorithm, const ColumnCryptoMetaData* crypto_metadata, + const ColumnChunkMetaData* column_chunk_metadata, + ExternalFileDecryptionProperties* external_file_decryption_properties); +}; + +} // namespace parquet::encryption diff --git a/cpp/src/parquet/encryption/external_dbpa_encryption_test.cc b/cpp/src/parquet/encryption/external_dbpa_encryption_test.cc new file mode 100644 index 000000000000..7bdda314b69b --- /dev/null +++ b/cpp/src/parquet/encryption/external_dbpa_encryption_test.cc @@ -0,0 +1,654 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include + +#include "arrow/util/key_value_metadata.h" +#include "parquet/encryption/encryption.h" +#include "parquet/encryption/external_dbpa_encryption.h" +#include "parquet/encryption/external/test_utils.h" +#include "parquet/encryption/encoding_properties.h" +#include "parquet/encryption/encryption_utils.h" + +namespace parquet::encryption::test { + +class ExternalDBPAEncryptorAdapterTest : public ::testing::Test { + protected: + void SetUp() override { + + // this library will use heuristics to load "libDBPATestAgent.so", needed for tests here. + std::string library_path = parquet::encryption::external::test::TestUtils::GetTestLibraryPath(); + + app_context_ = + "{\"user_id\": \"abc123\", \"location\": {\"lat\": 9.7489, \"lon\": -83.7534}}"; + connection_config_ = { + {"config_path", "path/to/file"}, + {"agent_library_path", library_path}, + {"agent_init_timeout_ms", "1000"}, + {"agent_encrypt_timeout_ms", "2000"}, + {"agent_decrypt_timeout_ms", "3000"} + }; + key_value_metadata_ = KeyValueMetadata::Make({"key1", "key2"}, {"value1", "value2"}); + } + + std::unique_ptr CreateEncryptor( + ParquetCipher::type algorithm, std::string column_name, std::string key_id, + Type::type data_type, Compression::type compression_type, Encoding::type encoding_type) { + return ExternalDBPAEncryptorAdapter::Make( + algorithm, column_name, key_id, data_type, + compression_type, encoding_type, app_context_, + connection_config_, std::nullopt); + } + + std::unique_ptr CreateDecryptor( + ParquetCipher::type algorithm, std::string column_name, std::string key_id, + Type::type data_type, Compression::type compression_type, Encoding::type encoding_type) { + return ExternalDBPADecryptorAdapter::Make( + algorithm, column_name, key_id, data_type, + compression_type, {encoding_type}, app_context_, + connection_config_, std::nullopt, key_value_metadata_); + } + + void RoundtripEncryption( + ParquetCipher::type algorithm, std::string column_name, std::string key_id, + Type::type data_type, Compression::type compression_type, Encoding::type encoding_type, + std::string plaintext) { + std::unique_ptr encryptor = ExternalDBPAEncryptorAdapter::Make( + algorithm, column_name, key_id, data_type, + compression_type, encoding_type, app_context_, + connection_config_, std::nullopt); + + // Create a simple EncodingProperties for testing using the builder pattern + EncodingPropertiesBuilder builder; + builder.ColumnPath("test_column") + .PhysicalType(data_type) + .CompressionCodec(compression_type) + .PageType(parquet::PageType::DATA_PAGE_V2) + .PageV2DefinitionLevelsByteLength(10) + .PageV2RepetitionLevelsByteLength(10) + .PageV2NumNulls(10) + .PageV2IsCompressed(true) + .DataPageMaxDefinitionLevel(10) + .DataPageMaxRepetitionLevel(1) + .PageEncoding(encoding_type) + .DataPageNumValues(100) + .Build(); + + encryptor->UpdateEncodingProperties(builder.Build()); + + ASSERT_LE(plaintext.size(), + static_cast(std::numeric_limits::max())); + int32_t expected_ciphertext_length = static_cast(plaintext.size()); + + std::shared_ptr ciphertext_buffer = AllocateBuffer( + ::arrow::default_memory_pool(), expected_ciphertext_length); + int32_t encryption_length = encryptor->EncryptWithManagedBuffer( + str2span(plaintext), ciphertext_buffer.get()); + ASSERT_EQ(expected_ciphertext_length, encryption_length); + + std::string ciphertext_str( + ciphertext_buffer->data(), ciphertext_buffer->data() + encryption_length); + + // We know this uses XOR encryption. Therefore, the ciphertext is the same as the plaintext. + // XOR encrytion encrypts each byte of the plaintext with 0xAA. + // See external/dbpa_test_agent.cc for the implementation. + + // Assert that plaintext and ciphertext have the same length + ASSERT_EQ(plaintext.size(), ciphertext_str.size()); + + std::unique_ptr decryptor = ExternalDBPADecryptorAdapter::Make( + algorithm, column_name, key_id, data_type, + compression_type, {encoding_type}, app_context_, + connection_config_, std::nullopt, key_value_metadata_); + + decryptor->UpdateEncodingProperties(builder.Build()); + + ASSERT_LE(ciphertext_str.size(), + static_cast(std::numeric_limits::max())); + int32_t expected_plaintext_length = static_cast(ciphertext_str.size()); + std::shared_ptr plaintext_buffer = AllocateBuffer( + ::arrow::default_memory_pool(), expected_plaintext_length); + int32_t decryption_length = decryptor->DecryptWithManagedBuffer( + str2span(ciphertext_str), plaintext_buffer.get()); + ASSERT_EQ(expected_plaintext_length, decryption_length); + + std::string plaintext_str( + plaintext_buffer->data(), plaintext_buffer->data() + decryption_length); + + // Assert that the decrypted plaintext matches the original plaintext + ASSERT_EQ(plaintext, plaintext_str); + } + +protected: + std::string empty_string = ""; + std::string app_context_; + std::map connection_config_; + std::shared_ptr key_value_metadata_; +}; + +TEST_F(ExternalDBPAEncryptorAdapterTest, RoundtripEncryptionSucceeds) { + ParquetCipher::type algorithm = ParquetCipher::EXTERNAL_DBPA_V1; + std::string column_name = "employee_name"; + std::string key_id = "employee_name_key"; + Type::type data_type = Type::BYTE_ARRAY; + Compression::type compression_type = Compression::UNCOMPRESSED; + Encoding::type encoding_type = Encoding::PLAIN; + std::string plaintext = "Jean-Luc Picard"; + + RoundtripEncryption( + algorithm, column_name, key_id, data_type, compression_type, encoding_type, plaintext); +} + +TEST_F(ExternalDBPAEncryptorAdapterTest, RoundtripEncryption_EmptyPlaintextDoesNotCrash) { + ParquetCipher::type algorithm = ParquetCipher::EXTERNAL_DBPA_V1; + std::string column_name = "employee_name"; + std::string key_id = "employee_name_key"; + Type::type data_type = Type::BYTE_ARRAY; + Compression::type compression_type = Compression::UNCOMPRESSED; + Encoding::type encoding_type = Encoding::PLAIN; + std::string plaintext = ""; + + RoundtripEncryption( + algorithm, column_name, key_id, data_type, compression_type, encoding_type, plaintext); +} + +TEST_F(ExternalDBPAEncryptorAdapterTest, GetKeyValueMetadataReturnsNullWhenEmpty) { + ParquetCipher::type algorithm = ParquetCipher::EXTERNAL_DBPA_V1; + std::string column_name = "employee_name"; + std::string key_id = "employee_name_key"; + Type::type data_type = Type::BYTE_ARRAY; + Compression::type compression_type = Compression::UNCOMPRESSED; + Encoding::type encoding_type = Encoding::PLAIN; + + auto encryptor = ExternalDBPAEncryptorAdapter::Make( + algorithm, column_name, key_id, data_type, compression_type, encoding_type, + app_context_, connection_config_, std::nullopt); + + // No encryption performed yet; metadata should be empty for any module + auto md_dict = encryptor->GetKeyValueMetadata(parquet::encryption::kDictionaryPage); + auto md_data = encryptor->GetKeyValueMetadata(parquet::encryption::kDataPage); + ASSERT_EQ(md_dict, nullptr); + ASSERT_EQ(md_data, nullptr); +} + +TEST_F(ExternalDBPAEncryptorAdapterTest, SignedFooterEncryptionThrowsException) { + ParquetCipher::type algorithm = ParquetCipher::EXTERNAL_DBPA_V1; + std::string column_name = "employee_name"; + std::string key_id = "employee_name_key"; + Type::type data_type = Type::BYTE_ARRAY; + Compression::type compression_type = Compression::UNCOMPRESSED; + Encoding::type encoding_type = Encoding::PLAIN; + std::unique_ptr encryptor = CreateEncryptor( + algorithm, column_name, key_id, data_type, compression_type, encoding_type); + std::vector encrypted_footer(10, '\0'); + EXPECT_THROW(encryptor->SignedFooterEncrypt( + str2span(/*footer*/""), str2span(/*key*/""), str2span(/*aad*/""), str2span(/*nonce*/""), + encrypted_footer), ParquetException); +} + +TEST_F(ExternalDBPAEncryptorAdapterTest, EncryptWithoutUpdateEncodingPropertiesThrows) { + ParquetCipher::type algorithm = ParquetCipher::EXTERNAL_DBPA_V1; + std::string column_name = "employee_name"; + std::string key_id = "employee_name_key"; + Type::type data_type = Type::BYTE_ARRAY; + Compression::type compression_type = Compression::UNCOMPRESSED; + Encoding::type encoding_type = Encoding::PLAIN; + + auto encryptor = ExternalDBPAEncryptorAdapter::Make( + algorithm, column_name, key_id, data_type, compression_type, encoding_type, + app_context_, connection_config_, std::nullopt); + + std::string plaintext = "abc"; + std::shared_ptr ciphertext_buffer = AllocateBuffer( + ::arrow::default_memory_pool(), 0); + EXPECT_THROW( + encryptor->EncryptWithManagedBuffer( + str2span(plaintext), ciphertext_buffer.get()), + ParquetException); +} + +TEST_F(ExternalDBPAEncryptorAdapterTest, DecryptWithoutUpdateEncodingPropertiesThrows) { + ParquetCipher::type algorithm = ParquetCipher::EXTERNAL_DBPA_V1; + std::string column_name = "employee_name"; + std::string key_id = "employee_name_key"; + Type::type data_type = Type::BYTE_ARRAY; + Compression::type compression_type = Compression::UNCOMPRESSED; + Encoding::type encoding_type = Encoding::PLAIN; + + auto decryptor = ExternalDBPADecryptorAdapter::Make( + algorithm, column_name, key_id, data_type, compression_type, {encoding_type}, + app_context_, connection_config_, std::nullopt, key_value_metadata_); + + std::string ciphertext = "xyz"; + std::shared_ptr plaintext_buffer = AllocateBuffer( + ::arrow::default_memory_pool(), 0); + EXPECT_THROW( + decryptor->DecryptWithManagedBuffer( + str2span(ciphertext), plaintext_buffer.get()), + ParquetException); +} + +TEST_F(ExternalDBPAEncryptorAdapterTest, EncryptorUnsupportedAlgorithmThrows) { + // Use AES_GCM_V1 (unsupported) to verify the adapter rejects algorithms other + // than EXTERNAL_DBPA_V1 + ParquetCipher::type unsupported_algo = ParquetCipher::AES_GCM_V1; + std::string column_name = "employee_name"; + std::string key_id = "employee_name_key"; + Type::type data_type = Type::BYTE_ARRAY; + Compression::type compression_type = Compression::UNCOMPRESSED; + Encoding::type encoding_type = Encoding::PLAIN; + + EXPECT_THROW( + CreateEncryptor( + unsupported_algo, column_name, key_id, data_type, compression_type, encoding_type), + ParquetException); + + // Also test AES_GCM_CTR_V1 + unsupported_algo = ParquetCipher::AES_GCM_CTR_V1; + EXPECT_THROW( + CreateEncryptor( + unsupported_algo, column_name, key_id, data_type, compression_type, encoding_type), + ParquetException); +} + +TEST_F(ExternalDBPAEncryptorAdapterTest, DecryptorUnsupportedAlgorithmThrows) { + // Use AES_GCM_V1 (unsupported) to verify the adapter rejects algorithms other + // than EXTERNAL_DBPA_V1 + ParquetCipher::type unsupported_algo = ParquetCipher::AES_GCM_V1; + std::string column_name = "employee_name"; + std::string key_id = "employee_name_key"; + Type::type data_type = Type::BYTE_ARRAY; + Compression::type compression_type = Compression::UNCOMPRESSED; + Encoding::type encoding_type = Encoding::PLAIN; + + EXPECT_THROW( + ExternalDBPADecryptorAdapter::Make( + unsupported_algo, column_name, key_id, data_type, compression_type, {encoding_type}, + app_context_, connection_config_, std::nullopt, key_value_metadata_), + ParquetException); + + // Also test AES_GCM_CTR_V1 + unsupported_algo = ParquetCipher::AES_GCM_CTR_V1; + EXPECT_THROW( + ExternalDBPADecryptorAdapter::Make( + unsupported_algo, column_name, key_id, data_type, compression_type, {encoding_type}, + app_context_, connection_config_, std::nullopt, key_value_metadata_), + std::exception); +} + +TEST_F(ExternalDBPAEncryptorAdapterTest, EncryptorMissingLibraryPathThrows) { + ParquetCipher::type algorithm = ParquetCipher::EXTERNAL_DBPA_V1; + std::string column_name = "employee_name"; + std::string key_id = "employee_name_key"; + Type::type data_type = Type::BYTE_ARRAY; + Compression::type compression_type = Compression::UNCOMPRESSED; + Encoding::type encoding_type = Encoding::PLAIN; + + std::map bad_config = { {"config_path", "path/to/file"} }; + std::string app_context = "{}"; + + EXPECT_THROW( + ExternalDBPAEncryptorAdapter::Make( + algorithm, column_name, key_id, data_type, compression_type, encoding_type, + app_context, bad_config, std::nullopt), + std::exception); +} + +TEST_F(ExternalDBPAEncryptorAdapterTest, EncryptorInvalidLibraryPathThrows) { + ParquetCipher::type algorithm = ParquetCipher::EXTERNAL_DBPA_V1; + std::string column_name = "employee_name"; + std::string key_id = "employee_name_key"; + Type::type data_type = Type::BYTE_ARRAY; + Compression::type compression_type = Compression::UNCOMPRESSED; + Encoding::type encoding_type = Encoding::PLAIN; + + std::map bad_config = { + {"agent_library_path", "/definitely/not/a/real/libDBPA.so"} + }; + std::string app_context = "{}"; + + EXPECT_THROW( + ExternalDBPAEncryptorAdapter::Make( + algorithm, column_name, key_id, data_type, compression_type, encoding_type, + app_context, bad_config, std::nullopt), + std::exception); +} + +TEST_F(ExternalDBPAEncryptorAdapterTest, EncryptorInvalidTimeoutValuesThrows) { + ParquetCipher::type algorithm = ParquetCipher::EXTERNAL_DBPA_V1; + std::string column_name = "employee_name"; + std::string key_id = "employee_name_key"; + Type::type data_type = Type::BYTE_ARRAY; + Compression::type compression_type = Compression::UNCOMPRESSED; + Encoding::type encoding_type = Encoding::PLAIN; + + std::string library_path = parquet::encryption::external::test::TestUtils::GetTestLibraryPath(); + std::map bad_config = { + {"config_path", "path/to/file"}, + {"agent_library_path", library_path}, + {"agent_init_timeout_ms", "nope"}, + }; + std::string app_context = "{}"; + + EXPECT_THROW( + ExternalDBPAEncryptorAdapter::Make( + algorithm, column_name, key_id, data_type, compression_type, encoding_type, + app_context, bad_config, std::nullopt), + std::exception); +} + +TEST_F(ExternalDBPAEncryptorAdapterTest, DecryptorMissingLibraryPathThrows) { + ParquetCipher::type algorithm = ParquetCipher::EXTERNAL_DBPA_V1; + std::string column_name = "employee_name"; + std::string key_id = "employee_name_key"; + Type::type data_type = Type::BYTE_ARRAY; + Compression::type compression_type = Compression::UNCOMPRESSED; + Encoding::type encoding_type = Encoding::PLAIN; + + std::map bad_config = { {"config_path", "path/to/file"} }; + std::string app_context = "{}"; + + EXPECT_THROW( + ExternalDBPADecryptorAdapter::Make( + algorithm, column_name, key_id, data_type, compression_type, {encoding_type}, + app_context, bad_config, std::nullopt, key_value_metadata_), + std::exception); +} + +TEST_F(ExternalDBPAEncryptorAdapterTest, DecryptorInvalidLibraryPathThrows) { + ParquetCipher::type algorithm = ParquetCipher::EXTERNAL_DBPA_V1; + std::string column_name = "employee_name"; + std::string key_id = "employee_name_key"; + Type::type data_type = Type::BYTE_ARRAY; + Compression::type compression_type = Compression::UNCOMPRESSED; + Encoding::type encoding_type = Encoding::PLAIN; + + std::map bad_config = { + {"agent_library_path", "/definitely/not/a/real/libDBPA.so"} + }; + std::string app_context = "{}"; + + EXPECT_THROW( + ExternalDBPADecryptorAdapter::Make( + algorithm, column_name, key_id, data_type, compression_type, {encoding_type}, + app_context, bad_config, std::nullopt, key_value_metadata_), + std::exception); +} + +TEST_F(ExternalDBPAEncryptorAdapterTest, DecryptorInvalidTimeoutValuesThrows) { + ParquetCipher::type algorithm = ParquetCipher::EXTERNAL_DBPA_V1; + std::string column_name = "employee_name"; + std::string key_id = "employee_name_key"; + Type::type data_type = Type::BYTE_ARRAY; + Compression::type compression_type = Compression::UNCOMPRESSED; + Encoding::type encoding_type = Encoding::PLAIN; + + std::string library_path = parquet::encryption::external::test::TestUtils::GetTestLibraryPath(); + std::map bad_config = { + {"config_path", "path/to/file"}, + {"agent_library_path", library_path}, + {"agent_init_timeout_ms", "nope"}, + }; + std::string app_context = "{}"; + + EXPECT_THROW( + ExternalDBPADecryptorAdapter::Make( + algorithm, column_name, key_id, data_type, compression_type, {encoding_type}, + app_context, bad_config, std::nullopt, key_value_metadata_), + std::exception); +} + +// Helper stub EncryptionResult for testing metadata accumulation +class StubEncryptionResult : public dbps::external::EncryptionResult { +public: + explicit StubEncryptionResult(std::map metadata) + : metadata_(std::move(metadata)) {} + + span ciphertext() const override { return {}; } + std::size_t size() const override { return 0; } + bool success() const override { return true; } + const std::optional> encryption_metadata() const override { + return metadata_; + } + const std::string& error_message() const override { return empty_; } + const std::map& error_fields() const override { return empty_fields_; } + +private: + std::optional> metadata_; + mutable std::string empty_; + mutable std::map empty_fields_; +}; + +TEST_F(ExternalDBPAEncryptorAdapterTest, UpdateEncryptorMetadataAccumulatesByModuleType) { + // Build EncodingProperties for dictionary page and data page V2 + EncodingPropertiesBuilder dict_builder; + dict_builder + .ColumnPath("col") + .PhysicalType(Type::BYTE_ARRAY) + .CompressionCodec(Compression::UNCOMPRESSED) + .PageType(parquet::PageType::DICTIONARY_PAGE) + .PageEncoding(Encoding::PLAIN); + auto dict_props = dict_builder.Build(); + + EncodingPropertiesBuilder data_builder; + data_builder + .ColumnPath("col") + .PhysicalType(Type::BYTE_ARRAY) + .CompressionCodec(Compression::UNCOMPRESSED) + .PageType(parquet::PageType::DATA_PAGE_V2) + .PageEncoding(Encoding::PLAIN) + .DataPageNumValues(100) + .DataPageMaxDefinitionLevel(1) + .DataPageMaxRepetitionLevel(0) + .PageV2DefinitionLevelsByteLength(4) + .PageV2RepetitionLevelsByteLength(4) + .PageV2NumNulls(0) + .PageV2IsCompressed(true); + auto data_props = data_builder.Build(); + + // Prepare result metadata + std::map meta1 = {{"m1", "v1"}, {"m2", "v2"}}; + std::map meta2 = {{"m2", "v2_override"}, {"m3", "v3"}}; + + StubEncryptionResult r1(meta1); + StubEncryptionResult r2(meta2); + + std::map> accum; + + // Call helper under test + UpdateEncryptorMetadata(accum, *dict_props, r1); + UpdateEncryptorMetadata(accum, *data_props, r2); + + // Verify dictionary page accumulation + ASSERT_NE(accum.find(parquet::encryption::kDictionaryPage), accum.end()); + const auto& dict_map = accum.at(parquet::encryption::kDictionaryPage); + ASSERT_EQ(dict_map.at("m1"), "v1"); + ASSERT_EQ(dict_map.at("m2"), "v2"); + + // Verify data page accumulation with overwrite behavior + ASSERT_NE(accum.find(parquet::encryption::kDataPage), accum.end()); + const auto& data_map = accum.at(parquet::encryption::kDataPage); + ASSERT_EQ(data_map.at("m2"), "v2_override"); + ASSERT_EQ(data_map.at("m3"), "v3"); +} + +TEST_F(ExternalDBPAEncryptorAdapterTest, DecryptWithWrongKeyIdFails) { + ParquetCipher::type algorithm = ParquetCipher::EXTERNAL_DBPA_V1; + std::string column_name = "employee_name"; + std::string correct_key_id = "employee_name_key"; + std::string wrong_key_id = "wrong_key_id"; + Type::type data_type = Type::BYTE_ARRAY; + Compression::type compression_type = Compression::UNCOMPRESSED; + Encoding::type encoding_type = Encoding::PLAIN; + std::string app_context = "{}"; + std::map config = { + {"agent_library_path", parquet::encryption::external::test::TestUtils::GetTestLibraryPath()} + }; + + auto encryptor = ExternalDBPAEncryptorAdapter::Make( + algorithm, column_name, correct_key_id, data_type, compression_type, encoding_type, + app_context, config, std::nullopt); + + // Build encoding properties + EncodingPropertiesBuilder builder; + builder.ColumnPath("test_column") + .PhysicalType(data_type) + .CompressionCodec(compression_type) + .PageType(parquet::PageType::DATA_PAGE_V2) + .PageV2DefinitionLevelsByteLength(10) + .PageV2RepetitionLevelsByteLength(10) + .PageV2NumNulls(10) + .PageV2IsCompressed(true) + .DataPageMaxDefinitionLevel(10) + .DataPageMaxRepetitionLevel(1) + .PageEncoding(encoding_type) + .DataPageNumValues(100) + .Build(); + + encryptor->UpdateEncodingProperties(builder.Build()); + + std::string plaintext = "Sensitive Data"; + std::shared_ptr ciphertext_buffer = AllocateBuffer( + ::arrow::default_memory_pool(), 0); + + std::string empty; + int32_t enc_len = encryptor->EncryptWithManagedBuffer( + str2span(plaintext), ciphertext_buffer.get()); + + std::string ciphertext_str(ciphertext_buffer->data(), ciphertext_buffer->data() + enc_len); + + auto decryptor = ExternalDBPADecryptorAdapter::Make( + algorithm, column_name, wrong_key_id, data_type, compression_type, {encoding_type}, + app_context, config, std::nullopt, key_value_metadata_); + + decryptor->UpdateEncodingProperties(builder.Build()); + + std::shared_ptr plaintext_buffer = AllocateBuffer( + ::arrow::default_memory_pool(), 0); + + bool threw = false; + int32_t dec_len = 0; + try { + dec_len = decryptor->DecryptWithManagedBuffer( + str2span(ciphertext_str), plaintext_buffer.get()); + } catch (const ParquetException&) { + threw = true; + } + + if (!threw) { + std::string decrypted(plaintext_buffer->data(), plaintext_buffer->data() + dec_len); + ASSERT_NE(plaintext, decrypted); + } +} + +TEST_F(ExternalDBPAEncryptorAdapterTest, EncryptCallShouldFail) { + ParquetCipher::type algorithm = ParquetCipher::EXTERNAL_DBPA_V1; + std::string column_name = "employee_name"; + std::string key_id = "employee_name_key"; + Type::type data_type = Type::BYTE_ARRAY; + Compression::type compression_type = Compression::UNCOMPRESSED; + Encoding::type encoding_type = Encoding::PLAIN; + std::string plaintext = "Jean-Luc Picard"; + std::vector ciphertext_buffer(plaintext.size(), '\0'); + + std::unique_ptr encryptor = CreateEncryptor( + algorithm, column_name, key_id, data_type, compression_type, encoding_type); + ASSERT_FALSE(encryptor->CanCalculateCiphertextLength()); + ASSERT_LE(plaintext.size(), + static_cast(std::numeric_limits::max())); + EXPECT_THROW( + (void) encryptor->CiphertextLength(static_cast(plaintext.size())), + ParquetException); + EXPECT_THROW( + encryptor->Encrypt( + str2span(plaintext), str2span(/*key*/""), str2span(/*aad*/""), ciphertext_buffer), + ParquetException); +} + +TEST_F(ExternalDBPAEncryptorAdapterTest, DecryptCallShouldFail) { + ParquetCipher::type algorithm = ParquetCipher::EXTERNAL_DBPA_V1; + std::string column_name = "employee_name"; + std::string key_id = "employee_name_key"; + Type::type data_type = Type::BYTE_ARRAY; + Compression::type compression_type = Compression::UNCOMPRESSED; + Encoding::type encoding_type = Encoding::PLAIN; + std::string ciphertext = "Jean-Luc Picard"; + std::vector plaintext_buffer(ciphertext.size(), '\0'); + + std::unique_ptr decryptor = CreateDecryptor( + algorithm, column_name, key_id, data_type, compression_type, encoding_type); + ASSERT_FALSE(decryptor->CanCalculateLengths()); + ASSERT_LE(ciphertext.size(), + static_cast(std::numeric_limits::max())); + EXPECT_THROW( + (void) decryptor->CiphertextLength(static_cast(ciphertext.size())), + ParquetException); + EXPECT_THROW( + (void) decryptor->PlaintextLength(static_cast(ciphertext.size())), + ParquetException); + EXPECT_THROW( + decryptor->Decrypt( + str2span(ciphertext), str2span(/*key*/""), str2span(/*aad*/""), plaintext_buffer), + ParquetException); +} + +TEST_F(ExternalDBPAEncryptorAdapterTest, KeyValueMetadataToStringMap_Nullptr) { + std::shared_ptr md = nullptr; + auto result = ExternalDBPAUtils::KeyValueMetadataToStringMap(md); + ASSERT_FALSE(result.has_value()); +} + +TEST_F(ExternalDBPAEncryptorAdapterTest, KeyValueMetadataToStringMap_Empty) { + auto md = ::arrow::key_value_metadata(std::vector{}, std::vector{}); + auto result = ExternalDBPAUtils::KeyValueMetadataToStringMap(md); + ASSERT_FALSE(result.has_value()); +} + +TEST_F(ExternalDBPAEncryptorAdapterTest, KeyValueMetadataToStringMap_Basic) { + auto md = KeyValueMetadata::Make({"k1", "k2"}, {"v1", "v2"}); + auto result = ExternalDBPAUtils::KeyValueMetadataToStringMap(md); + ASSERT_TRUE(result.has_value()); + const auto& m = result.value(); + ASSERT_EQ(m.size(), 2u); + ASSERT_EQ(m.at("k1"), std::string("v1")); + ASSERT_EQ(m.at("k2"), std::string("v2")); +} + +TEST_F(ExternalDBPAEncryptorAdapterTest, KeyValueMetadataToStringMap_MismatchedLengths) { + auto md_more_keys = KeyValueMetadata::Make({"a", "b", "c"}, {"1", "2"}); + auto res1 = ExternalDBPAUtils::KeyValueMetadataToStringMap(md_more_keys); + ASSERT_TRUE(res1.has_value()); + const auto& m1 = res1.value(); + ASSERT_EQ(m1.size(), 2u); + ASSERT_EQ(m1.at("a"), std::string("1")); + ASSERT_EQ(m1.at("b"), std::string("2")); + ASSERT_TRUE(m1.find("c") == m1.end()); + + auto md_more_vals = KeyValueMetadata::Make({"only"}, {"v1", "v2"}); + auto res2 = ExternalDBPAUtils::KeyValueMetadataToStringMap(md_more_vals); + ASSERT_TRUE(res2.has_value()); + const auto& m2 = res2.value(); + ASSERT_EQ(m2.size(), 1u); + ASSERT_EQ(m2.at("only"), std::string("v1")); +} + +} // namespace parquet::encryption::test diff --git a/cpp/src/parquet/encryption/external_dbpa_encryption_utils.h b/cpp/src/parquet/encryption/external_dbpa_encryption_utils.h new file mode 100644 index 000000000000..eb7fd0eb5fe8 --- /dev/null +++ b/cpp/src/parquet/encryption/external_dbpa_encryption_utils.h @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "arrow/util/logging.h" + +namespace parquet::encryption { + +// Configure Arrow logging threshold from environment (once per procall site). +// +// There is currently no mechanism to configure Arrow's logger. +// Given that most of the logging changes that we have added are related to external/dbpa encryption, +// we decided to keep most of the Arrow base code as-is, and made the config belong to external/dbpa. +// +// Env var: PARQUET_DBPA_LOG_LEVEL (case-insensitive: TRACE, DEBUG, INFO, WARN[ING], ERROR, FATAL, or -2..3) +inline void ConfigureArrowLogLevel() { + const char* env_val = std::getenv("PARQUET_DBPA_LOG_LEVEL"); + if (env_val == nullptr || *env_val == '\0') { + return; + } + std::string s(env_val); + std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return static_cast(std::toupper(c)); }); + + using ::arrow::util::ArrowLogLevel; + auto to_level = [&](const std::string& v) -> std::optional { + if (v == "TRACE") return ArrowLogLevel::ARROW_TRACE; + if (v == "DEBUG") return ArrowLogLevel::ARROW_DEBUG; + if (v == "INFO") return ArrowLogLevel::ARROW_INFO; + if (v == "WARN" || v == "WARNING") return ArrowLogLevel::ARROW_WARNING; + if (v == "ERROR") return ArrowLogLevel::ARROW_ERROR; + if (v == "FATAL") return ArrowLogLevel::ARROW_FATAL; + try { + int n = std::stoi(v); + switch (n) { + case -2: return ArrowLogLevel::ARROW_TRACE; + case -1: return ArrowLogLevel::ARROW_DEBUG; + case 0: return ArrowLogLevel::ARROW_INFO; + case 1: return ArrowLogLevel::ARROW_WARNING; + case 2: return ArrowLogLevel::ARROW_ERROR; + case 3: return ArrowLogLevel::ARROW_FATAL; + default: return std::nullopt; + } + } catch (...) { + return std::nullopt; + } + }; + + if (auto lvl = to_level(s)) { + ::arrow::util::ArrowLog::StartArrowLog("parquet-dbpa", *lvl, ""); + } +} + +inline void EnsureDbpaLoggingConfigured() { + static std::once_flag once; + std::call_once(once, [] { ConfigureArrowLogLevel(); }); +} + +} // namespace parquet::encryption diff --git a/cpp/src/parquet/encryption/file_key_wrapper.cc b/cpp/src/parquet/encryption/file_key_wrapper.cc index fd870ed1f3bb..e90c52569262 100644 --- a/cpp/src/parquet/encryption/file_key_wrapper.cc +++ b/cpp/src/parquet/encryption/file_key_wrapper.cc @@ -16,7 +16,7 @@ // under the License. #include "parquet/encryption/file_key_wrapper.h" -#include "parquet/encryption/encryption_internal.h" +#include "parquet/encryption/encryption_utils.h" #include "parquet/encryption/key_material.h" #include "parquet/encryption/key_metadata.h" #include "parquet/encryption/key_toolkit_internal.h" diff --git a/cpp/src/parquet/encryption/internal_file_decryptor.cc b/cpp/src/parquet/encryption/internal_file_decryptor.cc index b90d31585597..5b7cc3d0173a 100644 --- a/cpp/src/parquet/encryption/internal_file_decryptor.cc +++ b/cpp/src/parquet/encryption/internal_file_decryptor.cc @@ -19,8 +19,9 @@ #include "arrow/util/logging.h" #include "arrow/util/secure_string.h" +#include "parquet/encryption/aes_encryption.h" #include "parquet/encryption/encryption.h" -#include "parquet/encryption/encryption_internal.h" +#include "parquet/encryption/encryption_utils.h" #include "parquet/metadata.h" using arrow::util::SecureString; @@ -28,10 +29,10 @@ using arrow::util::SecureString; namespace parquet { // Decryptor -Decryptor::Decryptor(std::unique_ptr aes_decryptor, +Decryptor::Decryptor(std::unique_ptr decryptor_instance, SecureString key, std::string file_aad, std::string aad, ::arrow::MemoryPool* pool) - : aes_decryptor_(std::move(aes_decryptor)), + : decryptor_instance_(std::move(decryptor_instance)), key_(std::move(key)), file_aad_(std::move(file_aad)), aad_(std::move(aad)), @@ -39,17 +40,30 @@ Decryptor::Decryptor(std::unique_ptr aes_decryptor, Decryptor::~Decryptor() = default; +bool Decryptor::CanCalculateLengths() const { + return decryptor_instance_->CanCalculateLengths(); +} + int32_t Decryptor::PlaintextLength(int32_t ciphertext_len) const { - return aes_decryptor_->PlaintextLength(ciphertext_len); + return decryptor_instance_->PlaintextLength(ciphertext_len); } int32_t Decryptor::CiphertextLength(int32_t plaintext_len) const { - return aes_decryptor_->CiphertextLength(plaintext_len); + return decryptor_instance_->CiphertextLength(plaintext_len); +} + +void Decryptor::UpdateEncodingProperties(std::unique_ptr encoding_properties) { + decryptor_instance_->UpdateEncodingProperties(std::move(encoding_properties)); } int32_t Decryptor::Decrypt(::arrow::util::span ciphertext, ::arrow::util::span plaintext) { - return aes_decryptor_->Decrypt(ciphertext, key_.as_span(), str2span(aad_), plaintext); + return decryptor_instance_->Decrypt(ciphertext, key_.as_span(), str2span(aad_), plaintext); +} + +int32_t Decryptor::DecryptWithManagedBuffer(::arrow::util::span ciphertext, + ::arrow::ResizableBuffer* plaintext) { +return decryptor_instance_->DecryptWithManagedBuffer(ciphertext, plaintext); } // InternalFileDecryptor @@ -104,8 +118,9 @@ std::unique_ptr InternalFileDecryptor::GetFooterDecryptor( const SecureString& footer_key = GetFooterKey(); auto key_len = static_cast(footer_key.size()); - auto aes_decryptor = encryption::AesDecryptor::Make(algorithm_, key_len, metadata); - return std::make_unique(std::move(aes_decryptor), footer_key, file_aad_, aad, + // Metadata is decrypted with AES. + auto decryptor_instance = encryption::AesDecryptor::Make(algorithm_, key_len, metadata); + return std::make_unique(std::move(decryptor_instance), footer_key, file_aad_, aad, pool_); } @@ -130,19 +145,20 @@ SecureString InternalFileDecryptor::GetColumnKey(const std::string& column_path, return column_key; } -std::unique_ptr InternalFileDecryptor::GetColumnDecryptor( +std::unique_ptr InternalFileDecryptor::GetColumnMetaDecryptor( const std::string& column_path, const std::string& column_key_metadata, - const std::string& aad, bool metadata) { + const std::string& aad) { const SecureString& column_key = GetColumnKey(column_path, column_key_metadata); auto key_len = static_cast(column_key.size()); - auto aes_decryptor = encryption::AesDecryptor::Make(algorithm_, key_len, metadata); - return std::make_unique(std::move(aes_decryptor), column_key, file_aad_, aad, + auto decryptor_instance = encryption::AesDecryptor::Make(algorithm_, key_len, /*metadata=*/true); + return std::make_unique(std::move(decryptor_instance), column_key, file_aad_, aad, pool_); } std::function()> InternalFileDecryptor::GetColumnDecryptorFactory( - const ColumnCryptoMetaData* crypto_metadata, const std::string& aad, bool metadata) { + const ColumnCryptoMetaData* crypto_metadata, const std::string& aad, bool metadata, + const ColumnChunkMetaData* column_chunk_metadata) { if (crypto_metadata->encrypted_with_footer_key()) { return [this, aad, metadata]() { return GetFooterDecryptor(aad, metadata); }; } @@ -152,42 +168,60 @@ InternalFileDecryptor::GetColumnDecryptorFactory( const std::string column_path = crypto_metadata->path_in_schema()->ToDotString(); const SecureString& column_key = GetColumnKey(column_path, column_key_metadata); - return [this, aad, metadata, column_key = column_key]() { - auto key_len = static_cast(column_key.size()); - auto aes_decryptor = encryption::AesDecryptor::Make(algorithm_, key_len, metadata); - return std::make_unique(std::move(aes_decryptor), column_key, file_aad_, - aad, pool_); - }; -} + // If this is data decryption, check if the column is encrypted with its own algorithm. + ParquetCipher::type algorithm = algorithm_; + if (!metadata &&crypto_metadata->is_encryption_algorithm_set()) { + algorithm = crypto_metadata->encryption_algorithm().algorithm; + } + + return [this, aad, metadata, column_key = std::move(column_key), algorithm, + crypto_metadata, column_chunk_metadata]() { + auto key_len = static_cast(column_key.size()); + std::unique_ptr decryptor_instance; + + if (algorithm == ParquetCipher::EXTERNAL_DBPA_V1) { + if (dynamic_cast(properties_.get()) == nullptr) { + throw ParquetException( + "External DBPA decryption requires ExternalFileDecryptionProperties"); + } + decryptor_instance = external_dbpa_decryptor_factory_.GetDecryptor( + algorithm, crypto_metadata, column_chunk_metadata, + dynamic_cast(properties_.get())); + } else { + decryptor_instance = encryption::AesDecryptor::Make(algorithm, key_len, metadata); + } + return std::make_unique(std::move(decryptor_instance), column_key, file_aad_, + aad, pool_); + }; + } std::function()> InternalFileDecryptor::GetColumnMetaDecryptorFactory( - InternalFileDecryptor* file_descryptor, const ColumnCryptoMetaData* crypto_metadata, + InternalFileDecryptor* file_decryptor, const ColumnCryptoMetaData* crypto_metadata, const std::string& aad) { if (crypto_metadata == nullptr) { // Column is not encrypted return [] { return nullptr; }; } - if (file_descryptor == nullptr) { + if (file_decryptor == nullptr) { throw ParquetException("Column is noted as encrypted but no file decryptor"); } - return file_descryptor->GetColumnDecryptorFactory(crypto_metadata, aad, - /*metadata=*/true); + return file_decryptor->GetColumnDecryptorFactory(crypto_metadata, aad, /*metadata=*/true); } std::function()> InternalFileDecryptor::GetColumnDataDecryptorFactory( - InternalFileDecryptor* file_descryptor, const ColumnCryptoMetaData* crypto_metadata, - const std::string& aad) { + InternalFileDecryptor* file_decryptor, const ColumnCryptoMetaData* crypto_metadata, + const ColumnChunkMetaData* column_chunk_metadata, const std::string& aad) { if (crypto_metadata == nullptr) { // Column is not encrypted return [] { return nullptr; }; } - if (file_descryptor == nullptr) { + if (file_decryptor == nullptr) { throw ParquetException("Column is noted as encrypted but no file decryptor"); } - return file_descryptor->GetColumnDecryptorFactory(crypto_metadata, aad, - /*metadata=*/false); + return file_decryptor->GetColumnDecryptorFactory(crypto_metadata, aad, + /*metadata=*/false, column_chunk_metadata); } void UpdateDecryptor(Decryptor* decryptor, int16_t row_group_ordinal, diff --git a/cpp/src/parquet/encryption/internal_file_decryptor.h b/cpp/src/parquet/encryption/internal_file_decryptor.h index a365b4df4bf9..b83109a6e933 100644 --- a/cpp/src/parquet/encryption/internal_file_decryptor.h +++ b/cpp/src/parquet/encryption/internal_file_decryptor.h @@ -20,28 +20,26 @@ #include #include #include -#include #include "arrow/util/secure_string.h" -#include "parquet/schema.h" +#include "parquet/metadata.h" +#include "parquet/encryption/decryptor_interface.h" +#include "parquet/encryption/external_dbpa_encryption.h" namespace parquet { -namespace encryption { -class AesDecryptor; -class AesEncryptor; -} // namespace encryption - class ColumnCryptoMetaData; class DecryptionKeyRetriever; class FileDecryptionProperties; +using parquet::encryption::EncodingProperties; + // An object handling decryption using well-known encryption parameters // // CAUTION: Decryptor objects are not thread-safe. class PARQUET_EXPORT Decryptor { public: - Decryptor(std::unique_ptr decryptor, + Decryptor(std::unique_ptr decryptor, ::arrow::util::SecureString key, std::string file_aad, std::string aad, ::arrow::MemoryPool* pool); ~Decryptor(); @@ -50,13 +48,18 @@ class PARQUET_EXPORT Decryptor { void UpdateAad(const std::string& aad) { aad_ = aad; } ::arrow::MemoryPool* pool() { return pool_; } + [[nodiscard]] bool CanCalculateLengths() const; [[nodiscard]] int32_t PlaintextLength(int32_t ciphertext_len) const; [[nodiscard]] int32_t CiphertextLength(int32_t plaintext_len) const; int32_t Decrypt(::arrow::util::span ciphertext, ::arrow::util::span plaintext); + int32_t DecryptWithManagedBuffer(::arrow::util::span ciphertext, + ::arrow::ResizableBuffer* plaintext); + + void UpdateEncodingProperties(std::unique_ptr encoding_properties); private: - std::unique_ptr aes_decryptor_; + std::unique_ptr decryptor_instance_; ::arrow::util::SecureString key_; std::string file_aad_; std::string aad_; @@ -91,16 +94,7 @@ class InternalFileDecryptor { // Get a Decryptor instance for column chunk metadata. std::unique_ptr GetColumnMetaDecryptor( const std::string& column_path, const std::string& column_key_metadata, - const std::string& aad = "") { - return GetColumnDecryptor(column_path, column_key_metadata, aad, /*metadata=*/true); - } - - // Get a Decryptor instance for column chunk data. - std::unique_ptr GetColumnDataDecryptor( - const std::string& column_path, const std::string& column_key_metadata, - const std::string& aad = "") { - return GetColumnDecryptor(column_path, column_key_metadata, aad, /*metadata=*/false); - } + const std::string& aad = ""); // Get a Decryptor factory for column chunk metadata. // @@ -108,15 +102,17 @@ class InternalFileDecryptor { // This is a static function as it accepts a null `InternalFileDecryptor*` // argument if the column is not encrypted. static std::function()> GetColumnMetaDecryptorFactory( - InternalFileDecryptor*, const ColumnCryptoMetaData* crypto_metadata, + InternalFileDecryptor* file_decryptor, const ColumnCryptoMetaData* crypto_metadata, const std::string& aad = ""); + // Get a Decryptor factory for column chunk data. // // This is typically useful if multi-threaded decryption is expected. // This is a static function as it accepts a null `InternalFileDecryptor*` // argument if the column is not encrypted. static std::function()> GetColumnDataDecryptorFactory( - InternalFileDecryptor*, const ColumnCryptoMetaData* crypto_metadata, + InternalFileDecryptor* file_decryptor, const ColumnCryptoMetaData* crypto_metadata, + const ColumnChunkMetaData* column_chunk_metadata = nullptr, const std::string& aad = ""); private: @@ -126,6 +122,7 @@ class InternalFileDecryptor { ParquetCipher::type algorithm_; std::string footer_key_metadata_; ::arrow::MemoryPool* pool_; + encryption::ExternalDBPADecryptorAdapterFactory external_dbpa_decryptor_factory_; // Protects footer_key_ updates std::mutex mutex_; @@ -140,12 +137,9 @@ class InternalFileDecryptor { std::unique_ptr GetFooterDecryptor(const std::string& aad, bool metadata); - std::unique_ptr GetColumnDecryptor(const std::string& column_path, - const std::string& column_key_metadata, - const std::string& aad, bool metadata); - std::function()> GetColumnDecryptorFactory( - const ColumnCryptoMetaData* crypto_metadata, const std::string& aad, bool metadata); + const ColumnCryptoMetaData* crypto_metadata, const std::string& aad, bool metadata, + const ColumnChunkMetaData* column_chunk_metadata = nullptr); }; void UpdateDecryptor(Decryptor* decryptor, int16_t row_group_ordinal, diff --git a/cpp/src/parquet/encryption/internal_file_encryptor.cc b/cpp/src/parquet/encryption/internal_file_encryptor.cc index 3623aa05c662..fa3ae7d2ab89 100644 --- a/cpp/src/parquet/encryption/internal_file_encryptor.cc +++ b/cpp/src/parquet/encryption/internal_file_encryptor.cc @@ -16,30 +16,49 @@ // under the License. #include "parquet/encryption/internal_file_encryptor.h" + +#include "parquet/encryption/aes_encryption.h" #include "arrow/util/secure_string.h" #include "parquet/encryption/encryption.h" -#include "parquet/encryption/encryption_internal.h" +#include "parquet/encryption/encryption_utils.h" using arrow::util::SecureString; namespace parquet { // Encryptor -Encryptor::Encryptor(encryption::AesEncryptor* aes_encryptor, SecureString key, +Encryptor::Encryptor(encryption::EncryptorInterface* encryptor_instance, SecureString key, std::string file_aad, std::string aad, ::arrow::MemoryPool* pool) - : aes_encryptor_(aes_encryptor), + : encryptor_instance_(encryptor_instance), key_(std::move(key)), file_aad_(std::move(file_aad)), aad_(std::move(aad)), pool_(pool) {} int32_t Encryptor::CiphertextLength(int64_t plaintext_len) const { - return aes_encryptor_->CiphertextLength(plaintext_len); + return encryptor_instance_->CiphertextLength(plaintext_len); +} + +bool Encryptor::CanCalculateCiphertextLength() const { + return encryptor_instance_->CanCalculateCiphertextLength(); } int32_t Encryptor::Encrypt(::arrow::util::span plaintext, ::arrow::util::span ciphertext) { - return aes_encryptor_->Encrypt(plaintext, key_.as_span(), str2span(aad_), ciphertext); + return encryptor_instance_->Encrypt(plaintext, key_.as_span(), str2span(aad_), ciphertext); +} + +int32_t Encryptor::EncryptWithManagedBuffer(::arrow::util::span plaintext, + ::arrow::ResizableBuffer* ciphertext) { + return encryptor_instance_->EncryptWithManagedBuffer(plaintext, ciphertext); +} + +void Encryptor::UpdateEncodingProperties(std::unique_ptr encoding_properties) { + encryptor_instance_->UpdateEncodingProperties(std::move(encoding_properties)); +} + +std::shared_ptr Encryptor::GetKeyValueMetadata(int8_t module_type) { + return encryptor_instance_->GetKeyValueMetadata(module_type); } // InternalFileEncryptor @@ -55,9 +74,9 @@ std::shared_ptr InternalFileEncryptor::GetFooterEncryptor() { ParquetCipher::type algorithm = properties_->algorithm().algorithm; std::string footer_aad = encryption::CreateFooterAad(properties_->file_aad()); const SecureString& footer_key = properties_->footer_key(); - auto aes_encryptor = GetMetaAesEncryptor(algorithm, footer_key.size()); + auto encryptor_instance = GetMetaEncryptor(algorithm, footer_key.size()); footer_encryptor_ = std::make_shared( - aes_encryptor, footer_key, properties_->file_aad(), footer_aad, pool_); + encryptor_instance, footer_key, properties_->file_aad(), footer_aad, pool_); return footer_encryptor_; } @@ -69,9 +88,9 @@ std::shared_ptr InternalFileEncryptor::GetFooterSigningEncryptor() { ParquetCipher::type algorithm = properties_->algorithm().algorithm; std::string footer_aad = encryption::CreateFooterAad(properties_->file_aad()); const SecureString& footer_signing_key = properties_->footer_key(); - auto aes_encryptor = GetMetaAesEncryptor(algorithm, footer_signing_key.size()); + auto encryptor_instance = GetMetaEncryptor(algorithm, footer_signing_key.size()); footer_signing_encryptor_ = std::make_shared( - aes_encryptor, footer_signing_key, properties_->file_aad(), footer_aad, pool_); + encryptor_instance, footer_signing_key, properties_->file_aad(), footer_aad, pool_); return footer_signing_encryptor_; } @@ -81,13 +100,14 @@ std::shared_ptr InternalFileEncryptor::GetColumnMetaEncryptor( } std::shared_ptr InternalFileEncryptor::GetColumnDataEncryptor( - const std::string& column_path) { - return GetColumnEncryptor(column_path, false); + const std::string& column_path, const ColumnChunkMetaDataBuilder* column_chunk_metadata) { + return GetColumnEncryptor(column_path, false, column_chunk_metadata); } std::shared_ptr InternalFileEncryptor::InternalFileEncryptor::GetColumnEncryptor( - const std::string& column_path, bool metadata) { + const std::string& column_path, bool metadata, + const ColumnChunkMetaDataBuilder* column_chunk_metadata) { // first look if we already got the encryptor from before if (metadata) { if (column_metadata_map_.find(column_path) != column_metadata_map_.end()) { @@ -108,12 +128,19 @@ InternalFileEncryptor::InternalFileEncryptor::GetColumnEncryptor( : column_prop->key(); ParquetCipher::type algorithm = properties_->algorithm().algorithm; - auto aes_encryptor = metadata ? GetMetaAesEncryptor(algorithm, key.size()) - : GetDataAesEncryptor(algorithm, key.size()); + if (!metadata) { + // Column data encryption might specify a different algorithm + if (column_prop->parquet_cipher().has_value()) { + algorithm = column_prop->parquet_cipher().value(); + } + } + auto encryptor_instance = metadata ? GetMetaEncryptor(algorithm, key.size()) + : GetDataEncryptor(algorithm, key.size(), + column_chunk_metadata); std::string file_aad = properties_->file_aad(); std::shared_ptr encryptor = - std::make_shared(aes_encryptor, key, file_aad, "", pool_); + std::make_shared(encryptor_instance, key, file_aad, "", pool_); if (metadata) column_metadata_map_[column_path] = encryptor; else @@ -122,34 +149,25 @@ InternalFileEncryptor::InternalFileEncryptor::GetColumnEncryptor( return encryptor; } -int InternalFileEncryptor::MapKeyLenToEncryptorArrayIndex(int32_t key_len) const { - if (key_len == 16) - return 0; - else if (key_len == 24) - return 1; - else if (key_len == 32) - return 2; - throw ParquetException("encryption key must be 16, 24 or 32 bytes in length"); -} - -encryption::AesEncryptor* InternalFileEncryptor::GetMetaAesEncryptor( +encryption::EncryptorInterface* InternalFileEncryptor::GetMetaEncryptor( ParquetCipher::type algorithm, size_t key_size) { - auto key_len = static_cast(key_size); - int index = MapKeyLenToEncryptorArrayIndex(key_len); - if (meta_encryptor_[index] == nullptr) { - meta_encryptor_[index] = encryption::AesEncryptor::Make(algorithm, key_len, true); - } - return meta_encryptor_[index].get(); + // Metadata is encrypted with AES. + return aes_encryptor_factory_.GetMetaAesEncryptor(algorithm, key_size); } -encryption::AesEncryptor* InternalFileEncryptor::GetDataAesEncryptor( - ParquetCipher::type algorithm, size_t key_size) { - auto key_len = static_cast(key_size); - int index = MapKeyLenToEncryptorArrayIndex(key_len); - if (data_encryptor_[index] == nullptr) { - data_encryptor_[index] = encryption::AesEncryptor::Make(algorithm, key_len, false); +encryption::EncryptorInterface* InternalFileEncryptor::GetDataEncryptor( + ParquetCipher::type algorithm, size_t key_size, + const ColumnChunkMetaDataBuilder* column_chunk_metadata) { + if (algorithm == ParquetCipher::EXTERNAL_DBPA_V1) { + if (dynamic_cast(properties_) == nullptr) { + throw ParquetException("External DBPA encryption requires ExternalFileEncryptionProperties."); + } + + return external_dbpa_encryptor_factory_.GetEncryptor( + algorithm, column_chunk_metadata, + dynamic_cast(properties_)); } - return data_encryptor_[index].get(); + return aes_encryptor_factory_.GetDataAesEncryptor(algorithm, key_size); } } // namespace parquet diff --git a/cpp/src/parquet/encryption/internal_file_encryptor.h b/cpp/src/parquet/encryption/internal_file_encryptor.h index ee15fe32de96..d7b4434ae6c8 100644 --- a/cpp/src/parquet/encryption/internal_file_encryptor.h +++ b/cpp/src/parquet/encryption/internal_file_encryptor.h @@ -20,33 +20,47 @@ #include #include #include -#include +#include "parquet/encryption/aes_encryption.h" +#include "parquet/encryption/external_dbpa_encryption.h" #include "parquet/encryption/encryption.h" -#include "parquet/schema.h" +#include "parquet/encryption/encryptor_interface.h" +#include "parquet/encryption/encoding_properties.h" +#include "parquet/metadata.h" namespace parquet { -namespace encryption { -class AesEncryptor; -} // namespace encryption +using ::parquet::encryption::EncodingProperties; class FileEncryptionProperties; class ColumnEncryptionProperties; class PARQUET_EXPORT Encryptor { public: - Encryptor(encryption::AesEncryptor* aes_encryptor, ::arrow::util::SecureString key, + Encryptor(encryption::EncryptorInterface* encryptor_interface, ::arrow::util::SecureString key, std::string file_aad, std::string aad, ::arrow::MemoryPool* pool); const std::string& file_aad() { return file_aad_; } void UpdateAad(const std::string& aad) { aad_ = aad; } ::arrow::MemoryPool* pool() { return pool_; } + [[nodiscard]] bool CanCalculateCiphertextLength() const; [[nodiscard]] int32_t CiphertextLength(int64_t plaintext_len) const; int32_t Encrypt(::arrow::util::span plaintext, ::arrow::util::span ciphertext); + int32_t EncryptWithManagedBuffer(::arrow::util::span plaintext, + ::arrow::ResizableBuffer* ciphertext); + + void UpdateEncodingProperties(std::unique_ptr encoding_properties); + + /// After the column_writer writes a dictionary or a data page, this method will be called + /// so that each encryptor can provide any encryptor-specific column metadata that should be + /// stored in the Parquet file. The keys and values are added to the column metadata, any + /// conflicting key and value pairs are overwritten. There is no need to clear the metadata + /// after the call. + std::shared_ptr GetKeyValueMetadata(int8_t module_type); + bool EncryptColumnMetaData( bool encrypted_footer, const std::shared_ptr& column_encryption_properties) { @@ -60,7 +74,7 @@ class PARQUET_EXPORT Encryptor { } private: - encryption::AesEncryptor* aes_encryptor_; + encryption::EncryptorInterface* encryptor_instance_; ::arrow::util::SecureString key_; std::string file_aad_; std::string aad_; @@ -75,7 +89,9 @@ class InternalFileEncryptor { std::shared_ptr GetFooterEncryptor(); std::shared_ptr GetFooterSigningEncryptor(); std::shared_ptr GetColumnMetaEncryptor(const std::string& column_path); - std::shared_ptr GetColumnDataEncryptor(const std::string& column_path); + std::shared_ptr GetColumnDataEncryptor( + const std::string& column_path, + const ColumnChunkMetaDataBuilder* column_chunk_metadata = nullptr); private: FileEncryptionProperties* properties_; @@ -86,22 +102,19 @@ class InternalFileEncryptor { std::shared_ptr footer_signing_encryptor_; std::shared_ptr footer_encryptor_; - // Key must be 16, 24 or 32 bytes in length. Thus there could be up to three - // types of meta_encryptors and data_encryptors. - std::unique_ptr meta_encryptor_[3]; - std::unique_ptr data_encryptor_[3]; - ::arrow::MemoryPool* pool_; + encryption::AesEncryptorFactory aes_encryptor_factory_; + encryption::ExternalDBPAEncryptorAdapterFactory external_dbpa_encryptor_factory_; - std::shared_ptr GetColumnEncryptor(const std::string& column_path, - bool metadata); + std::shared_ptr GetColumnEncryptor( + const std::string& column_path, bool metadata, + const ColumnChunkMetaDataBuilder* column_chunk_metadata = nullptr); - encryption::AesEncryptor* GetMetaAesEncryptor(ParquetCipher::type algorithm, - size_t key_len); - encryption::AesEncryptor* GetDataAesEncryptor(ParquetCipher::type algorithm, - size_t key_len); + encryption::EncryptorInterface* GetMetaEncryptor(ParquetCipher::type algorithm, size_t key_len); - int MapKeyLenToEncryptorArrayIndex(int32_t key_len) const; + encryption::EncryptorInterface* GetDataEncryptor( + ParquetCipher::type algorithm, size_t key_len, + const ColumnChunkMetaDataBuilder* column_chunk_metadata = nullptr); }; } // namespace parquet diff --git a/cpp/src/parquet/encryption/key_toolkit_internal.cc b/cpp/src/parquet/encryption/key_toolkit_internal.cc index 60a8a52206c3..5c08d778498f 100644 --- a/cpp/src/parquet/encryption/key_toolkit_internal.cc +++ b/cpp/src/parquet/encryption/key_toolkit_internal.cc @@ -18,7 +18,8 @@ #include "arrow/util/base64.h" #include "arrow/util/secure_string.h" -#include "parquet/encryption/encryption_internal.h" +#include "parquet/encryption/encryption.h" +#include "parquet/encryption/aes_encryption.h" #include "parquet/encryption/key_toolkit_internal.h" using arrow::util::SecureString; diff --git a/cpp/src/parquet/encryption/properties_test.cc b/cpp/src/parquet/encryption/properties_test.cc index 1ceda7ac032f..ca4086594486 100644 --- a/cpp/src/parquet/encryption/properties_test.cc +++ b/cpp/src/parquet/encryption/properties_test.cc @@ -48,6 +48,46 @@ TEST(TestColumnEncryptionProperties, ColumnEncryptedWithFooterKey) { ASSERT_EQ(true, column_props_1->is_encrypted_with_footer_key()); } +TEST(TestColumnEncryptionProperties, ColumnParquetCipherNotSpecified) { + std::string column_path = "column_path"; + ColumnEncryptionProperties::Builder column_builder(column_path); + std::shared_ptr properties = column_builder.build(); + + ASSERT_EQ(column_path, properties->column_path()); + ASSERT_EQ(false, properties->parquet_cipher().has_value()); +} + +TEST(TestColumnEncryptionProperties, ColumnParquetCipherSpecified) { + std::string column_path = "column_path"; + ColumnEncryptionProperties::Builder column_builder(column_path); + column_builder.parquet_cipher(ParquetCipher::AES_GCM_CTR_V1); + std::shared_ptr properties = column_builder.build(); + + ASSERT_EQ(column_path, properties->column_path()); + ASSERT_EQ(true, properties->parquet_cipher().has_value()); + ASSERT_EQ(ParquetCipher::AES_GCM_CTR_V1, properties->parquet_cipher().value()); +} + +TEST(TestColumnDecryptionProperties, ColumnParquetCipherNotSpecified) { + std::string column_path = "column_path"; + ColumnDecryptionProperties::Builder column_builder(column_path); + std::shared_ptr properties = column_builder.build(); + + ASSERT_EQ(column_path, properties->column_path()); + ASSERT_EQ(false, properties->parquet_cipher().has_value()); +} + +TEST(TestColumnDecryptionProperties, ColumnParquetCipherSpecified) { + std::string column_path = "column_path"; + ColumnDecryptionProperties::Builder column_builder(column_path); + column_builder.parquet_cipher(ParquetCipher::AES_GCM_CTR_V1); + std::shared_ptr properties = column_builder.build(); + + ASSERT_EQ(column_path, properties->column_path()); + ASSERT_EQ(true, properties->parquet_cipher().has_value()); + ASSERT_EQ(ParquetCipher::AES_GCM_CTR_V1, properties->parquet_cipher().value()); +} + // Encrypt all columns and the footer with the same key. // (uniform encryption) TEST(TestEncryptionProperties, UniformEncryption) { @@ -210,6 +250,128 @@ TEST(TestEncryptionProperties, UseAES_GCM_CTR_V1Algorithm) { ASSERT_EQ(ParquetCipher::AES_GCM_CTR_V1, props->algorithm().algorithm); } +TEST(TestExternalFileEncryptionProperties, SuperClassFieldsSetCorrectly) { + std::string column_name_1 = "column_1"; + ColumnEncryptionProperties::Builder column_builder_1(column_name_1); + column_builder_1.key(kColumnEncryptionKey1); + column_builder_1.key_id("kc1"); + + std::string column_name_2 = "column_2"; + ColumnEncryptionProperties::Builder column_builder_2(column_name_2); + column_builder_2.key(kColumnEncryptionKey2); + column_builder_2.key_id("kc2"); + + std::map> encrypted_columns; + encrypted_columns[column_name_1] = column_builder_1.build(); + encrypted_columns[column_name_2] = column_builder_2.build(); + + ExternalFileEncryptionProperties::Builder builder(kFooterEncryptionKey); + builder.footer_key_metadata("kf"); + builder.encrypted_columns(encrypted_columns); + std::shared_ptr props = builder.build_external(); + + ASSERT_EQ(true, props->encrypted_footer()); + ASSERT_EQ(kDefaultEncryptionAlgorithm, props->algorithm().algorithm); + ASSERT_EQ(kFooterEncryptionKey, props->footer_key()); + + std::shared_ptr out_col_props_1 = + props->column_encryption_properties(column_name_1); + + ASSERT_EQ(column_name_1, out_col_props_1->column_path()); + ASSERT_EQ(true, out_col_props_1->is_encrypted()); + ASSERT_EQ(false, out_col_props_1->is_encrypted_with_footer_key()); + ASSERT_EQ(kColumnEncryptionKey1, out_col_props_1->key()); + ASSERT_EQ("kc1", out_col_props_1->key_metadata()); + + std::shared_ptr out_col_props_2 = + props->column_encryption_properties(column_name_2); + + ASSERT_EQ(column_name_2, out_col_props_2->column_path()); + ASSERT_EQ(true, out_col_props_2->is_encrypted()); + ASSERT_EQ(false, out_col_props_2->is_encrypted_with_footer_key()); + ASSERT_EQ(kColumnEncryptionKey2, out_col_props_2->key()); + ASSERT_EQ("kc2", out_col_props_2->key_metadata()); + + std::string column_name_3 = "column_3"; + std::shared_ptr out_col_props_3 = + props->column_encryption_properties(column_name_3); + + ASSERT_EQ(NULLPTR, out_col_props_3); + ASSERT_EQ(true, props->app_context().empty()); + ASSERT_EQ(true, props->connection_config().size() == 0); +} + +// The subclass adds two additional fields +TEST(TestExternalFileEncryptionProperties, SetExternalContextAndConfig) { + std::string app_context = "{\n" + " \"user_id\": \"abc123\",\n" + " \"location\": {\n" + " \"lat\": 10.0,\n" + " \"lon\": -84.0\n" + " }\n" + "}"; + std::map> connection_config; + connection_config[ParquetCipher::AES_GCM_V1]["lib_location"] = "path/to/lib.so"; + connection_config[ParquetCipher::AES_GCM_V1]["config_file"] = "path/to/config/file"; + + ExternalFileEncryptionProperties::Builder builder(kFooterEncryptionKey); + builder.app_context(app_context); + builder.connection_config(connection_config); + std::shared_ptr props = builder.build_external(); + + ASSERT_EQ(false, props->app_context().empty()); + ASSERT_EQ(app_context, props->app_context()); + ASSERT_EQ(false, props->connection_config().size() == 0); + ASSERT_EQ(connection_config, props->connection_config()); +} + +TEST(TestExternalFileEncryptionProperties, EncryptTwoColumnsWithDifferentAlgorithms) { + std::string column_name_1 = "column_1"; + ColumnEncryptionProperties::Builder column_builder_1(column_name_1); + column_builder_1.key(kColumnEncryptionKey1); + column_builder_1.key_id("kc1"); + column_builder_1.parquet_cipher(ParquetCipher::AES_GCM_V1); + + std::string column_name_2 = "column_2"; + ColumnEncryptionProperties::Builder column_builder_2(column_name_2); + column_builder_2.key(kColumnEncryptionKey2); + column_builder_2.key_id("kc2"); + column_builder_2.parquet_cipher(ParquetCipher::AES_GCM_CTR_V1); + + std::map> encrypted_columns; + encrypted_columns[column_name_1] = column_builder_1.build(); + encrypted_columns[column_name_2] = column_builder_2.build(); + + ExternalFileEncryptionProperties::Builder builder(kFooterEncryptionKey); + builder.footer_key_metadata("kf"); + builder.encrypted_columns(encrypted_columns); + std::shared_ptr props = builder.build_external(); + + ASSERT_EQ(true, props->encrypted_footer()); + ASSERT_EQ(kDefaultEncryptionAlgorithm, props->algorithm().algorithm); + ASSERT_EQ(kFooterEncryptionKey, props->footer_key()); + + std::shared_ptr out_col_props_1 = + props->column_encryption_properties(column_name_1); + + ASSERT_EQ(column_name_1, out_col_props_1->column_path()); + ASSERT_EQ(true, out_col_props_1->is_encrypted()); + ASSERT_EQ(false, out_col_props_1->is_encrypted_with_footer_key()); + ASSERT_EQ(kColumnEncryptionKey1, out_col_props_1->key()); + ASSERT_EQ(ParquetCipher::AES_GCM_V1, out_col_props_1->parquet_cipher()); + ASSERT_EQ("kc1", out_col_props_1->key_metadata()); + + std::shared_ptr out_col_props_2 = + props->column_encryption_properties(column_name_2); + + ASSERT_EQ(column_name_2, out_col_props_2->column_path()); + ASSERT_EQ(true, out_col_props_2->is_encrypted()); + ASSERT_EQ(false, out_col_props_2->is_encrypted_with_footer_key()); + ASSERT_EQ(kColumnEncryptionKey2, out_col_props_2->key()); + ASSERT_EQ(ParquetCipher::AES_GCM_CTR_V1, out_col_props_2->parquet_cipher()); + ASSERT_EQ("kc2", out_col_props_2->key_metadata()); +} + TEST(TestDecryptionProperties, UseKeyRetriever) { std::shared_ptr string_kr1 = std::make_shared(); @@ -269,4 +431,66 @@ TEST(TestDecryptionProperties, UsingExplicitFooterAndColumnKeys) { ASSERT_EQ(kColumnEncryptionKey2, props->column_key(column_path_2)); } +TEST(TestExternalFileDecryptionProperties, SuperClassFieldsSetCorrectly) { + std::shared_ptr string_kr1 = + std::make_shared(); + string_kr1->PutKey("kf", kFooterEncryptionKey); + string_kr1->PutKey("kc1", kColumnEncryptionKey1); + string_kr1->PutKey("kc2", kColumnEncryptionKey2); + std::shared_ptr kr1 = + std::static_pointer_cast(string_kr1); + + + auto builder = parquet::ExternalFileDecryptionProperties::Builder(); + builder.footer_key(kFooterEncryptionKey); + builder.plaintext_files_allowed(); + builder.key_retriever(kr1); + std::shared_ptr props = builder.build_external(); + + ASSERT_EQ(true, props->plaintext_files_allowed()); + ASSERT_EQ(kFooterEncryptionKey, props->footer_key()); + ASSERT_EQ(true, props->app_context().empty()); + ASSERT_EQ(true, props->connection_config().size() == 0); + + auto out_key_retriever = props->key_retriever(); + ASSERT_EQ(kFooterEncryptionKey, out_key_retriever->GetKey("kf")); + ASSERT_EQ(kColumnEncryptionKey1, out_key_retriever->GetKey("kc1")); + ASSERT_EQ(kColumnEncryptionKey2, out_key_retriever->GetKey("kc2")); +} + +TEST(TestExternalFileDecryptionProperties, SetExternalContextAndConfig) { + std::shared_ptr string_kr1 = + std::make_shared(); + std::string app_context = "{\n" + " \"user_id\": \"abc123\",\n" + " \"location\": {\n" + " \"lat\": 10.0,\n" + " \"lon\": -84.0\n" + " }\n" + "}"; + std::map> connection_config; + std::map inner_config; + inner_config["lib_location"] = "path/to/lib.so"; + inner_config["config_file"] = "path/to/config/file"; + connection_config[ParquetCipher::AES_GCM_CTR_V1] = inner_config; + + auto builder = parquet::ExternalFileDecryptionProperties::Builder(); + builder.footer_key(kFooterEncryptionKey); + builder.app_context(app_context); + builder.connection_config(connection_config); + std::shared_ptr props = builder.build_external(); + + ASSERT_EQ(false, props->app_context().empty()); + ASSERT_EQ(app_context, props->app_context()); + ASSERT_EQ(false, props->connection_config().size() == 0); + ASSERT_EQ(connection_config, props->connection_config()); +} + +TEST(TestExternalFileDecryptionProperties, SetInvalidAppContext) { + std::string invalid_app_context = "invalid_app_context"; + parquet::ExternalFileDecryptionProperties::Builder builder; + builder.footer_key(kFooterEncryptionKey); + ASSERT_THROW(builder.app_context(invalid_app_context), parquet::ParquetException); +} + } // namespace parquet::encryption::test diff --git a/cpp/src/parquet/file_deserialize_test.cc b/cpp/src/parquet/file_deserialize_test.cc index 7fa5e2f167e2..36f7c97c2ab2 100644 --- a/cpp/src/parquet/file_deserialize_test.cc +++ b/cpp/src/parquet/file_deserialize_test.cc @@ -18,20 +18,32 @@ #include #include +#include #include #include +#include #include #include +#include +#include #include "parquet/column_page.h" #include "parquet/column_reader.h" #include "parquet/exception.h" #include "parquet/file_reader.h" +#include "parquet/file_writer.h" #include "parquet/metadata.h" #include "parquet/platform.h" +#include "parquet/properties.h" #include "parquet/test_util.h" #include "parquet/thrift_internal.h" #include "parquet/types.h" +// Added for encoding properties tests +#include "parquet/encryption/decryptor_interface.h" +#include "parquet/encryption/encoding_properties.h" +#include "parquet/encryption/internal_file_decryptor.h" +#include "parquet/encryption/external/test_utils.h" +#include "parquet/schema.h" #include "arrow/io/memory.h" #include "arrow/status.h" @@ -40,6 +52,7 @@ #include "arrow/util/config.h" #include "arrow/util/crc32.h" #include "arrow/util/logging_internal.h" +#include "arrow/util/secure_string.h" namespace parquet { @@ -978,3 +991,365 @@ TEST_F(TestParquetFileReader, IncompleteMetadata) { } } // namespace parquet + +// ---------------------------------------------------------------------- +// EncodingProperties tests using SerializedPageReader and a capturing decryptor + +namespace parquet { + + namespace { + + struct CapturedEncodingProps { + std::vector> entries; + }; + + class CapturingTestDecryptor : public parquet::encryption::DecryptorInterface { + public: + CapturingTestDecryptor(std::shared_ptr sink, + std::string column_path, + parquet::Type::type physical_type, + ::arrow::Compression::type compression_codec) + : sink_(std::move(sink)), + column_path_(std::move(column_path)), + physical_type_(physical_type), + compression_codec_(compression_codec) {} + + [[nodiscard]] bool CanCalculateLengths() const override { + return true; + } + + [[nodiscard]] int32_t PlaintextLength(int32_t ciphertext_len) const override { + return ciphertext_len; + } + + [[nodiscard]] int32_t CiphertextLength(int32_t plaintext_len) const override { + return plaintext_len; + } + + int32_t Decrypt(::arrow::util::span ciphertext, + ::arrow::util::span /*key*/, + ::arrow::util::span /*aad*/, + ::arrow::util::span plaintext) override { + std::copy(ciphertext.begin(), ciphertext.end(), plaintext.begin()); + return static_cast(ciphertext.size()); + } + + int32_t DecryptWithManagedBuffer(::arrow::util::span ciphertext, + ::arrow::ResizableBuffer* plaintext) override { + throw ParquetException("DecryptWithManagedBuffer not supported"); + } + + void UpdateEncodingProperties( + std::unique_ptr encoding_properties) override { + // Fill column-level properties so validate() succeeds + encoding_properties->set_column_path(column_path_); + encoding_properties->set_physical_type(physical_type_); + encoding_properties->set_compression_codec(compression_codec_); + + encoding_properties->validate(); + sink_->entries.emplace_back(encoding_properties->ToPropertiesMap()); + } + + private: + std::shared_ptr sink_; + std::string column_path_; + parquet::Type::type physical_type_; + ::arrow::Compression::type compression_codec_; + }; + + static std::shared_ptr<::parquet::SchemaDescriptor> MakeSingleInt32Schema( + const std::string& col_name = "col") { + using ::parquet::schema::GroupNode; + using ::parquet::schema::NodePtr; + using ::parquet::schema::NodeVector; + using ::parquet::schema::PrimitiveNode; + + NodeVector fields; + fields.push_back( + PrimitiveNode::Make(col_name, ::parquet::Repetition::REQUIRED, ::parquet::Type::INT32)); + NodePtr schema = GroupNode::Make("schema", ::parquet::Repetition::REQUIRED, fields); + + auto descr = std::make_shared<::parquet::SchemaDescriptor>(); + descr->Init(schema); + return descr; + } + + static std::shared_ptr<::parquet::SchemaDescriptor> MakeNestedOptionalRepeatedIntSchema() { + using ::parquet::schema::GroupNode; + using ::parquet::schema::NodePtr; + using ::parquet::schema::NodeVector; + using ::parquet::schema::PrimitiveNode; + + // Schema: + // required group schema { + // optional group optgrp { + // repeated group list { + // optional int32 element; + // } + // } + // } + // Expected: max_definition_level = 2 (optgrp optional + element optional) + // max_repetition_level = 1 (list repeated) + NodePtr element = PrimitiveNode::Make("element", ::parquet::Repetition::OPTIONAL, + ::parquet::Type::INT32); + NodeVector list_children; + list_children.push_back(element); + NodePtr list = GroupNode::Make("list", ::parquet::Repetition::REPEATED, list_children); + NodeVector optgrp_children; + optgrp_children.push_back(list); + NodePtr optgrp = GroupNode::Make("optgrp", ::parquet::Repetition::OPTIONAL, optgrp_children); + NodeVector root_fields; + root_fields.push_back(optgrp); + NodePtr schema = GroupNode::Make("schema", ::parquet::Repetition::REQUIRED, root_fields); + + auto descr = std::make_shared<::parquet::SchemaDescriptor>(); + descr->Init(schema); + return descr; + } + + } // namespace + + class EncodingPropertiesSerdeTest : public TestPageSerde { + protected: + void OpenWithCryptoContext(int64_t num_rows, Compression::type codec, + const ReaderProperties& properties, + const CryptoContext& crypto_ctx) { + EndStream(); + auto stream = std::make_shared<::arrow::io::BufferReader>(out_buffer_); + page_reader_ = PageReader::Open(stream, num_rows, codec, properties, + /*always_compressed=*/false, &crypto_ctx); + } + }; + + TEST_F(EncodingPropertiesSerdeTest, CapturesDictionaryPageEncodingProperties) { + // Prepare a small dictionary page + const int32_t num_rows = 5; + dictionary_page_header_.encoding = format::Encoding::PLAIN; + dictionary_page_header_.num_values = num_rows; + + int data_size = 16; + ASSERT_NO_FATAL_FAILURE(WriteDictionaryPageHeader(data_size, data_size)); + std::vector faux_data(data_size); + ASSERT_OK(out_stream_->Write(faux_data.data(), data_size)); + + // Build schema and crypto context + auto schema = MakeNestedOptionalRepeatedIntSchema(); + const ColumnDescriptor* descr = schema->Column(0); + auto sink = std::make_shared(); + + CryptoContext ctx; + ctx.column_descriptor = descr; + ctx.data_decryptor_factory = [sink, descr]() { + auto iface = std::make_unique( + sink, descr->path()->ToDotString(), descr->physical_type(), + ::arrow::Compression::UNCOMPRESSED); + return std::make_unique(std::move(iface), /*key*/ ::arrow::util::SecureString(), + /*file_aad*/ std::string("aad_test"), + /*aad*/ std::string(), + ::arrow::default_memory_pool()); + }; + + ReaderProperties reader_props; + OpenWithCryptoContext(/*num_rows=*/num_rows, Compression::UNCOMPRESSED, reader_props, ctx); + + std::shared_ptr page = page_reader_->NextPage(); + ASSERT_NE(page, nullptr); + ASSERT_EQ(PageType::DICTIONARY_PAGE, page->type()); + + ASSERT_EQ(sink->entries.size(), 1); + const auto& props = sink->entries[0]; + ASSERT_EQ(props.at("column_path"), descr->path()->ToDotString()); + ASSERT_EQ(props.at("physical_type"), std::string("INT32")); + ASSERT_EQ(props.at("compression_codec"), std::string("UNCOMPRESSED")); + ASSERT_EQ(props.at("page_type"), std::string("DICTIONARY_PAGE")); + ASSERT_EQ(props.at("page_encoding"), std::string("PLAIN")); + } + + TEST_F(EncodingPropertiesSerdeTest, CapturesDataPageV1EncodingProperties) { + // Prepare a small V1 data page + const int32_t num_values = 42; + data_page_header_.encoding = format::Encoding::PLAIN; + data_page_header_.definition_level_encoding = format::Encoding::RLE; + data_page_header_.repetition_level_encoding = format::Encoding::RLE; + data_page_header_.num_values = num_values; + + int data_size = 32; + ASSERT_NO_FATAL_FAILURE(WriteDataPageHeader(/*max_serialized_len=*/1024, data_size, + /*compressed_size=*/data_size)); + std::vector faux_data(data_size); + ASSERT_OK(out_stream_->Write(faux_data.data(), data_size)); + + auto schema = MakeSingleInt32Schema("c0"); + const ColumnDescriptor* descr = schema->Column(0); + auto sink = std::make_shared(); + + CryptoContext ctx; + ctx.column_descriptor = descr; + ctx.data_decryptor_factory = [sink, descr]() { + auto iface = std::make_unique( + sink, descr->path()->ToDotString(), descr->physical_type(), + ::arrow::Compression::UNCOMPRESSED); + return std::make_unique(std::move(iface), /*key*/ ::arrow::util::SecureString(), + /*file_aad*/ std::string("aad_test"), + /*aad*/ std::string(), + ::arrow::default_memory_pool()); + }; + + ReaderProperties reader_props; + OpenWithCryptoContext(/*num_rows=*/num_values, Compression::UNCOMPRESSED, reader_props, ctx); + + std::shared_ptr page = page_reader_->NextPage(); + ASSERT_NE(page, nullptr); + ASSERT_EQ(PageType::DATA_PAGE, page->type()); + + ASSERT_EQ(sink->entries.size(), 1); + const auto& props = sink->entries[0]; + ASSERT_EQ(props.at("page_type"), std::string("DATA_PAGE_V1")); + ASSERT_EQ(props.at("data_page_num_values"), std::string("42")); + // Levels should match what the descriptor reports + ASSERT_EQ(props.at("data_page_max_definition_level"), + std::to_string(descr->max_definition_level())); + ASSERT_EQ(props.at("data_page_max_repetition_level"), + std::to_string(descr->max_repetition_level())); + } + + TEST_F(EncodingPropertiesSerdeTest, CapturesDataPageV2EncodingProperties) { + // Prepare a small V2 data page + data_page_header_v2_.encoding = format::Encoding::PLAIN; + data_page_header_v2_.num_values = 12; + data_page_header_v2_.num_nulls = 3; + data_page_header_v2_.definition_levels_byte_length = 5; + data_page_header_v2_.repetition_levels_byte_length = 7; + data_page_header_v2_.is_compressed = false; + + int data_size = 24; + ASSERT_NO_FATAL_FAILURE(WriteDataPageHeaderV2(/*max_serialized_len=*/1024, data_size, + /*compressed_size=*/data_size)); + std::vector faux_data(data_size); + ASSERT_OK(out_stream_->Write(faux_data.data(), data_size)); + + auto schema = MakeNestedOptionalRepeatedIntSchema(); + const ColumnDescriptor* descr = schema->Column(0); + auto sink = std::make_shared(); + + CryptoContext ctx; + ctx.column_descriptor = descr; + ctx.data_decryptor_factory = [sink, descr]() { + auto iface = std::make_unique( + sink, descr->path()->ToDotString(), descr->physical_type(), + ::arrow::Compression::UNCOMPRESSED); + return std::make_unique(std::move(iface), /*key*/ ::arrow::util::SecureString(), + /*file_aad*/ std::string("aad_test"), + /*aad*/ std::string(), + ::arrow::default_memory_pool()); + }; + + ReaderProperties reader_props; + OpenWithCryptoContext(/*num_rows=*/12, Compression::UNCOMPRESSED, reader_props, ctx); + + std::shared_ptr page = page_reader_->NextPage(); + ASSERT_NE(page, nullptr); + ASSERT_EQ(PageType::DATA_PAGE_V2, page->type()); + + ASSERT_EQ(sink->entries.size(), 1); + const auto& props = sink->entries[0]; + ASSERT_EQ(props.at("page_type"), std::string("DATA_PAGE_V2")); + ASSERT_EQ(props.at("data_page_num_values"), std::string("12")); + ASSERT_EQ(props.at("page_v2_definition_levels_byte_length"), std::string("5")); + ASSERT_EQ(props.at("page_v2_repetition_levels_byte_length"), std::string("7")); + ASSERT_EQ(props.at("page_v2_num_nulls"), std::string("3")); + ASSERT_EQ(props.at("page_v2_is_compressed"), std::string("false")); + // Levels should match what the descriptor reports + ASSERT_EQ(props.at("data_page_max_definition_level"), + std::to_string(descr->max_definition_level())); + ASSERT_EQ(props.at("data_page_max_repetition_level"), + std::to_string(descr->max_repetition_level())); + } + + TEST(PlaintextFooter_EncryptionAlgorithmsSetCorrectly, ExternalAndAES) { + using schema::GroupNode; + using schema::NodePtr; + using schema::PrimitiveNode; + + NodePtr root = GroupNode::Make( + "schema", Repetition::REQUIRED, + {PrimitiveNode::Make("col", Repetition::REQUIRED, Type::INT32)}); + auto schema = std::static_pointer_cast(root); + + std::vector values = {1, 2, 3, 4, 5}; + + const std::string column_name = "col"; + ::arrow::util::SecureString column_key_id(std::string("0123456789ABCDEF")); + ::arrow::util::SecureString footer_key(std::string("1234567890123456")); // 16 bytes for tests + std::string app_context = + "{\"user_id\":\"test_user\",\"location\":{\"lat\":0.0,\"lon\":0.0}}"; + std::string library_path = + parquet::encryption::external::test::TestUtils::GetTestLibraryPath(); + + std::map> enc_cols; + auto col_enc_builder = parquet::ColumnEncryptionProperties::Builder(column_name); + col_enc_builder.key(column_key_id)->key_id(std::string(column_key_id.as_view())) + ->parquet_cipher(parquet::ParquetCipher::EXTERNAL_DBPA_V1); + enc_cols[column_name] = col_enc_builder.build(); + + auto fep_builder = parquet::ExternalFileEncryptionProperties::Builder(footer_key); + fep_builder.footer_key_metadata("kf") + ->set_plaintext_footer() + ->algorithm(parquet::ParquetCipher::AES_GCM_CTR_V1) + ->encrypted_columns(enc_cols) + ->app_context(app_context) + ->connection_config({{parquet::ParquetCipher::EXTERNAL_DBPA_V1, + {{"agent_library_path", library_path}}}}); + auto file_enc_props = fep_builder.build_external(); + + auto sink = CreateOutputStream(); + auto writer_props = parquet::WriterProperties::Builder().encryption(file_enc_props)->build(); + auto file_writer = parquet::ParquetFileWriter::Open(sink, schema, writer_props); + { + auto rg = file_writer->AppendRowGroup(); + auto w = static_cast(rg->NextColumn()); + w->WriteBatch(static_cast(values.size()), nullptr, nullptr, values.data()); + w->Close(); + } + file_writer->Close(); + ASSERT_OK_AND_ASSIGN(auto buffer, sink->Finish()); + + std::map> dec_cols; + auto col_dec_builder = parquet::ColumnDecryptionProperties::Builder(column_name); + dec_cols[column_name] = col_dec_builder.key(column_key_id)->build(); + + parquet::ReaderProperties reader_props; + auto dep_builder = parquet::ExternalFileDecryptionProperties::Builder(); + dep_builder.footer_key(footer_key) + ->column_keys(dec_cols) + ->app_context(app_context) + ->connection_config({{parquet::ParquetCipher::EXTERNAL_DBPA_V1, + {{"agent_library_path", library_path}}}}); + reader_props.file_decryption_properties(dep_builder.build_external()); + + auto file_reader = parquet::ParquetFileReader::Open( + std::make_shared<::arrow::io::BufferReader>(buffer), reader_props); + ASSERT_NE(file_reader, nullptr); + + auto file_md = file_reader->metadata(); + auto file_algo = file_md->encryption_algorithm(); + EXPECT_EQ(file_algo.algorithm, parquet::ParquetCipher::AES_GCM_CTR_V1); + + auto rg_md = file_md->RowGroup(0); + auto col_md = rg_md->ColumnChunk(0); + auto crypto_md = col_md->crypto_metadata(); + ASSERT_NE(crypto_md, nullptr); + ASSERT_TRUE(crypto_md->is_encryption_algorithm_set()); + EXPECT_EQ(crypto_md->encryption_algorithm().algorithm, + parquet::ParquetCipher::EXTERNAL_DBPA_V1); + + auto rg_reader = file_reader->RowGroup(0); + auto col_reader = std::static_pointer_cast>(rg_reader->Column(0)); + std::vector out(values.size()); + int64_t read = 0; + col_reader->ReadBatch(static_cast(values.size()), nullptr, nullptr, out.data(), &read); + ASSERT_EQ(read, static_cast(values.size())); + ASSERT_EQ(values, out); + } + + } // namespace parque diff --git a/cpp/src/parquet/file_reader.cc b/cpp/src/parquet/file_reader.cc index b246feaf732b..8d0a3f999cbf 100644 --- a/cpp/src/parquet/file_reader.cc +++ b/cpp/src/parquet/file_reader.cc @@ -40,7 +40,7 @@ #include "parquet/bloom_filter_reader.h" #include "parquet/column_reader.h" #include "parquet/column_scanner.h" -#include "parquet/encryption/encryption_internal.h" +#include "parquet/encryption/encryption_utils.h" #include "parquet/encryption/internal_file_decryptor.h" #include "parquet/exception.h" #include "parquet/file_writer.h" @@ -267,7 +267,7 @@ class SerializedRowGroup : public RowGroupReader::Contents { auto meta_decryptor_factory = InternalFileDecryptor::GetColumnMetaDecryptorFactory( file_decryptor, crypto_metadata.get()); auto data_decryptor_factory = InternalFileDecryptor::GetColumnDataDecryptorFactory( - file_decryptor, crypto_metadata.get()); + file_decryptor, crypto_metadata.get(), col.get()); constexpr auto kEncryptedOrdinalLimit = 32767; if (ARROW_PREDICT_FALSE(row_group_ordinal_ > kEncryptedOrdinalLimit)) { @@ -277,8 +277,11 @@ class SerializedRowGroup : public RowGroupReader::Contents { throw ParquetException("Encrypted files cannot contain more than 32767 columns"); } + const ColumnDescriptor* descr = file_metadata_->schema()->Column(i); CryptoContext ctx{col->has_dictionary_page(), - static_cast(row_group_ordinal_), static_cast(i), + static_cast(row_group_ordinal_), + static_cast(i), + descr, std::move(meta_decryptor_factory), std::move(data_decryptor_factory)}; return PageReader::Open(stream, col->num_values(), col->compression(), properties_, diff --git a/cpp/src/parquet/file_writer.cc b/cpp/src/parquet/file_writer.cc index 8c19aecb0df4..e00775822e3b 100644 --- a/cpp/src/parquet/file_writer.cc +++ b/cpp/src/parquet/file_writer.cc @@ -27,7 +27,6 @@ #include "arrow/util/key_value_metadata.h" #include "arrow/util/logging_internal.h" #include "parquet/column_writer.h" -#include "parquet/encryption/encryption_internal.h" #include "parquet/encryption/internal_file_encryptor.h" #include "parquet/exception.h" #include "parquet/page_index.h" @@ -272,7 +271,7 @@ class RowGroupSerializer : public RowGroupWriter::Contents { file_encryptor_ ? file_encryptor_->GetColumnMetaEncryptor(path->ToDotString()) : nullptr; auto data_encryptor = - file_encryptor_ ? file_encryptor_->GetColumnDataEncryptor(path->ToDotString()) + file_encryptor_ ? file_encryptor_->GetColumnDataEncryptor(path->ToDotString(), col_meta) : nullptr; auto ci_builder = page_index_builder_ && column_properties.page_index_enabled() ? page_index_builder_->GetColumnIndexBuilder(column_ordinal) diff --git a/cpp/src/parquet/metadata.cc b/cpp/src/parquet/metadata.cc index 4b1822c0dae1..74ba020743cc 100644 --- a/cpp/src/parquet/metadata.cc +++ b/cpp/src/parquet/metadata.cc @@ -32,7 +32,8 @@ #include "arrow/util/key_value_metadata.h" #include "arrow/util/logging_internal.h" #include "arrow/util/pcg_random.h" -#include "parquet/encryption/encryption_internal.h" +#include "parquet/encryption/aes_encryption.h" +#include "parquet/encryption/encryption_utils.h" #include "parquet/encryption/internal_file_decryptor.h" #include "parquet/exception.h" #include "parquet/schema.h" @@ -220,6 +221,12 @@ class ColumnCryptoMetaData::ColumnCryptoMetaDataImpl { const std::string& key_metadata() const { return crypto_metadata_->ENCRYPTION_WITH_COLUMN_KEY.key_metadata; } + bool is_encryption_algorithm_set() const { + return crypto_metadata_->ENCRYPTION_WITH_COLUMN_KEY.__isset.encryption_algorithm; + } + EncryptionAlgorithm encryption_algorithm() const { + return FromThrift(crypto_metadata_->ENCRYPTION_WITH_COLUMN_KEY.encryption_algorithm); + } private: const format::ColumnCryptoMetaData* crypto_metadata_; @@ -245,6 +252,12 @@ bool ColumnCryptoMetaData::encrypted_with_footer_key() const { const std::string& ColumnCryptoMetaData::key_metadata() const { return impl_->key_metadata(); } +bool ColumnCryptoMetaData::is_encryption_algorithm_set() const { + return impl_->is_encryption_algorithm_set(); +} +EncryptionAlgorithm ColumnCryptoMetaData::encryption_algorithm() const { + return impl_->encryption_algorithm(); +} // ColumnChunk metadata class ColumnChunkMetaData::ColumnChunkMetaDataImpl { @@ -382,6 +395,10 @@ class ColumnChunkMetaData::ColumnChunkMetaDataImpl { return LoadEnumSafe(&column_metadata_->codec); } + const ColumnDescriptor* descr() const { return descr_; } + + const ReaderProperties* properties() const { return &properties_; } + const std::vector& encodings() const { return encodings_; } const std::vector& encoding_stats() const { return encoding_stats_; } @@ -557,6 +574,10 @@ Compression::type ColumnChunkMetaData::compression() const { return impl_->compression(); } +const ColumnDescriptor* ColumnChunkMetaData::descr() const { return impl_->descr(); } + +const ReaderProperties* ColumnChunkMetaData::properties() const { return impl_->properties(); } + bool ColumnChunkMetaData::can_decompress() const { return ::arrow::util::Codec::IsAvailable(compression()); } @@ -868,8 +889,18 @@ class FileMetaData::FileMetaDataImpl { serialized_len); // encrypt the footer key - std::vector encrypted_data(encryptor->CiphertextLength(serialized_len)); - int32_t encrypted_len = encryptor->Encrypt(serialized_data_span, encrypted_data); + std::vector encrypted_data; + int32_t encrypted_len; + if (encryptor->CanCalculateCiphertextLength()) { + encrypted_data = std::vector(encryptor->CiphertextLength(serialized_len)); + encrypted_len = encryptor->Encrypt(serialized_data_span, encrypted_data); + } else { + auto resizable_buffer = ::arrow::AllocateResizableBuffer(0); + encrypted_len = encryptor->EncryptWithManagedBuffer( + serialized_data_span, resizable_buffer->get()); + encrypted_data.assign(resizable_buffer->get()->data(), + resizable_buffer->get()->data() + encrypted_len); + } // write unencrypted footer PARQUET_THROW_NOT_OK(dst->Write(serialized_data, serialized_len)); @@ -1618,6 +1649,8 @@ class ColumnChunkMetaDataBuilder::ColumnChunkMetaDataBuilderImpl { const void* contents() const { return column_chunk_; } + const WriterProperties* properties() const { return properties_.get(); } + // column chunk void set_file_path(const std::string& val) { column_chunk_->__set_file_path(val); } @@ -1715,6 +1748,12 @@ class ColumnChunkMetaDataBuilder::ColumnChunkMetaDataBuilderImpl { format::EncryptionWithColumnKey eck; eck.__set_key_metadata(encrypt_md->key_metadata()); eck.__set_path_in_schema(column_->path()->ToDotVector()); + // check if column has its own encryption algorithm + if (encrypt_md->parquet_cipher().has_value()) { + EncryptionAlgorithm column_encryption_algorithm; + column_encryption_algorithm.algorithm = encrypt_md->parquet_cipher().value(); + eck.__set_encryption_algorithm(ToThrift(column_encryption_algorithm)); + } ccmd.__isset.ENCRYPTION_WITH_COLUMN_KEY = true; ccmd.__set_ENCRYPTION_WITH_COLUMN_KEY(eck); } @@ -1737,8 +1776,18 @@ class ColumnChunkMetaDataBuilder::ColumnChunkMetaDataBuilderImpl { ::arrow::util::span serialized_data_span(serialized_data, serialized_len); - std::vector encrypted_data(encryptor->CiphertextLength(serialized_len)); - int32_t encrypted_len = encryptor->Encrypt(serialized_data_span, encrypted_data); + std::vector encrypted_data; + int32_t encrypted_len; + if (encryptor->CanCalculateCiphertextLength()) { + encrypted_data = std::vector(encryptor->CiphertextLength(serialized_len)); + encrypted_len = encryptor->Encrypt(serialized_data_span, encrypted_data); + } else { + auto resizable_buffer = ::arrow::AllocateResizableBuffer(0); + encrypted_len = encryptor->EncryptWithManagedBuffer( + serialized_data_span, resizable_buffer->get()); + encrypted_data.assign(resizable_buffer->get()->data(), + resizable_buffer->get()->data() + encrypted_len); + } const char* temp = const_cast(reinterpret_cast(encrypted_data.data())); @@ -1771,6 +1820,16 @@ class ColumnChunkMetaDataBuilder::ColumnChunkMetaDataBuilderImpl { key_value_metadata_ = std::move(key_value_metadata); } + void AddKeyValueMetadata(std::shared_ptr key_value_metadata) { + if (key_value_metadata == nullptr) return; + + if (key_value_metadata_ == nullptr) { + key_value_metadata_ = std::move(key_value_metadata); + } else { + key_value_metadata_ = key_value_metadata_->Merge(*key_value_metadata); + } + } + private: void Init(format::ColumnChunk* column_chunk) { column_chunk_ = column_chunk; @@ -1839,6 +1898,10 @@ const ColumnDescriptor* ColumnChunkMetaDataBuilder::descr() const { return impl_->descr(); } +const WriterProperties* ColumnChunkMetaDataBuilder::properties() const { + return impl_->properties(); +} + void ColumnChunkMetaDataBuilder::SetStatistics(const EncodedStatistics& result) { impl_->SetStatistics(result); } @@ -1852,6 +1915,11 @@ void ColumnChunkMetaDataBuilder::SetGeoStatistics( impl_->SetGeoStatistics(result); } +void ColumnChunkMetaDataBuilder::AddKeyValueMetadata( + std::shared_ptr key_value_metadata) { +impl_->AddKeyValueMetadata(std::move(key_value_metadata)); +} + void ColumnChunkMetaDataBuilder::SetKeyValueMetadata( std::shared_ptr key_value_metadata) { impl_->SetKeyValueMetadata(std::move(key_value_metadata)); @@ -2087,7 +2155,7 @@ class FileMetaDataBuilder::FileMetaDataBuilderImpl { if (!algo.aad.supply_aad_prefix) { signing_algorithm.aad.aad_prefix = algo.aad.aad_prefix; } - signing_algorithm.algorithm = ParquetCipher::AES_GCM_V1; + signing_algorithm.algorithm = algo.algorithm; metadata_->__set_encryption_algorithm(ToThrift(signing_algorithm)); const std::string& footer_signing_key_metadata = diff --git a/cpp/src/parquet/metadata.h b/cpp/src/parquet/metadata.h index 3380adbf56aa..5a89fb5d8906 100644 --- a/cpp/src/parquet/metadata.h +++ b/cpp/src/parquet/metadata.h @@ -86,6 +86,8 @@ class PARQUET_EXPORT ColumnCryptoMetaData { std::shared_ptr path_in_schema() const; bool encrypted_with_footer_key() const; const std::string& key_metadata() const; + bool is_encryption_algorithm_set() const; + EncryptionAlgorithm encryption_algorithm() const; private: explicit ColumnCryptoMetaData(const uint8_t* metadata); @@ -147,6 +149,12 @@ class PARQUET_EXPORT ColumnChunkMetaData { std::shared_ptr size_statistics() const; std::shared_ptr geo_statistics() const; + // get the column descriptor + const ColumnDescriptor* descr() const; + + // get the reader properties + const ReaderProperties* properties() const; + Compression::type compression() const; // Indicate if the ColumnChunk compression is supported by the current // compiled parquet library. @@ -446,11 +454,15 @@ class PARQUET_EXPORT ColumnChunkMetaDataBuilder { // column geometry statistics void SetGeoStatistics(const geospatial::EncodedGeoStatistics& geo_stats); + void AddKeyValueMetadata(std::shared_ptr key_value_metadata); void SetKeyValueMetadata(std::shared_ptr key_value_metadata); // get the column descriptor const ColumnDescriptor* descr() const; + // get the writer properties + const WriterProperties* properties() const; + int64_t total_compressed_size() const; // commit the metadata diff --git a/cpp/src/parquet/page_index.cc b/cpp/src/parquet/page_index.cc index 5c2e78c68db5..3de91546caa2 100644 --- a/cpp/src/parquet/page_index.cc +++ b/cpp/src/parquet/page_index.cc @@ -17,7 +17,7 @@ #include "parquet/page_index.h" #include "parquet/encoding.h" -#include "parquet/encryption/encryption_internal.h" +#include "parquet/encryption/encryption_utils.h" #include "parquet/encryption/internal_file_decryptor.h" #include "parquet/encryption/internal_file_encryptor.h" #include "parquet/exception.h" diff --git a/cpp/src/parquet/parquet.thrift b/cpp/src/parquet/parquet.thrift index e3cc5adb9648..3ebee57015f7 100644 --- a/cpp/src/parquet/parquet.thrift +++ b/cpp/src/parquet/parquet.thrift @@ -940,6 +940,38 @@ struct ColumnMetaData { 17: optional GeospatialStatistics geospatial_statistics; } +struct AesGcmV1 { + /** AAD prefix **/ + 1: optional binary aad_prefix + + /** Unique file identifier part of AAD suffix **/ + 2: optional binary aad_file_unique + + /** In files encrypted with AAD prefix without storing it, + * readers must supply the prefix **/ + 3: optional bool supply_aad_prefix +} + +struct AesGcmCtrV1 { + /** AAD prefix **/ + 1: optional binary aad_prefix + + /** Unique file identifier part of AAD suffix **/ + 2: optional binary aad_file_unique + + /** In files encrypted with AAD prefix without storing it, + * readers must supply the prefix **/ + 3: optional bool supply_aad_prefix +} + +struct ExternalDBPAV1 {} + +union EncryptionAlgorithm { + 1: AesGcmV1 AES_GCM_V1 + 2: AesGcmCtrV1 AES_GCM_CTR_V1 + 3: ExternalDBPAV1 EXTERNAL_DBPA_V1 +} + struct EncryptionWithFooterKey { } @@ -949,6 +981,9 @@ struct EncryptionWithColumnKey { /** Retrieval metadata of column encryption key **/ 2: optional binary key_metadata + + /** Column specific encryption algorithm **/ + 3: optional EncryptionAlgorithm encryption_algorithm } union ColumnCryptoMetaData { @@ -1213,35 +1248,6 @@ struct ColumnIndex { 7: optional list definition_level_histograms; } -struct AesGcmV1 { - /** AAD prefix **/ - 1: optional binary aad_prefix - - /** Unique file identifier part of AAD suffix **/ - 2: optional binary aad_file_unique - - /** In files encrypted with AAD prefix without storing it, - * readers must supply the prefix **/ - 3: optional bool supply_aad_prefix -} - -struct AesGcmCtrV1 { - /** AAD prefix **/ - 1: optional binary aad_prefix - - /** Unique file identifier part of AAD suffix **/ - 2: optional binary aad_file_unique - - /** In files encrypted with AAD prefix without storing it, - * readers must supply the prefix **/ - 3: optional bool supply_aad_prefix -} - -union EncryptionAlgorithm { - 1: AesGcmV1 AES_GCM_V1 - 2: AesGcmCtrV1 AES_GCM_CTR_V1 -} - /** * Description for file metadata */ diff --git a/cpp/src/parquet/thrift_internal.h b/cpp/src/parquet/thrift_internal.h index 8f82adae928d..fd9b1a4e73dd 100644 --- a/cpp/src/parquet/thrift_internal.h +++ b/cpp/src/parquet/thrift_internal.h @@ -262,6 +262,11 @@ static inline AadMetadata FromThrift(format::AesGcmCtrV1 aesGcmCtrV1) { aesGcmCtrV1.supply_aad_prefix}; } +static inline AadMetadata FromThrift(format::ExternalDBPAV1 externalDBPAV1) { + // Set default values for AAD, which is not supported by ExternalDBPAV1 + return AadMetadata{/*aad_prefix*/"", /*aad_file_unique*/"", /*supply_aad_prefix*/false}; +} + static inline EncodedStatistics FromThrift(const format::Statistics& stats) { EncodedStatistics out; @@ -357,6 +362,9 @@ static inline EncryptionAlgorithm FromThrift(format::EncryptionAlgorithm encrypt } else if (encryption.__isset.AES_GCM_CTR_V1) { encryption_algorithm.algorithm = ParquetCipher::AES_GCM_CTR_V1; encryption_algorithm.aad = FromThrift(encryption.AES_GCM_CTR_V1); + } else if (encryption.__isset.EXTERNAL_DBPA_V1) { + encryption_algorithm.algorithm = ParquetCipher::EXTERNAL_DBPA_V1; + encryption_algorithm.aad = FromThrift(encryption.EXTERNAL_DBPA_V1); } else { throw ParquetException("Unsupported algorithm"); } @@ -533,12 +541,19 @@ static inline format::AesGcmCtrV1 ToAesGcmCtrV1Thrift(AadMetadata aad) { return aesGcmCtrV1; } +static inline format::ExternalDBPAV1 ToExternalDBPAV1Thrift() { + format::ExternalDBPAV1 externalDBPAV1; + return externalDBPAV1; +} + static inline format::EncryptionAlgorithm ToThrift(EncryptionAlgorithm encryption) { format::EncryptionAlgorithm encryption_algorithm; if (encryption.algorithm == ParquetCipher::AES_GCM_V1) { encryption_algorithm.__set_AES_GCM_V1(ToAesGcmV1Thrift(encryption.aad)); - } else { + } else if (encryption.algorithm == ParquetCipher::AES_GCM_CTR_V1) { encryption_algorithm.__set_AES_GCM_CTR_V1(ToAesGcmCtrV1Thrift(encryption.aad)); + } else { + encryption_algorithm.__set_EXTERNAL_DBPA_V1(ToExternalDBPAV1Thrift()); } return encryption_algorithm; } @@ -579,6 +594,15 @@ class ThriftDeserializer { // thrift message is not encrypted DeserializeUnencryptedMessage(buf, len, deserialized_msg); } else { + // This method is only used to deserialize metadata or footer data, so it is not expected + // to be called with a decryptor that can't calculate lengths. + if (!decryptor->CanCalculateLengths()) { + std::stringstream ss; + ss << "Decryptor can't calculate plaintext or ciphertext lengths when deserializing "; + ss << "metadata or footer and should not be used to deserialize metadata or footer data"; + throw ParquetException(ss.str()); + } + // thrift message is encrypted uint32_t clen; clen = *len; @@ -698,11 +722,17 @@ class ThriftSerializer { int64_t SerializeEncryptedObj(ArrowOutputStream* out, const uint8_t* out_buffer, uint32_t out_length, Encryptor* encryptor) { - auto cipher_buffer = - AllocateBuffer(encryptor->pool(), encryptor->CiphertextLength(out_length)); - ::arrow::util::span out_span(out_buffer, out_length); - int32_t cipher_buffer_len = - encryptor->Encrypt(out_span, cipher_buffer->mutable_span_as()); + int32_t cipher_buffer_len; + std::shared_ptr cipher_buffer; + if (encryptor->CanCalculateCiphertextLength()) { + cipher_buffer = AllocateBuffer(encryptor->pool(), encryptor->CiphertextLength(out_length)); + ::arrow::util::span out_span(out_buffer, out_length); + cipher_buffer_len = encryptor->Encrypt(out_span, cipher_buffer->mutable_span_as()); + } else { + cipher_buffer = AllocateBuffer(encryptor->pool(), 0); + ::arrow::util::span out_span(out_buffer, out_length); + cipher_buffer_len = encryptor->EncryptWithManagedBuffer(out_span, cipher_buffer.get()); + } PARQUET_THROW_NOT_OK(out->Write(cipher_buffer->data(), cipher_buffer_len)); return static_cast(cipher_buffer_len); diff --git a/cpp/src/parquet/types.cc b/cpp/src/parquet/types.cc index f545f5122018..799fe78d8e5c 100644 --- a/cpp/src/parquet/types.cc +++ b/cpp/src/parquet/types.cc @@ -59,6 +59,18 @@ bool IsCodecSupported(Compression::type codec) { } } +bool IsParquetCipherSupported(ParquetCipher::type cipher) { + switch (cipher) { + case ParquetCipher::AES_GCM_V1: + case ParquetCipher::AES_GCM_CTR_V1: + case ParquetCipher::EXTERNAL_DBPA_V1: + return true; + // There is no default case on purpose, so the compiler will warn if a new cipher is added + // without updating this function. + } + return false; +} + std::unique_ptr GetCodec(Compression::type codec) { return GetCodec(codec, CodecOptions()); } diff --git a/cpp/src/parquet/types.h b/cpp/src/parquet/types.h index 7e8a18fc94d6..93c35d278803 100644 --- a/cpp/src/parquet/types.h +++ b/cpp/src/parquet/types.h @@ -568,9 +568,13 @@ PARQUET_EXPORT std::unique_ptr GetCodec(Compression::type codec, int compression_level); struct ParquetCipher { - enum type { AES_GCM_V1 = 0, AES_GCM_CTR_V1 = 1 }; + enum type { AES_GCM_V1 = 0, AES_GCM_CTR_V1 = 1, EXTERNAL_DBPA_V1 = 2 }; }; +/// Check whether a requested encryption algorithm is supported by the Parquet library. +/// Used in the crypto factory to validate the requested encryption algorithm is available. +bool IsParquetCipherSupported(ParquetCipher::type cipher); + struct AadMetadata { std::string aad_prefix; std::string aad_file_unique; diff --git a/cpp/src/parquet/types_test.cc b/cpp/src/parquet/types_test.cc index 6c77662d58f7..c1f24a30993a 100644 --- a/cpp/src/parquet/types_test.cc +++ b/cpp/src/parquet/types_test.cc @@ -212,6 +212,12 @@ TEST(TestInt96Timestamp, Decoding) { check(2547339, 0xffffffffffffffffULL); } +TEST(TestIsParquetCipherSupported, SupportedCiphers) { + ASSERT_TRUE(IsParquetCipherSupported(ParquetCipher::AES_GCM_V1)); + ASSERT_TRUE(IsParquetCipherSupported(ParquetCipher::AES_GCM_CTR_V1)); + ASSERT_TRUE(IsParquetCipherSupported(ParquetCipher::EXTERNAL_DBPA_V1)); +} + #if !(defined(_WIN32) || defined(__CYGWIN__)) # pragma GCC diagnostic pop #elif _MSC_VER diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd index 94365f0f7c76..629fdb15d833 100644 --- a/python/pyarrow/_parquet.pxd +++ b/python/pyarrow/_parquet.pxd @@ -38,6 +38,21 @@ cdef class FileEncryptionProperties: cdef inline shared_ptr[CFileEncryptionProperties] unwrap(self): return self.properties +cdef class ExternalFileEncryptionProperties(FileEncryptionProperties): + cdef: + shared_ptr[CExternalFileEncryptionProperties] properties + + @staticmethod + cdef inline ExternalFileEncryptionProperties wrap_external( + shared_ptr[CExternalFileEncryptionProperties] properties): + + result = ExternalFileEncryptionProperties() + result.properties = properties + return result + + cdef inline shared_ptr[CExternalFileEncryptionProperties] unwrap_external(self): + return self.properties + cdef shared_ptr[WriterProperties] _create_writer_properties( use_dictionary=*, compression=*, @@ -150,3 +165,19 @@ cdef class FileDecryptionProperties: cdef inline shared_ptr[CFileDecryptionProperties] unwrap(self): return self.properties + +cdef class ExternalFileDecryptionProperties(FileDecryptionProperties): + """File-level decryption properties for the low-level API""" + cdef: + shared_ptr[CExternalFileDecryptionProperties] properties + + @staticmethod + cdef inline ExternalFileDecryptionProperties wrap_external( + shared_ptr[CExternalFileDecryptionProperties] properties): + + result = ExternalFileDecryptionProperties() + result.properties = properties + return result + + cdef inline shared_ptr[CExternalFileDecryptionProperties] unwrap_external(self): + return self.properties diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx index d59c70a27447..364472185cdf 100644 --- a/python/pyarrow/_parquet.pyx +++ b/python/pyarrow/_parquet.pyx @@ -47,6 +47,10 @@ from pyarrow.lib import (ArrowException, NativeFile, BufferOutputStream, tobytes, frombytes, is_threading_enabled) cimport cpython as cp +from libcpp.memory cimport shared_ptr + +cdef extern from "memory" namespace "std": + shared_ptr[T] static_pointer_cast[T, U](shared_ptr[U] r) _DEFAULT_ROW_GROUP_SIZE = 1024*1024 _MAX_ROW_GROUP_SIZE = 64*1024*1024 @@ -1626,8 +1630,12 @@ cdef class ParquetReader(_Weakrefable): thrift_container_size_limit) if decryption_properties is not None: - properties.file_decryption_properties( - decryption_properties.unwrap()) + if isinstance(decryption_properties, ExternalFileDecryptionProperties): + properties.file_decryption_properties( + static_pointer_cast[CFileDecryptionProperties, CExternalFileDecryptionProperties] ( + (decryption_properties).unwrap_external())) + else: + properties.file_decryption_properties((decryption_properties).unwrap()) arrow_props.set_pre_buffer(pre_buffer) @@ -2165,8 +2173,12 @@ cdef shared_ptr[WriterProperties] _create_writer_properties( # encryption if encryption_properties is not None: - props.encryption( - (encryption_properties).unwrap()) + if isinstance(encryption_properties, ExternalFileEncryptionProperties): + props.encryption( + static_pointer_cast[CFileEncryptionProperties, CExternalFileEncryptionProperties] ( + (encryption_properties).unwrap_external())) + else: + props.encryption((encryption_properties).unwrap()) # For backwards compatibility reasons we cap the maximum row group size # at 64Mi rows. This could be changed in the future, though it would be diff --git a/python/pyarrow/_parquet_encryption.pxd b/python/pyarrow/_parquet_encryption.pxd index d52669501a40..84ba44076cbd 100644 --- a/python/pyarrow/_parquet_encryption.pxd +++ b/python/pyarrow/_parquet_encryption.pxd @@ -22,11 +22,16 @@ from pyarrow.includes.common cimport * from pyarrow.includes.libparquet_encryption cimport * from pyarrow._parquet cimport (ParquetCipher, CFileEncryptionProperties, + CExternalFileEncryptionProperties, CFileDecryptionProperties, + CExternalFileDecryptionProperties, FileEncryptionProperties, + ExternalFileEncryptionProperties, FileDecryptionProperties, + ExternalFileDecryptionProperties, ParquetCipher_AES_GCM_V1, - ParquetCipher_AES_GCM_CTR_V1) + ParquetCipher_AES_GCM_CTR_V1, + ParquetCipher_EXTERNAL_DBPA_V1) from pyarrow.lib cimport _Weakrefable cdef class CryptoFactory(_Weakrefable): @@ -49,8 +54,18 @@ cdef class KmsConnectionConfig(_Weakrefable): @staticmethod cdef wrap(const CKmsConnectionConfig& config) +cdef class ExternalEncryptionConfiguration(EncryptionConfiguration): + cdef shared_ptr[CExternalEncryptionConfiguration] external_configuration + cdef inline shared_ptr[CExternalEncryptionConfiguration] unwrap_external(self) nogil + +cdef class ExternalDecryptionConfiguration(DecryptionConfiguration): + cdef shared_ptr[CExternalDecryptionConfiguration] external_configuration + cdef inline shared_ptr[CExternalDecryptionConfiguration] unwrap_external(self) nogil + cdef shared_ptr[CCryptoFactory] pyarrow_unwrap_cryptofactory(object crypto_factory) except * cdef shared_ptr[CKmsConnectionConfig] pyarrow_unwrap_kmsconnectionconfig(object kmsconnectionconfig) except * cdef shared_ptr[CEncryptionConfiguration] pyarrow_unwrap_encryptionconfig(object encryptionconfig) except * cdef shared_ptr[CDecryptionConfiguration] pyarrow_unwrap_decryptionconfig(object decryptionconfig) except * +cdef shared_ptr[CExternalEncryptionConfiguration] pyarrow_unwrap_external_encryptionconfig(object externalencryptionconfig) except * +cdef shared_ptr[CExternalDecryptionConfiguration] pyarrow_unwrap_external_decryptionconfig(object externaldecryptionconfig) except * diff --git a/python/pyarrow/_parquet_encryption.pyx b/python/pyarrow/_parquet_encryption.pyx index f95464e3031d..048be845e554 100644 --- a/python/pyarrow/_parquet_encryption.pyx +++ b/python/pyarrow/_parquet_encryption.pyx @@ -28,6 +28,7 @@ from pyarrow.includes.libarrow cimport * from pyarrow.lib cimport _Weakrefable from pyarrow.lib import tobytes, frombytes +import json cdef ParquetCipher cipher_from_name(name): name = name.upper() @@ -35,8 +36,10 @@ cdef ParquetCipher cipher_from_name(name): return ParquetCipher_AES_GCM_V1 elif name == 'AES_GCM_CTR_V1': return ParquetCipher_AES_GCM_CTR_V1 + elif name == 'EXTERNAL_DBPA_V1': + return ParquetCipher_EXTERNAL_DBPA_V1 else: - raise ValueError(f'Invalid cipher name: {name!r}') + raise ValueError(f'Invalid cipher name: {name}') cdef cipher_to_name(ParquetCipher cipher): @@ -44,6 +47,8 @@ cdef cipher_to_name(ParquetCipher cipher): return 'AES_GCM_V1' elif ParquetCipher_AES_GCM_CTR_V1 == cipher: return 'AES_GCM_CTR_V1' + elif ParquetCipher_EXTERNAL_DBPA_V1 == cipher: + return 'EXTERNAL_DBPA_V1' else: raise ValueError(f'Invalid cipher value: {cipher}') @@ -190,6 +195,215 @@ cdef class EncryptionConfiguration(_Weakrefable): cdef inline shared_ptr[CEncryptionConfiguration] unwrap(self) nogil: return self.configuration +cdef class ExternalEncryptionConfiguration(EncryptionConfiguration): + """ExternalEncryptionConfiguration inherits from EncryptionConfiguration.""" + __slots__ = () + + def __init__(self, footer_key, *, column_keys=None, + encryption_algorithm=None, + plaintext_footer=None, double_wrapping=None, + cache_lifetime=None, internal_key_material=None, + data_key_length_bits=None, per_column_encryption=None, + app_context=None, connection_config=None): + + # Initialize pointer first so the get/set forwards work. + self.external_configuration.reset( + new CExternalEncryptionConfiguration(tobytes(footer_key))) + + super().__init__(footer_key, + column_keys=column_keys, + encryption_algorithm=encryption_algorithm, + plaintext_footer=plaintext_footer, + double_wrapping=double_wrapping, + cache_lifetime=cache_lifetime, + internal_key_material=internal_key_material, + data_key_length_bits=data_key_length_bits) + + self.external_configuration.get().footer_key = \ + self.configuration.get().footer_key + + if app_context is not None: + self.app_context = app_context + if connection_config is not None: + self.connection_config = connection_config + if per_column_encryption is not None: + self.per_column_encryption = per_column_encryption + + """ Forward all attributes get/set methods to the superclass """ + """ The superclass already converts to/from bytes and does additional processing needed """ + @property + def column_keys(self): + return EncryptionConfiguration.column_keys.__get__(self) + + @column_keys.setter + def column_keys(self, dict value): + EncryptionConfiguration.column_keys.__set__(self, value) + self.external_configuration.get().column_keys = self.configuration.get().column_keys + + @property + def encryption_algorithm(self): + return EncryptionConfiguration.encryption_algorithm.__get__(self) + + @encryption_algorithm.setter + def encryption_algorithm(self, value): + EncryptionConfiguration.encryption_algorithm.__set__(self, value) + self.external_configuration.get().encryption_algorithm = \ + self.configuration.get().encryption_algorithm + + @property + def plaintext_footer(self): + return EncryptionConfiguration.plaintext_footer.__get__(self) + + @plaintext_footer.setter + def plaintext_footer(self, value): + EncryptionConfiguration.plaintext_footer.__set__(self, value) + self.external_configuration.get().plaintext_footer = value + + @property + def double_wrapping(self): + return EncryptionConfiguration.double_wrapping.__get__(self) + + @double_wrapping.setter + def double_wrapping(self, value): + EncryptionConfiguration.double_wrapping.__set__(self, value) + self.external_configuration.get().double_wrapping = value + + @property + def cache_lifetime(self): + return EncryptionConfiguration.cache_lifetime.__get__(self) + + @cache_lifetime.setter + def cache_lifetime(self, value): + EncryptionConfiguration.cache_lifetime.__set__(self, value) + self.external_configuration.get().cache_lifetime_seconds = value.total_seconds() + + @property + def internal_key_material(self): + return EncryptionConfiguration.internal_key_material.__get__(self) + + @internal_key_material.setter + def internal_key_material(self, value): + EncryptionConfiguration.internal_key_material.__set__(self, value) + self.external_configuration.get().internal_key_material = value + + @property + def data_key_length_bits(self): + return EncryptionConfiguration.data_key_length_bits.__get__(self) + + @data_key_length_bits.setter + def data_key_length_bits(self, value): + EncryptionConfiguration.data_key_length_bits.__set__(self, value) + self.external_configuration.get().data_key_length_bits = value + + @property + def app_context(self): + """Get the application context as a dictionary.""" + app_context_str = frombytes(self.external_configuration.get().app_context) + if not app_context_str: + return {} + try: + return json.loads(app_context_str) + except Exception: + raise ValueError(f"Invalid JSON stored in app_context: {app_context_str}") + + @app_context.setter + def app_context(self, dict value): + """Set the application context from a dictionary.""" + if value is None: + raise ValueError("app_context must be JSON-serializable") + + try: + serialized = json.dumps(value) + self.external_configuration.get().app_context = tobytes(serialized) + except Exception: + raise TypeError(f"Failed to serialize app_context: {repr(value)}") + + @property + def connection_config(self): + """Get the connection configuration as a Python dictionary.""" + + cdef pair[ParquetCipher, unordered_map[c_string, c_string]] outer_pair + cdef pair[c_string, c_string] inner_pair + result = {} + + for outer_pair in self.external_configuration.get().connection_config: + cipher_name = cipher_to_name(outer_pair.first) + inner_map = {} + for inner_pair in outer_pair.second: + inner_map[frombytes(inner_pair.first)] = frombytes(inner_pair.second) + result[cipher_name] = inner_map + + return result + + @connection_config.setter + def connection_config(self, dict value): + """Set the connection configuration from a Python dictionary.""" + if value is None: + raise ValueError("Connection config value cannot be None") + + cdef unordered_map[ParquetCipher, unordered_map[c_string, c_string]] cpp_map + cdef unordered_map[c_string, c_string] inner_cpp_map + cdef ParquetCipher cipher_enum + + for cipher_name, inner_dict in value.items(): + cipher_enum = cipher_from_name(cipher_name) + if not isinstance(inner_dict, dict): + raise TypeError(f"Inner value for cipher {cipher_name} must be a dict") + # Clear the map from the values of the previous iteration + inner_cpp_map.clear() + + for k, v in inner_dict.items(): + if not isinstance(k, str) or not isinstance(v, str): + raise TypeError("All inner config keys/values must be str") + inner_cpp_map[tobytes(k)] = tobytes(v) + cpp_map[cipher_enum] = inner_cpp_map + + self.external_configuration.get().connection_config = cpp_map + + @property + def per_column_encryption(self): + """Get the per_column_encryption as a Python dictionary.""" + + py_dict = {} + + for pair in self.external_configuration.get().per_column_encryption: + py_dict[frombytes(pair.first)] = { + "encryption_algorithm": cipher_to_name(pair.second.parquet_cipher), + "encryption_key": frombytes(pair.second.key_id) + } + + return py_dict + + @per_column_encryption.setter + def per_column_encryption(self, dict py_column_encryption): + """Set the per_column_encryption from a Python dictionary.""" + if py_column_encryption is None: + raise TypeError("per_column_encryption cannot be None") + + # Clear the existing C++ map first + self.external_configuration.get().per_column_encryption.clear() + + cdef CColumnEncryptionAttributes cpp_attrs + # Iterate over the Python dictionary + for py_key, py_attrs in py_column_encryption.items(): + if not isinstance(py_key, str) or not isinstance(py_attrs, dict): + raise TypeError("column_encryption keys must be strings and values must be dictionaries.") + + # Convert encryption_algorithm string to C++ ParquetCipher enum + if "encryption_algorithm" not in py_attrs or not isinstance(py_attrs["encryption_algorithm"], str): + raise ValueError("Each column must have 'encryption_algorithm' (string).") + + # Convert encryption_key string to C++ c_string + if "encryption_key" not in py_attrs or not isinstance(py_attrs["encryption_key"], str): + raise ValueError("Each column must have 'encryption_key' (string).") + + cpp_attrs.parquet_cipher = cipher_from_name(py_attrs["encryption_algorithm"]) + cpp_attrs.key_id = tobytes(py_attrs["encryption_key"]) + + self.external_configuration.get().per_column_encryption[tobytes(py_key)] = cpp_attrs + + cdef inline shared_ptr[CExternalEncryptionConfiguration] unwrap_external(self) nogil: + return self.external_configuration cdef class DecryptionConfiguration(_Weakrefable): """Configuration of the decryption, such as cache timeout.""" @@ -213,6 +427,101 @@ cdef class DecryptionConfiguration(_Weakrefable): cdef inline shared_ptr[CDecryptionConfiguration] unwrap(self) nogil: return self.configuration +cdef class ExternalDecryptionConfiguration(DecryptionConfiguration): + """Configuration of the external decryption""" + # Avoid mistakingly creating attributes + __slots__ = () + + def __init__(self, *, cache_lifetime=None, app_context=None, connection_config=None): + # Initialize the pointer first so the get/set forwards work. + # Super init will run the setters/getters below so we need the pointer to exist. + self.external_configuration.reset(new CExternalDecryptionConfiguration()) + super().__init__(cache_lifetime=cache_lifetime) + + self.external_configuration.get().cache_lifetime_seconds = \ + self.configuration.get().cache_lifetime_seconds + + if app_context is not None: + self.app_context = app_context + if connection_config is not None: + self.connection_config = connection_config + + """ Forward all attributes get/set methods to the superclass """ + """ The superclass already converts to/from bytes and does additional processing needed """ + @property + def cache_lifetime(self): + return DecryptionConfiguration.cache_lifetime.__get__(self) + + @cache_lifetime.setter + def cache_lifetime(self, value): + DecryptionConfiguration.cache_lifetime.__set__(self, value) + self.external_configuration.get().cache_lifetime_seconds = value.total_seconds() + + @property + def app_context(self): + """Get the application context as a dictionary.""" + app_context_str = frombytes(self.external_configuration.get().app_context) + if not app_context_str: + return {} + try: + return json.loads(app_context_str) + except Exception: + raise ValueError(f"Invalid JSON stored in app_context: {app_context_str}") + + @app_context.setter + def app_context(self, dict value): + """Set the application context from a dictionary.""" + if value is None: + raise ValueError("app_context must be JSON-serializable") + + try: + serialized = json.dumps(value) + self.external_configuration.get().app_context = tobytes(serialized) + except Exception: + raise TypeError(f"Failed to serialize app_context: {repr(value)}") + + @property + def connection_config(self): + """Get the connection configuration as a Python dictionary.""" + + cdef pair[ParquetCipher, unordered_map[c_string, c_string]] outer_pair + cdef pair[c_string, c_string] inner_pair + result = {} + + for outer_pair in self.external_configuration.get().connection_config: + cipher_name = cipher_to_name(outer_pair.first) + inner_map = {} + for inner_pair in outer_pair.second: + inner_map[frombytes(inner_pair.first)] = frombytes(inner_pair.second) + result[cipher_name] = inner_map + + return result + + @connection_config.setter + def connection_config(self, dict value): + """Set the connection configuration from a Python dictionary.""" + if value is None: + raise ValueError("Connection config value cannot be None") + + cdef unordered_map[ParquetCipher, unordered_map[c_string, c_string]] cpp_map + cdef unordered_map[c_string, c_string] inner_cpp_map + cdef ParquetCipher cipher_enum + + for cipher_name, inner_dict in value.items(): + cipher_enum = cipher_from_name(cipher_name) + if not isinstance(inner_dict, dict): + raise TypeError(f"Inner value for cipher {cipher_name} must be a dict") + inner_cpp_map.clear() + for k, v in inner_dict.items(): + if not isinstance(k, str) or not isinstance(v, str): + raise TypeError("All inner config keys/values must be str") + inner_cpp_map[tobytes(k)] = tobytes(v) + cpp_map[cipher_enum] = inner_cpp_map + + self.external_configuration.get().connection_config = cpp_map + + cdef inline shared_ptr[CExternalDecryptionConfiguration] unwrap_external(self) nogil: + return self.external_configuration cdef class KmsConnectionConfig(_Weakrefable): """Configuration of the connection to the Key Management Service (KMS)""" @@ -430,6 +739,21 @@ cdef class CryptoFactory(_Weakrefable): file_encryption_properties_result) return FileEncryptionProperties.wrap(file_encryption_properties) + def external_file_encryption_properties(self, + KmsConnectionConfig kms_connection_config, + ExternalEncryptionConfiguration external_encryption_config): + cdef: + CResult[shared_ptr[CExternalFileEncryptionProperties]] \ + external_file_encryption_properties_result + with nogil: + external_file_encryption_properties_result = \ + self.factory.get().SafeGetExternalFileEncryptionProperties( + deref(kms_connection_config.unwrap().get()), + deref(external_encryption_config.unwrap_external().get())) + external_file_encryption_properties = GetResultValue( + external_file_encryption_properties_result) + return ExternalFileEncryptionProperties.wrap_external(external_file_encryption_properties) + def file_decryption_properties( self, KmsConnectionConfig kms_connection_config, @@ -467,6 +791,40 @@ cdef class CryptoFactory(_Weakrefable): c_file_decryption_properties) return FileDecryptionProperties.wrap(file_decryption_properties) + def external_file_decryption_properties( + self, + KmsConnectionConfig kms_connection_config, + ExternalDecryptionConfiguration decryption_config): + """Create file decryption properties. + Parameters + ---------- + kms_connection_config : KmsConnectionConfig + Configuration of connection to KMS + decryption_config : ExternalDecryptionConfiguration + Configuration of the decryption, such as cache timeout and the information on how to + connect the external decryption service. + Returns + ------- + file_decryption_properties : ExternalFileDecryptionProperties + File decryption properties. + """ + cdef: + CExternalDecryptionConfiguration c_decryption_config + CResult[shared_ptr[CExternalFileDecryptionProperties]] \ + c_file_decryption_properties + if decryption_config is None: + c_decryption_config = CExternalDecryptionConfiguration() + else: + c_decryption_config = deref(decryption_config.unwrap_external().get()) + with nogil: + c_file_decryption_properties = \ + self.factory.get().SafeGetExternalFileDecryptionProperties( + deref(kms_connection_config.unwrap().get()), + c_decryption_config) + file_decryption_properties = GetResultValue( + c_file_decryption_properties) + return ExternalFileDecryptionProperties.wrap_external(file_decryption_properties) + def remove_cache_entries_for_token(self, access_token): self.factory.get().RemoveCacheEntriesForToken(tobytes(access_token)) @@ -500,3 +858,15 @@ cdef shared_ptr[CDecryptionConfiguration] pyarrow_unwrap_decryptionconfig(object if isinstance(decryptionconfig, DecryptionConfiguration): return ( decryptionconfig).unwrap() raise TypeError("Expected DecryptionConfiguration, got %s" % type(decryptionconfig)) + + +cdef shared_ptr[CExternalEncryptionConfiguration] pyarrow_unwrap_external_encryptionconfig(object encryptionconfig) except *: + if isinstance(encryptionconfig, ExternalEncryptionConfiguration): + return ( encryptionconfig).unwrap_external() + raise TypeError("Expected ExternalEncryptionConfiguration, got %s" % type(encryptionconfig)) + + +cdef shared_ptr[CExternalDecryptionConfiguration] pyarrow_unwrap_external_decryptionconfig(object decryptionconfig) except *: + if isinstance(decryptionconfig, ExternalDecryptionConfiguration): + return ( decryptionconfig).unwrap_external() + raise TypeError("Expected ExternalDecryptionConfiguration, got %s" % type(decryptionconfig)) diff --git a/python/pyarrow/includes/libparquet.pxd b/python/pyarrow/includes/libparquet.pxd index d9dd9d1aec94..e7524432822a 100644 --- a/python/pyarrow/includes/libparquet.pxd +++ b/python/pyarrow/includes/libparquet.pxd @@ -212,6 +212,7 @@ cdef extern from "parquet/api/schema.h" namespace "parquet" nogil: enum ParquetCipher" parquet::ParquetCipher::type": ParquetCipher_AES_GCM_V1" parquet::ParquetCipher::AES_GCM_V1" ParquetCipher_AES_GCM_CTR_V1" parquet::ParquetCipher::AES_GCM_CTR_V1" + ParquetCipher_EXTERNAL_DBPA_V1" parquet::ParquetCipher::EXTERNAL_DBPA_V1" struct AadMetadata: c_string aad_prefix @@ -630,3 +631,11 @@ cdef extern from "parquet/encryption/encryption.h" namespace "parquet" nogil: cdef cppclass CFileEncryptionProperties\ " parquet::FileEncryptionProperties": pass + + cdef cppclass CExternalFileDecryptionProperties\ + " parquet::ExternalFileDecryptionProperties": + pass + + cdef cppclass CExternalFileEncryptionProperties\ + " parquet::ExternalFileEncryptionProperties": + pass diff --git a/python/pyarrow/includes/libparquet_encryption.pxd b/python/pyarrow/includes/libparquet_encryption.pxd index 7e031925af60..0ac688cc4358 100644 --- a/python/pyarrow/includes/libparquet_encryption.pxd +++ b/python/pyarrow/includes/libparquet_encryption.pxd @@ -21,9 +21,12 @@ from pyarrow.includes.common cimport * from pyarrow.includes.libarrow cimport CSecureString from pyarrow._parquet cimport (ParquetCipher, CFileEncryptionProperties, + CExternalFileEncryptionProperties, CFileDecryptionProperties, + CExternalFileDecryptionProperties, ParquetCipher_AES_GCM_V1, - ParquetCipher_AES_GCM_CTR_V1) + ParquetCipher_AES_GCM_CTR_V1, + ParquetCipher_EXTERNAL_DBPA_V1) cdef extern from "parquet/encryption/kms_client.h" \ @@ -81,20 +84,54 @@ cdef extern from "parquet/encryption/crypto_factory.h" \ c_bool internal_key_material int32_t data_key_length_bits + cdef cppclass CColumnEncryptionAttributes\ + " parquet::encryption::ColumnEncryptionAttributes": + CColumnEncryptionAttributes() except + + ParquetCipher parquet_cipher + c_string key_id + + cdef cppclass CExternalEncryptionConfiguration\ + " parquet::encryption::ExternalEncryptionConfiguration": + CExternalEncryptionConfiguration(const c_string& footer_key) except + + c_string footer_key + c_string column_keys + ParquetCipher encryption_algorithm + c_bool plaintext_footer + c_bool double_wrapping + double cache_lifetime_seconds + c_bool internal_key_material + int32_t data_key_length_bits + unordered_map[c_string, CColumnEncryptionAttributes] per_column_encryption + c_string app_context + unordered_map[ParquetCipher, unordered_map[c_string, c_string]] connection_config + cdef cppclass CDecryptionConfiguration\ " parquet::encryption::DecryptionConfiguration": CDecryptionConfiguration() except + double cache_lifetime_seconds + cdef cppclass CExternalDecryptionConfiguration\ + " parquet::encryption::ExternalDecryptionConfiguration": + CExternalDecryptionConfiguration() except + + double cache_lifetime_seconds + c_string app_context + unordered_map[ParquetCipher, unordered_map[c_string, c_string]] connection_config + cdef cppclass CCryptoFactory" parquet::encryption::CryptoFactory": void RegisterKmsClientFactory( shared_ptr[CKmsClientFactory] kms_client_factory) except + shared_ptr[CFileEncryptionProperties] GetFileEncryptionProperties( const CKmsConnectionConfig& kms_connection_config, const CEncryptionConfiguration& encryption_config) except +* + shared_ptr[CExternalFileEncryptionProperties] GetExternalFileEncryptionProperties( + const CKmsConnectionConfig& kms_connection_config, + const CExternalEncryptionConfiguration& external_encryption_config) except +* shared_ptr[CFileDecryptionProperties] GetFileDecryptionProperties( const CKmsConnectionConfig& kms_connection_config, const CDecryptionConfiguration& decryption_config) except +* + shared_ptr[CExternalFileDecryptionProperties] GetExternalFileDecryptionProperties( + const CKmsConnectionConfig& kms_connection_config, + const CExternalDecryptionConfiguration& decryption_config) except +* void RemoveCacheEntriesForToken(const c_string& access_token) except + void RemoveCacheEntriesForAllTokens() except + @@ -126,7 +163,15 @@ cdef extern from "arrow/python/parquet_encryption.h" \ SafeGetFileEncryptionProperties( const CKmsConnectionConfig& kms_connection_config, const CEncryptionConfiguration& encryption_config) + CResult[shared_ptr[CExternalFileEncryptionProperties]] \ + SafeGetExternalFileEncryptionProperties( + const CKmsConnectionConfig& kms_connection_config, + const CExternalEncryptionConfiguration& external_encryption_config) CResult[shared_ptr[CFileDecryptionProperties]] \ SafeGetFileDecryptionProperties( const CKmsConnectionConfig& kms_connection_config, const CDecryptionConfiguration& decryption_config) + CResult[shared_ptr[CExternalFileDecryptionProperties]] \ + SafeGetExternalFileDecryptionProperties( + const CKmsConnectionConfig& kms_connection_config, + const CExternalDecryptionConfiguration& decryption_config) diff --git a/python/pyarrow/parquet/encryption.py b/python/pyarrow/parquet/encryption.py index df6eed913fa5..a9633710c719 100644 --- a/python/pyarrow/parquet/encryption.py +++ b/python/pyarrow/parquet/encryption.py @@ -17,7 +17,9 @@ # specific language governing permissions and limitations # under the License. from pyarrow._parquet_encryption import (CryptoFactory, # noqa + ExternalEncryptionConfiguration, EncryptionConfiguration, DecryptionConfiguration, + ExternalDecryptionConfiguration, KmsConnectionConfig, KmsClient) diff --git a/python/pyarrow/src/arrow/python/parquet_encryption.cc b/python/pyarrow/src/arrow/python/parquet_encryption.cc index 1016cdd3a375..fb8f17c989b5 100644 --- a/python/pyarrow/src/arrow/python/parquet_encryption.cc +++ b/python/pyarrow/src/arrow/python/parquet_encryption.cc @@ -84,6 +84,14 @@ PyCryptoFactory::SafeGetFileEncryptionProperties( this->GetFileEncryptionProperties(kms_connection_config, encryption_config)); } +arrow::Result> +PyCryptoFactory::SafeGetExternalFileEncryptionProperties( + const ::parquet::encryption::KmsConnectionConfig& kms_connection_config, + const ::parquet::encryption::ExternalEncryptionConfiguration& external_encryption_config) { + PARQUET_CATCH_AND_RETURN( + this->GetExternalFileEncryptionProperties(kms_connection_config, external_encryption_config)); +} + arrow::Result> PyCryptoFactory::SafeGetFileDecryptionProperties( const ::parquet::encryption::KmsConnectionConfig& kms_connection_config, @@ -92,6 +100,14 @@ PyCryptoFactory::SafeGetFileDecryptionProperties( this->GetFileDecryptionProperties(kms_connection_config, decryption_config)); } +arrow::Result> +PyCryptoFactory::SafeGetExternalFileDecryptionProperties( + const ::parquet::encryption::KmsConnectionConfig& kms_connection_config, + const ::parquet::encryption::ExternalDecryptionConfiguration& external_decryption_config) { + PARQUET_CATCH_AND_RETURN( + this->GetExternalFileDecryptionProperties(kms_connection_config, external_decryption_config)); +} + } // namespace encryption } // namespace parquet } // namespace py diff --git a/python/pyarrow/src/arrow/python/parquet_encryption.h b/python/pyarrow/src/arrow/python/parquet_encryption.h index 3e57a7619454..468889070019 100644 --- a/python/pyarrow/src/arrow/python/parquet_encryption.h +++ b/python/pyarrow/src/arrow/python/parquet_encryption.h @@ -118,6 +118,11 @@ class ARROW_PYTHON_PARQUET_ENCRYPTION_EXPORT PyCryptoFactory const ::parquet::encryption::KmsConnectionConfig& kms_connection_config, const ::parquet::encryption::EncryptionConfiguration& encryption_config); + arrow::Result> + SafeGetExternalFileEncryptionProperties( + const ::parquet::encryption::KmsConnectionConfig& kms_connection_config, + const ::parquet::encryption::ExternalEncryptionConfiguration& external_encryption_config); + /// The returned FileDecryptionProperties object will use the cache inside this /// CryptoFactory object, so please keep this /// CryptoFactory object alive along with the returned @@ -126,6 +131,14 @@ class ARROW_PYTHON_PARQUET_ENCRYPTION_EXPORT PyCryptoFactory SafeGetFileDecryptionProperties( const ::parquet::encryption::KmsConnectionConfig& kms_connection_config, const ::parquet::encryption::DecryptionConfiguration& decryption_config); + + /// The returned ExternalFileDecryptionProperties object will use the cache inside this + /// CryptoFactory object, so please keep this CryptoFactory object alive along with the returned + /// ExternalFileDecryptionProperties object. + arrow::Result> + SafeGetExternalFileDecryptionProperties( + const ::parquet::encryption::KmsConnectionConfig& kms_connection_config, + const ::parquet::encryption::ExternalDecryptionConfiguration& external_decryption_config); }; } // namespace encryption diff --git a/python/pyarrow/tests/parquet/test_external_encryption.py b/python/pyarrow/tests/parquet/test_external_encryption.py new file mode 100644 index 000000000000..708b5807406a --- /dev/null +++ b/python/pyarrow/tests/parquet/test_external_encryption.py @@ -0,0 +1,619 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import base64 +import datetime +import os +import platform +import pyarrow +import pyarrow.parquet as pp +import pyarrow.parquet.encryption as ppe +import pytest +import re + + +class FooKmsClient(ppe.KmsClient): + + def __init__(self, kms_connection_config): + ppe.KmsClient.__init__(self) + self.master_keys_map = kms_connection_config.custom_kms_conf + + def wrap_key(self, key_bytes, master_key_identifier): + master_key_bytes = self.master_keys_map[master_key_identifier].encode('utf-8') + joint_key = b"".join([master_key_bytes, key_bytes]) + return base64.b64encode(joint_key) + + def unwrap_key(self, wrapped_key, master_key_identifier): + expected_master = self.master_keys_map[master_key_identifier] + decoded_key = base64.b64decode(wrapped_key) + master_key_bytes = decoded_key[:16] + decrypted_key = decoded_key[16:] + if (expected_master == master_key_bytes.decode('utf-8')): + return decrypted_key + raise ValueError( + f"Bad master key used [{master_key_bytes}] - [{decrypted_key}]") + + +def get_data_table(): + sample_data = { + "orderId": [1001, 1002, 1003], + "productId": [152, 268, 6548], + "price": [3.25, 6.48, 2.12], + "vat": [0.0, 0.2, 0.05] + } + return pyarrow.Table.from_pydict(sample_data) + + +def kms_client_factory(kms_connection_config): + return FooKmsClient(kms_connection_config) + + +def get_agent_library_path(): + # TODO: move this code to a common library + # See https://github.com/protegrity/arrow/issues/191 + return os.environ.get( + 'DBPA_LIBRARY_PATH', + 'libDBPATestAgent.so' if platform.system() == 'Linux' else 'libDBPATestAgent.dylib') + + +def get_kms_connection_config(): + return ppe.KmsConnectionConfig( + custom_kms_conf={ + "footer_key": "012footer_secret", + "orderid_key": "column_secret001", + "productid_key": "column_secret002", + "price_key": "column_secret003", + "vat_key": "column_secret004" + } + ) + + +def get_encryption_config(): + return ppe.EncryptionConfiguration( + footer_key="footer_key", + column_keys={ + "orderid_key": ["orderId"], + "productid_key": ["productId"] + }, + encryption_algorithm="AES_GCM_V1", + cache_lifetime=datetime.timedelta(minutes=2.0), + data_key_length_bits=128, + plaintext_footer=True + ) + + +def get_encryption_properties(): + encryption_config = get_encryption_config() + crypto_factory = ppe.CryptoFactory(kms_client_factory) + return crypto_factory.file_encryption_properties( + get_kms_connection_config(), encryption_config) + + +def get_external_encryption_config(plaintext_footer=True): + return ppe.ExternalEncryptionConfiguration( + footer_key="footer_key", + column_keys={ + "productid_key": ["productId"] + }, + encryption_algorithm="AES_GCM_V1", + cache_lifetime=datetime.timedelta(minutes=2.0), + data_key_length_bits=128, + plaintext_footer=plaintext_footer, + per_column_encryption={ + "orderId": { + "encryption_algorithm": "EXTERNAL_DBPA_V1", + "encryption_key": "orderid_key" + }, + }, + app_context={ + "user_id": "Picard1701", + "location": "Presidio" + }, + connection_config={ + "EXTERNAL_DBPA_V1": { + "config_file": "path/to/config/file", + "config_file_decryption_key": "some_key", + "agent_library_path": get_agent_library_path() + } + } + ) + + +def get_external_encryption_properties(): + encryption_config = get_external_encryption_config() + crypto_factory = ppe.CryptoFactory(kms_client_factory) + return crypto_factory.external_file_encryption_properties( + get_kms_connection_config(), encryption_config) + + +def get_decryption_config(): + return ppe.DecryptionConfiguration(cache_lifetime=datetime.timedelta(minutes=10.0)) + + +def get_decryption_properties(): + decryption_config = get_decryption_config() + crypto_factory = ppe.CryptoFactory(kms_client_factory) + return crypto_factory.file_decryption_properties( + get_kms_connection_config(), decryption_config) + + +def get_external_decryption_config(): + return ppe.ExternalDecryptionConfiguration( + cache_lifetime=datetime.timedelta(minutes=10.0), + app_context={ + "user_id": "Picard1701", + "location": "Presidio" + }, + connection_config={ + "EXTERNAL_DBPA_V1": { + "config_file": "path/to/config/file", + "config_file_decryption_key": "some_key", + "agent_library_path": get_agent_library_path(), + } + } + ) + + +def get_external_decryption_properties(): + decryption_config = get_external_decryption_config() + crypto_factory = ppe.CryptoFactory(kms_client_factory) + return crypto_factory.external_file_decryption_properties( + get_kms_connection_config(), decryption_config) + + +def write_parquet(table, location, encryption_properties): + writer = pp.ParquetWriter( + location, + table.schema, + encryption_properties=encryption_properties) + writer.write_table(table) + + +def read_parquet(location, decryption_properties): + reader = pp.ParquetFile(location, decryption_properties=decryption_properties) + return reader.read() + + +def round_trip_encryption_and_decryption(tmp_path, encryption_properties, decryption_properties): + data_table = get_data_table() + parquet_path = tmp_path / "test.parquet" + write_parquet(data_table, parquet_path, encryption_properties) + + read_data_table = read_parquet(parquet_path, decryption_properties) + assert read_data_table.equals(data_table) + assert read_data_table.num_rows == data_table.num_rows + assert read_data_table.num_columns == data_table.num_columns + assert read_data_table.schema.equals(data_table.schema) + assert read_data_table.column_names == data_table.column_names + + +def test_encryption_configuration_properties(): + """Test the standard EncryptionConfiguration properties to avoid regressions.""" + + config = ppe.EncryptionConfiguration( + footer_key="footer-key-name", + column_keys={ + "key_1": ["a"], + }, + encryption_algorithm="EXTERNAL_DBPA_V1", + plaintext_footer=True, + double_wrapping=True, + cache_lifetime=datetime.timedelta(minutes=5.0), + internal_key_material=True, + data_key_length_bits=256 + ) + + assert isinstance(config, ppe.EncryptionConfiguration) + + assert config.footer_key == "footer-key-name" + assert config.column_keys == { + "key_1": ["a"] + } + assert config.encryption_algorithm == "EXTERNAL_DBPA_V1" + assert config.plaintext_footer is True + assert config.double_wrapping is True + assert config.cache_lifetime == datetime.timedelta(minutes=5.0) + assert config.internal_key_material is True + assert config.data_key_length_bits == 256 + + +def test_external_encryption_configuration_properties(): + """Test the ExternalEncryptionConfig including external-specific fields.""" + + external_encryption_config = get_external_encryption_config() + assert isinstance(external_encryption_config, ppe.ExternalEncryptionConfiguration) + + assert external_encryption_config.footer_key == "footer_key" + assert external_encryption_config.column_keys == { + "productid_key": ["productId"] + } + assert external_encryption_config.encryption_algorithm == "AES_GCM_V1" + assert external_encryption_config.plaintext_footer is True + assert external_encryption_config.double_wrapping is True + assert external_encryption_config.cache_lifetime == datetime.timedelta(minutes=2.0) + assert external_encryption_config.internal_key_material is True + assert external_encryption_config.data_key_length_bits == 128 + + assert external_encryption_config.app_context == { + "user_id": "Picard1701", + "location": "Presidio" + } + + assert external_encryption_config.connection_config == { + "EXTERNAL_DBPA_V1": { + "config_file": "path/to/config/file", + "config_file_decryption_key": "some_key", + "agent_library_path": get_agent_library_path() + } + } + + assert external_encryption_config.per_column_encryption == { + "orderId": { + "encryption_algorithm": "EXTERNAL_DBPA_V1", + "encryption_key": "orderid_key" + }, + } + + +def test_external_encryption_app_context_invalid_json(): + """Ensure app_context raises TypeError for non-JSON-serializable input.""" + with pytest.raises( + TypeError, + match="Failed to serialize app_context: {'invalid': {1, 2, 3}}" + ): + ppe.ExternalEncryptionConfiguration( + footer_key="key", + app_context={"invalid": set([1, 2, 3])} # sets are not JSON-serializable + ) + + +def test_external_encryption_per_column_encryption_invalid_algorithm(): + """Ensure invalid encryption_algorithm raises a ValueError or is rejected.""" + + with pytest.raises(ValueError, match="Invalid cipher name: INVALID"): + ppe.ExternalEncryptionConfiguration( + footer_key="key", + per_column_encryption={ + "a": { + "encryption_algorithm": "INVALID", + "encryption_key": "some_key" + } + } + ) + + +def test_external_encryption_per_column_encryption_new_algorithm(): + """Ensure new encryption_algorithm is accepted.""" + + ppe.ExternalEncryptionConfiguration( + footer_key="key", + per_column_encryption={ + "a": { + "encryption_algorithm": "EXTERNAL_DBPA_V1", + "encryption_key": "key_2" + } + } + ) + + +def test_external_encryption_connection_config_invalid_types(): + """Ensure connection_config rejects non-string keys or values.""" + with pytest.raises( + TypeError, + match="All inner config keys/values must be str"): + config = ppe.ExternalEncryptionConfiguration( + footer_key="key" + ) + config.connection_config = { + "EXTERNAL_DBPA_V1": { + "config_file": "path/to/file", + 123: "should-fail" # Invalid: key is not a string + } + } + + with pytest.raises( + TypeError, + match="All inner config keys/values must be str"): + config = ppe.ExternalEncryptionConfiguration( + footer_key="key" + ) + config.connection_config = { + "EXTERNAL_DBPA_V1": { + "config_file": ["not", "a", "string"] # Invalid: value is not a string + } + } + + +def test_external_encryption_rejects_none_values(): + """Ensure None values are rejected.""" + config = ppe.ExternalEncryptionConfiguration(footer_key="key") + + # per_column_encryption: expect ValueError + with pytest.raises(TypeError, match="per_column_encryption cannot be None"): + config.per_column_encryption = None + + # app_context: expect ValueError due to None not being JSON-serializable + with pytest.raises(ValueError, match="app_context must be JSON-serializable"): + config.app_context = None + + # connection_config: expect ValueError due to None not being iterable + with pytest.raises(ValueError, match="Connection config value cannot be None"): + config.connection_config = None + + +def test_external_file_encryption_properties_rejects_column_in_two_places(): + """Ensure a column cannot be defined in both column_keys + and per_column_encryption.""" + config = ppe.ExternalEncryptionConfiguration( + footer_key="footer_key", + column_keys={"orderid_key": ["a"]}, + per_column_encryption={"a": { + "encryption_algorithm": "AES_GCM_V1", + "encryption_key": "key_2" + }}, + ) + factory = ppe.CryptoFactory(kms_client_factory) + with pytest.raises( + OSError, + match=re.escape("Multiple keys defined for column [a]") + ): + factory.external_file_encryption_properties(get_kms_connection_config(), config) + + +def test_external_file_encryption_properties_valid(): + """Check class name and module because + ExternalFileEncryptionProperties is not visible.""" + external_encryption_properties = get_external_encryption_properties() + + assert ( + external_encryption_properties.__class__.__name__ + == "ExternalFileEncryptionProperties" + ) + assert external_encryption_properties.__class__.__module__ == "pyarrow._parquet" + + +def test_decryption_configuration_properties(): + """Test the standard DecryptionConfiguration properties to avoid regressions.""" + + config = ppe.DecryptionConfiguration() + config.cache_lifetime = datetime.timedelta(minutes=5.0) + + assert isinstance(config, ppe.DecryptionConfiguration) + assert config.cache_lifetime == datetime.timedelta(minutes=5.0) + + +def test_external_decryption_configuration_properties(): + """Test the ExternalDecryptionConfiguration properties + including external-specific fields.""" + + external_decryption_config = get_external_decryption_config() + assert isinstance(external_decryption_config, ppe.ExternalDecryptionConfiguration) + assert external_decryption_config.cache_lifetime == datetime.timedelta(minutes=10.0) + assert external_decryption_config.app_context == { + "user_id": "Picard1701", + "location": "Presidio" + } + assert external_decryption_config.connection_config == { + "EXTERNAL_DBPA_V1": { + "config_file": "path/to/config/file", + "config_file_decryption_key": "some_key", + "agent_library_path": get_agent_library_path() + } + } + + +def test_external_decryption_connection_config_invalid_types(): + """Ensure connection_config rejects non-string keys or values.""" + + # Outer key is not a string (int instead of cipher name string) + with pytest.raises(AttributeError, match="'int' object has no attribute 'upper'"): + config = ppe.ExternalDecryptionConfiguration() + config.connection_config = { + 123: { # invalid outer key + "config_file": "should-fail" + } + } + + # Outer value is not a dict + with pytest.raises( + TypeError, + match="Inner value for cipher AES_GCM_V1 must be a dict" + ): + config = ppe.ExternalDecryptionConfiguration() + config.connection_config = { + "AES_GCM_V1": ["not", "a", "dict"] # invalid outer value (should be dict) + } + + # Inner key is not a string + with pytest.raises(TypeError, match="All inner config keys/values must be str"): + config = ppe.ExternalDecryptionConfiguration() + config.connection_config = { + "AES_GCM_V1": { + 123: "should-fail" # invalid inner key + } + } + + # Inner value is not a string + with pytest.raises(TypeError, match="All inner config keys/values must be str"): + config = ppe.ExternalDecryptionConfiguration() + config.connection_config = { + "AES_GCM_V1": { + "config_file": ["not", "a", "string"] # invalid inner value + } + } + + +def test_external_file_decryption_properties_valid(): + """Check class name and module because + ExternalFileDecryptionProperties is not visible.""" + + external_decryption_properties = get_external_decryption_properties() + + assert ( + external_decryption_properties.__class__.__name__ + == "ExternalFileDecryptionProperties" + ) + assert external_decryption_properties.__class__.__module__ == "pyarrow._parquet" + + +def test_read_and_write_standard_encryption(tmp_path): + # Test a roundtrip encryption and decryption using standard encryption. + round_trip_encryption_and_decryption(tmp_path, get_encryption_properties(), + get_decryption_properties()) + + +def test_read_and_write_external_encryption(tmp_path): + # Test a roundtrip encryption and decryption using external encryption. + round_trip_encryption_and_decryption(tmp_path, get_external_encryption_properties(), + get_external_decryption_properties()) + + +def get_custom_external_encryption_properties(encryption_algorithm, per_column_encryption, + plaintext_footer): + encryption_config = ppe.ExternalEncryptionConfiguration( + footer_key="footer_key", + column_keys={ + "productid_key": ["productId"] + }, + encryption_algorithm=encryption_algorithm, + cache_lifetime=datetime.timedelta(minutes=2.0), + data_key_length_bits=128, + plaintext_footer=plaintext_footer, + per_column_encryption=per_column_encryption, + app_context={ + "user_id": "Picard1701", + "location": "Presidio" + }, + connection_config={ + "EXTERNAL_DBPA_V1": { + "config_file": "path/to/config/file", + "config_file_decryption_key": "some_key", + "agent_library_path": get_agent_library_path() + } + } + ) + crypto_factory = ppe.CryptoFactory(kms_client_factory) + return crypto_factory.external_file_encryption_properties( + get_kms_connection_config(), encryption_config) + + +def test_encrypt_aes_gcm_file_all_algorithms_in_columns_plaintext_footer(tmp_path): + encryption_properties = get_custom_external_encryption_properties( + "AES_GCM_V1", # encryption_algorithm + { + "orderId": { + "encryption_algorithm": "AES_GCM_CTR_V1", + "encryption_key": "orderid_key" + }, + "price": { + "encryption_algorithm": "EXTERNAL_DBPA_V1", + "encryption_key": "price_key" + }, + "vat": { + "encryption_algorithm": "AES_GCM_V1", + "encryption_key": "vat_key" + }, + }, # per_column_encryption + True # plaintext_footer + ) + round_trip_encryption_and_decryption(tmp_path, encryption_properties, + get_external_decryption_properties()) + + +def test_encrypt_aes_gcm_file_all_algorithms_in_columns_encrypted_footer(tmp_path): + encryption_properties = get_custom_external_encryption_properties( + "AES_GCM_V1", # encryption_algorithm + { + "orderId": { + "encryption_algorithm": "AES_GCM_CTR_V1", + "encryption_key": "orderid_key" + }, + "price": { + "encryption_algorithm": "EXTERNAL_DBPA_V1", + "encryption_key": "price_key" + }, + "vat": { + "encryption_algorithm": "AES_GCM_V1", + "encryption_key": "vat_key" + }, + }, # per_column_encryption + False # plaintext_footer + ) + round_trip_encryption_and_decryption(tmp_path, encryption_properties, + get_external_decryption_properties()) + + +def test_encrypt_aes_gcm_ctr_file_all_algorithms_in_columns_plaintext_footer(tmp_path): + encryption_properties = get_custom_external_encryption_properties( + "AES_GCM_CTR_V1", # encryption_algorithm + { + "orderId": { + "encryption_algorithm": "AES_GCM_CTR_V1", + "encryption_key": "orderid_key" + }, + "price": { + "encryption_algorithm": "EXTERNAL_DBPA_V1", + "encryption_key": "price_key" + }, + "vat": { + "encryption_algorithm": "AES_GCM_V1", + "encryption_key": "vat_key" + }, + }, # per_column_encryption + True # plaintext_footer + ) + round_trip_encryption_and_decryption(tmp_path, encryption_properties, + get_external_decryption_properties()) + + +def test_encrypt_aes_gcm_ctr_file_all_algorithms_in_columns_encrypted_footer(tmp_path): + encryption_properties = get_custom_external_encryption_properties( + "AES_GCM_CTR_V1", # encryption_algorithm + { + "orderId": { + "encryption_algorithm": "AES_GCM_CTR_V1", + "encryption_key": "orderid_key" + }, + "price": { + "encryption_algorithm": "EXTERNAL_DBPA_V1", + "encryption_key": "price_key" + }, + "vat": { + "encryption_algorithm": "AES_GCM_V1", + "encryption_key": "vat_key" + }, + }, # per_column_encryption + False # plaintext_footer + ) + round_trip_encryption_and_decryption(tmp_path, encryption_properties, + get_external_decryption_properties()) + + +def test_encrypt_external_dbpa_file_raises_error(tmp_path): + encryption_properties = get_custom_external_encryption_properties( + "EXTERNAL_DBPA_V1", # encryption_algorithm + { + "orderId": { + "encryption_algorithm": "AES_GCM_CTR_V1", + "encryption_key": "orderid_key" + }, + }, # per_column_encryption + True # plaintext_footer + ) + with pytest.raises(ValueError, match="Parquet crypto signature verification failed"): + round_trip_encryption_and_decryption(tmp_path, encryption_properties, + get_external_decryption_properties()) \ No newline at end of file diff --git a/python/scripts/base_app.py b/python/scripts/base_app.py new file mode 100644 index 000000000000..a8858ca9e59e --- /dev/null +++ b/python/scripts/base_app.py @@ -0,0 +1,347 @@ +""" +base_app.py + +@author sbrenes +""" + +import base64 +import datetime +import os +import pyarrow +import pyarrow.parquet as pp +import pyarrow.parquet.encryption as ppe +import platform + +class FooKmsClient(ppe.KmsClient): + + def __init__(self, kms_connection_config): + ppe.KmsClient.__init__(self) + self.master_keys_map = kms_connection_config.custom_kms_conf + + def wrap_key(self, key_bytes, master_key_identifier): + master_key_bytes = self.master_keys_map[master_key_identifier].encode('utf-8') + joint_key = b"".join([master_key_bytes, key_bytes]) + return base64.b64encode(joint_key) + + def unwrap_key(self, wrapped_key, master_key_identifier): + expected_master = self.master_keys_map[master_key_identifier] + decoded_key = base64.b64decode(wrapped_key) + master_key_bytes = decoded_key[:16] + decrypted_key = decoded_key[16:] + if (expected_master == master_key_bytes.decode('utf-8')): + return decrypted_key + raise ValueError(f"Bad master key used [{master_key_bytes}] - [{decrypted_key}]") + + +def kms_client_factory(kms_connection_config): + return FooKmsClient(kms_connection_config) + + +def write_parquet(table, location, encryption_config=None): + encryption_properties = None + + if encryption_config: + crypto_factory = ppe.CryptoFactory(kms_client_factory) + encryption_properties = crypto_factory.external_file_encryption_properties( + get_kms_connection_config(), encryption_config) + + # Change scenario ID to test different cases. + # https://github.com/protegrity/arrow/issues/204 for more details. + + scenario_id = 5 + + match scenario_id: + case 1: + # Case 1: Uncompressed data, using plain data encoding. + print("\n!!! Writing uncompressed data, using plain data encoding. !!!\n") + pp.write_table(table, location, use_dictionary=False, + encryption_properties=encryption_properties, + compression="NONE") + case 2: + # Case 2: Compressed data, using RLE dictionary encoding. + print("\n!!! Writing compressed data, using RLE dictionary encoding. !!!\n") + pp.write_table(table, location, use_dictionary=True, + encryption_properties=encryption_properties, + compression="SNAPPY") + case 3: + # Case 3: Uncompressed data, using RLE dictionary encoding. + print("\n!!! Writing uncompressed data, using RLE dictionary encoding. !!!\n") + pp.write_table(table, location, use_dictionary=True, + encryption_properties=encryption_properties, + compression="NONE") + + case 4: + # Case 4: Compressed data, using plain data encoding and data page version 1.0. + print("\n!!! Writing compressed data, using plain data encoding and page version 1.0. !!!\n") + pp.write_table(table, location, data_page_version="1.0", use_dictionary=False, + encryption_properties=encryption_properties, compression="SNAPPY") + + case 5: + # Case 5: Compressed data, using plain data encoding and data page version 2.0. + print("\n!!! Writing compressed data, using plain data encoding and page version 2.0. !!!\n") + pp.write_table(table, location, data_page_version="2.0", use_dictionary=False, + encryption_properties=encryption_properties, compression="SNAPPY") + + case 6: + # Case 6: Compressed data (using unsupported compression), using plain data encoding and data page version 2.0. + print("\n!!! Writing compressed data (using unsupported compression), using plain data encoding and page version 2.0. !!!\n") + pp.write_table(table, location, data_page_version="2.0", use_dictionary=False, + encryption_properties=encryption_properties, compression="GZIP") + + case _: + raise ValueError(f"Invalid scenario ID: {scenario_id}") + +def encrypted_data_and_footer_sample(data_table): + parquet_path = "sample.parquet" + encryption_config = get_external_encryption_config() + write_parquet(data_table, parquet_path, + encryption_config=encryption_config) + print(f"Written to [{parquet_path}]") + + +def create_and_encrypt_parquet(): + sample_data = { + "orderId": [1001, 1002, 1003], + "productId": [152, 268, 6548], + "price": [3.25, 6.48, 2.12], + "vat": [0.0, 0.2, 0.05], + "customer_name": ["Alice", "Bob", "Charlotte"], + "has_subscription": [True, False, True] + } + data_table = pyarrow.Table.from_pydict(sample_data) + + print("\nWriting parquet.") + + encrypted_data_and_footer_sample(data_table) + + +def read_and_print_parquet(): + print("\n-----------------------------------------------\nNow reading parquet file") + parquet_path = "sample.parquet" + + metadata = pp.read_metadata(parquet_path) + print("\nMetadata:") + print(metadata) + print("\n") + + decryption_config = get_external_decryption_config() + read_data_table = read_parquet(parquet_path, + decryption_config=decryption_config) + data_frame = read_data_table.to_pandas() + print("\nDecrypted data:") + print(data_frame.head()) + print("\n") + + +def read_and_print_dbps_metadata(): + """ + Read and print DBPS encryption metadata from a Parquet file. + + DBPS metadata is stored in the column chunk's key-value metadata. + This includes information like: + - dbps_agent_version: The version of DBPS used for encryption + - encryption_mode: The encryption mode (e.g., "PER_VALUE", "PER_BLOCK") + """ + print("\n-----------------------------------------------\nReading DBPS metadata from parquet file") + parquet_path = "sample.parquet" + + # Read metadata (decryption properties needed if metadata is encrypted) + decryption_config = get_external_decryption_config() + crypto_factory = ppe.CryptoFactory(kms_client_factory) + decryption_properties = crypto_factory.external_file_decryption_properties( + get_kms_connection_config(), decryption_config) + + parquet_file = pp.ParquetFile(parquet_path, decryption_properties=decryption_properties) + metadata = parquet_file.metadata + + print(f"\nFile has {metadata.num_row_groups} row group(s)") + print(f"File has {metadata.num_columns} column(s)\n") + + # Iterate through all row groups and columns to find DBPS metadata + for row_group_idx in range(metadata.num_row_groups): + row_group = metadata.row_group(row_group_idx) + print(f"Row Group {row_group_idx}:") + + for col_idx in range(row_group.num_columns): + column_chunk = row_group.column(col_idx) + column_name = column_chunk.path_in_schema + + # Access the key-value metadata (this is where DBPS metadata is stored) + kv_metadata = column_chunk.metadata + + if kv_metadata is not None: + print(f" Column '{column_name}':") + print(f" Has metadata: Yes") + + # Convert bytes keys/values to strings for display + metadata_dict = {} + for key, value in kv_metadata.items(): + try: + key_str = key.decode('utf-8') if isinstance(key, bytes) else key + value_str = value.decode('utf-8') if isinstance(value, bytes) else value + metadata_dict[key_str] = value_str + except: + metadata_dict[str(key)] = str(value) + + # Print DBPS-specific metadata + dbps_keys = ['dbps_agent_version', 'encrypt_mode_dict_page', 'encrypt_mode_data_page'] + has_dbps_metadata = False + for key in dbps_keys: + if key in metadata_dict: + print(f" {key}: {metadata_dict[key]}") + has_dbps_metadata = True + + # Print all metadata if there are other keys + if not has_dbps_metadata and metadata_dict: + print(f" All metadata: {metadata_dict}") + elif metadata_dict: + # Print any additional metadata keys + other_keys = [k for k in metadata_dict.keys() if k not in dbps_keys] + if other_keys: + print(f" Other metadata: {dict((k, metadata_dict[k]) for k in other_keys)}") + else: + print(f" Column '{column_name}': No metadata") + print() + + +def read_parquet(location, decryption_config=None, read_metadata=False): + decryption_properties = None + + if decryption_config: + crypto_factory = ppe.CryptoFactory(kms_client_factory) + decryption_properties = crypto_factory.external_file_decryption_properties( + get_kms_connection_config(), decryption_config) + + if read_metadata: + metadata = pp.read_metadata(location, decryption_properties=decryption_properties) + return metadata + + data_table = pp.ParquetFile(location, decryption_properties=decryption_properties).read() + return data_table + + +def get_kms_connection_config(): + return ppe.KmsConnectionConfig( + custom_kms_conf={ + "footer_key": "012footer_secret", + "orderid_key": "column_secret001", + "productid_key": "column_secret002", + "price_key": "column_secret003", + "customer_key": "column_secret004", + "has_subscription_key": "column_secret005" + } + ) + +def get_external_encryption_config(plaintext_footer=True): + return ppe.ExternalEncryptionConfiguration( + footer_key = "footer_key", + column_keys = { + "productid_key": ["productId"] + }, + encryption_algorithm = "AES_GCM_V1", + cache_lifetime=datetime.timedelta(minutes=2.0), + data_key_length_bits = 128, + plaintext_footer=plaintext_footer, + per_column_encryption = { + "orderId": { + "encryption_algorithm": "EXTERNAL_DBPA_V1", + "encryption_key": "orderid_key" + }, + "price": { + "encryption_algorithm": "EXTERNAL_DBPA_V1", #"AES_GCM_CTR_V1", + "encryption_key": "price_key" + }, + "customer_name": { + "encryption_algorithm": "EXTERNAL_DBPA_V1", + "encryption_key": "customer_key" + }#, intentionally left out to test per-column encryption for the 'has_subscription' column. + # "has_subscription": { + # "encryption_algorithm": "EXTERNAL_DBPA_V1", + # "encryption_key": "has_subscription_key" + # } + }, + app_context = { + "user_id": "Picard1701", + "location": "Presidio" + }, + connection_config = get_dbpa_connection_config() + ) + +def get_encryption_config(plaintext_footer=True): + return ppe.EncryptionConfiguration( + footer_key = "footer_key", + column_keys = { + "orderid_key": ["orderId"], + "productid_key": ["productId"] + }, + encryption_algorithm = "AES_GCM_CTR_V1", + cache_lifetime=datetime.timedelta(minutes=2.0), + data_key_length_bits = 128, + plaintext_footer=plaintext_footer + ) + +def get_decryption_config(): + return ppe.DecryptionConfiguration(cache_lifetime=datetime.timedelta(minutes=2.0)) + +def get_external_decryption_config(): + return ppe.ExternalDecryptionConfiguration( + cache_lifetime=datetime.timedelta(minutes=2.0), + app_context = { + "user_id": "Picard1701", + "location": "Presidio" + }, + connection_config = get_dbpa_connection_config() + ) + +def get_config_file(): + config_file_name = os.environ.get('DBPA_CONFIG_FILE_NAME', 'test_connection_config_file.json') + + #verify if the file exists (assuming full path) + if os.path.exists(config_file_name): + return config_file_name + + #did not find the file. check if it exists in the same directory as the script + script_directory = os.path.dirname(os.path.abspath(__file__)) + config_path = os.path.join(script_directory, config_file_name) + if os.path.exists(config_path): + return config_path + + #did not find the file. return None and let the caller handle it. + #throw an error + + raise FileNotFoundError(f"Connection config [{config_file_name}] file not found") + + +def get_dbpa_connection_config(): + #we read the name of the external DBPA agent library from the environment variable DBPA_LIBRARY_PATH. + #if not available, we use the default to 'libDBPATestAgent.so'. + #this library performs key-independent, XOR encryption/decryption, and is built as part of the Parquet Arrow tests. + #It is located in cpp/src/parquet/encryption/external/dbpa_test_agent.cc + agent_library_path = os.environ.get( + 'DBPA_LIBRARY_PATH', + 'libDBPATestAgent.so' if platform.system() == 'Linux' else 'libDBPATestAgent.dylib') + + connection_config = { + "EXTERNAL_DBPA_V1": { + "agent_library_path": agent_library_path, + "agent_init_timeout_ms": "15000", + "agent_encrypt_timeout_ms": "35000", + "agent_decrypt_timeout_ms": "35000" + } + } + + #TODO: need a better way to perform this check + config_file_required = "remote" in agent_library_path.lower() + + if (config_file_required): + config_path = get_config_file() + connection_config["EXTERNAL_DBPA_V1"]["connection_config_file_path"] = config_path + + return connection_config + + +if __name__ == "__main__": + create_and_encrypt_parquet() + read_and_print_parquet() + read_and_print_dbps_metadata() + print("\nPlayground finished!\n") \ No newline at end of file diff --git a/python/scripts/test_connection_config_file.json b/python/scripts/test_connection_config_file.json new file mode 100644 index 000000000000..e881ac7c1a9e --- /dev/null +++ b/python/scripts/test_connection_config_file.json @@ -0,0 +1,3 @@ +{ + "server_url": "http://localhost:8080" +} \ No newline at end of file diff --git a/python/scripts/use_external_dbpa_encryption.py b/python/scripts/use_external_dbpa_encryption.py new file mode 100644 index 000000000000..4e4c7dc19160 --- /dev/null +++ b/python/scripts/use_external_dbpa_encryption.py @@ -0,0 +1,314 @@ +# ################################################################################################ + +""" +use_external_dbpa_encryption.py +Use this script as a template/guide to use the external DBPA encryption library in Parquet Arrow. +""" + +import base64 +import datetime +import platform +import pyarrow +import pyarrow.parquet as pp +import pyarrow.parquet.encryption as ppe + +# ################################################################################################ + +""" +A sample KMS client that uses a map of master keys to wrap and unwrap keys. +Replace this with a real KMS client or provider, the same way you would provide this for +regular Parquet Arrow encryption. +Make sure you include the column keys in the custom_kms_conf, even for those columms that will +be encrypted with the external DBPA agent. +""" +class FooKmsClient(ppe.KmsClient): + + def __init__(self, kms_connection_config): + ppe.KmsClient.__init__(self) + self.master_keys_map = kms_connection_config.custom_kms_conf + + def wrap_key(self, key_bytes, master_key_identifier): + master_key_bytes = self.master_keys_map[master_key_identifier].encode('utf-8') + joint_key = b"".join([master_key_bytes, key_bytes]) + return base64.b64encode(joint_key) + + def unwrap_key(self, wrapped_key, master_key_identifier): + expected_master = self.master_keys_map[master_key_identifier] + decoded_key = base64.b64decode(wrapped_key) + master_key_bytes = decoded_key[:16] + decrypted_key = decoded_key[16:] + if (expected_master == master_key_bytes.decode('utf-8')): + return decrypted_key + raise ValueError(f"Bad master key used [{master_key_bytes}] - [{decrypted_key}]") + +def get_kms_connection_config(): + return ppe.KmsConnectionConfig( + custom_kms_conf={ + "footer_key": "012footer_secret", + "orderid_key": "column_secret001", + "productid_key": "column_secret002", + "price_key": "column_secret003", + "customer_key": "column_secret004" + } + ) + +def kms_client_factory(kms_connection_config): + return FooKmsClient(kms_connection_config) + +# ################################################################################################ + +""" +Set up all encryption configuration parameters. +External DBPA encryption requires the use of the ExternalEncryptionConfiguration and the +ExternalFileEncryptionProperties classes. These allow you to specify per column encryption +algorithms and keys. +For this example, the following configuration will apply to each column: +- productId: encrypted withthe file-level encryption using AES_GCM_V1 and key productid_key +- orderId: encrypted with the AES_GCM_CTR_V1 encryption using key orderid_key +- price: encrypted with the external DBPA encryption using key price_key +- customer_name: encrypted with the external DBPA encryption using key customer_key +- vat: not encrypted +This is also where you send the application specific context to the external encryptor, and +where you specify how to connect to the external DBPA encryptor: whether via a library file, +or via a remote service. +All other parameters are the same as for regular Parquet Arrow encryption. +""" +def get_external_encryption_config(use_remote_service): + return ppe.ExternalEncryptionConfiguration( + footer_key="footer_key", + # File level encryption algorithm. This is the default algorithm that will apply + # when no per column encryption algorithm is specified. + encryption_algorithm="AES_GCM_V1", + # These are the usual column keys that will be used for the file-level encryption. + column_keys={ + "productid_key": ["productId"] + }, + cache_lifetime=datetime.timedelta(minutes=2.0), + data_key_length_bits=128, + plaintext_footer=True, + # Specify each column's encryption algorithm and key. You can use any of the encryption + # algorithms supported by Parquet Arrow. Keep in mind: + # - A column may appear in either the column_keys or per_column_encryption, but not both. + # - If a column appears in both, an exception will be thrown. + # - If a column does not appear in either, it will not be encrypted. + # - Any misspelling of a column name or algorithm name will result in an exception. + per_column_encryption={ + "orderId": { + "encryption_algorithm": "AES_GCM_CTR_V1", + "encryption_key": "orderid_key" + }, + "price": { + "encryption_algorithm": "EXTERNAL_DBPA_V1", + "encryption_key": "price_key" + }, + "customer_name": { + "encryption_algorithm": "EXTERNAL_DBPA_V1", + "encryption_key": "customer_key" + } + }, + # Additional context for the external encryptor. Arrow will just forward this value to + # the external encryptor, and does not read its contents. + app_context=get_app_context(), + # Connection configuration for the external encryptor. + connection_config=get_dbpa_connection_config(use_remote_service) + ) + +def get_external_file_encryption_properties(external_encryption_config): + crypto_factory = ppe.CryptoFactory(kms_client_factory) + return crypto_factory.external_file_encryption_properties( + get_kms_connection_config(), external_encryption_config) + +# ################################################################################################ + +""" +Set up all decryption configuration parameters. +External DBPA decryption requires the use of the ExternalDecryptionConfiguration and the +ExternalFileDecryptionProperties classes. These allow you to specify the application specific +context for the external decryptor, and how to instantiate the external DBPA decryptor: whether +via a local instance, or via a remote one. +All other parameters are the same as for regular Parquet Arrow decryption. +""" +def get_external_decryption_config(use_remote_service): + return ppe.ExternalDecryptionConfiguration( + cache_lifetime=datetime.timedelta(minutes=2.0), + # Additional context for the external decryptor. + app_context=get_app_context(), + # Connection configuration for the external decryptor. + connection_config=get_dbpa_connection_config(use_remote_service) + ) + +def get_external_file_decryption_properties(external_decryption_config): + crypto_factory = ppe.CryptoFactory(kms_client_factory) + return crypto_factory.external_file_decryption_properties( + get_kms_connection_config(), external_decryption_config) + +# ################################################################################################ + +""" +Set up the application specific context for the external DBPA service. This is a contract +exclusively between the application and the external DBPA service. Arrow makes no use of this. +""" +def get_app_context(): + return { + "user_id": "Picard1701", + "location": "Presidio" + } + +# ################################################################################################ + +""" +Set up the connection configuration for the external DBPA encryptor. +Each application can provide its own timeout values for the external DBPA encryptor operations. +If none are provided, default values are used on the encryptor side. +These timeout values are not network related, but rather a protection mechanism to avoid the +external encryptor or decryptor from taking too long to complete their operations. +The application must know the path to the external DBPA agent library file. Important! Please +ensure that LD_LIBRARY_PATH (or its equivalent) has been modified to include the location of the +shared library files for the external DBPA encryptor and decryptor. +When the external DBPA encryptor is running as a remote service, the application must also provide +the *absolute* path to the connection config file, which must be a valid JSON that contains all the +information needed to connect to the external DBPA service. +This includes the server URL and the authentication credentials, which the application must procure +on its own. +""" +def get_dbpa_connection_config(use_remote_service): + connection_config = { + "EXTERNAL_DBPA_V1": { + "agent_init_timeout_ms": "15000", + "agent_encrypt_timeout_ms": "35000", + "agent_decrypt_timeout_ms": "35000" + } + } + if use_remote_service: + agent_library_path = ( + 'libdbpsRemoteAgent.so' if platform.system() == 'Linux' else 'libdbpsRemoteAgent.dylib') + connection_config["EXTERNAL_DBPA_V1"]["agent_library_path"] = agent_library_path + # Make sure this is the absolute path to the connection config file. + remote_file_path = '/arrowdev/python/scripts/test_connection_config_file.json' + connection_config["EXTERNAL_DBPA_V1"]["connection_config_file_path"] = remote_file_path + else: + agent_library_path = ( + 'libdbpsLocalAgent.so' if platform.system() == 'Linux' else 'libdbpsLocalAgent.dylib') + connection_config["EXTERNAL_DBPA_V1"]["agent_library_path"] = agent_library_path + + return connection_config + +# ################################################################################################ + +""" +Write an encrypted Parquet file using the external DBPA encryption library. +The current implementation of the external DBPA library can support a per-value encryption algorithm +(which we call "best case encryption") only when the following conditions are met: +- No compression algorithm is used. +- Dictionary encoding is disabled. +- Column encoding is set to PLAIN for all columns. +If any of these conditions are not met, the external DBPA library will perform traditional +per-page (as opposed to per-value) encryption. +""" +def write_encrypted_parquet_file(parquet_path, use_remote_service, scenario_id): + print("\n------------------------------------------------------------") + print(f"Writing encrypted parquet file to {parquet_path}") + print(f"Using {'remote' if use_remote_service else 'local'} external DBPA encryptor service") + print(f"Using scenario {scenario_id}") + print("------------------------------------------------------------\n") + + sample_data = get_sample_data() + encryption_config = get_external_encryption_config(use_remote_service) + external_file_encryption_properties = get_external_file_encryption_properties(encryption_config) + + match scenario_id: + case 1: + # This is the simplest way to write an encrypted Parquet file. It will use the default + # values for compression (SNAPPY) and encoding (RLE_DICTIONARY). + pp.write_table(sample_data, parquet_path, + encryption_properties=external_file_encryption_properties) + case 2: + # By specifying a combination of parameters that use a plain encoding (not dictionary), + # and no compression, we can ensure that the current external DBPA library will + # perform per-value encryption on the data. + pp.write_table(sample_data, parquet_path, + encryption_properties=external_file_encryption_properties, + use_dictionary=False, compression="NONE") + case 3: + # Other parameters that can be specified involve the data page version (which impacts + # how the data bytes are formatted), and specific column encodings. + pp.write_table(sample_data, parquet_path, + encryption_properties=external_file_encryption_properties, + data_page_version="2.0") + + print("\n------------------------------------------------------------") + print(f"Encrypted parquet file written to {parquet_path}") + print("------------------------------------------------------------\n") + +def get_sample_data(): + # Creating a simple table for encryption. Use your real data, or load data file as needed. + return pyarrow.Table.from_pydict({ + "orderId": [1024, 1025, 1026], + "productId": [152, 268, 6548], + "price": [3.25, 6.48, 2.12], + "vat": [0.0, 0.2, 0.05], + "customer_name": ["Alice", "Bob", "Charlotte"] + }) + +# ################################################################################################ + +""" +Read an encrypted parquet file and print the metadata and data table. +Use external file decryption properties to decrypt the parquet file. As with regular +Parquet Arrow decryption, there is no need to configure much, since the encryption details +are in the Parquet file metadata. +""" + +def read_encrypted_parquet_file(parquet_path, use_remote_service): + print("\n------------------------------------------------------------") + print(f"Reading encrypted parquet file from {parquet_path}") + print("------------------------------------------------------------\n") + + metadata = pp.read_metadata(parquet_path) + print("\n------------------------------------------------------------") + print(f"Decrypted parquet file metadata:\n {metadata}") + print("------------------------------------------------------------\n") + + decryption_config = get_external_decryption_config(use_remote_service) + external_file_decryption_properties = get_external_file_decryption_properties(decryption_config) + parquet_file = pp.ParquetFile( + parquet_path, decryption_properties=external_file_decryption_properties) + data_table = parquet_file.read() + print("\n------------------------------------------------------------") + print(f"Decrypted data table:\n {data_table.to_pandas().head()}") + print("------------------------------------------------------------\n") + +# ################################################################################################ + +""" +Perform round trip encryption and decryption of example Parquet files. +We exercise the following cases for using the external DBPA encryptor services: +- scenario 1: default values for compression (SNAPPY) and encoding (DICTIONARY) +- scenario 2: plain encoding (not dictionary) and no compression +- scenario 3: data page version "2.0" and column encoding "BYTE_STREAM_SPLIT" +See the write_encrypted_parquet_file() function for more details on each scenario and its +implications. +""" +def round_trip_parquet_file_encryption(): + #for use_remote_service in [True, False]: + for use_remote_service in [False]: + service_path_prefix = 'remote' if use_remote_service else 'local' + for scenario_id in [1, 2, 3]: + parquet_path = f"{service_path_prefix}_scenario_{scenario_id}_sample.parquet" + write_encrypted_parquet_file(parquet_path, use_remote_service, scenario_id) + read_encrypted_parquet_file(parquet_path, use_remote_service) + +# ################################################################################################ + +""" +In order for the script to work, you must ensure that LD_LIBRARY_PATH (or its equivalent) has been +modified to include the location of the shared library files for the external DBPA encryptor and +decryptor. +""" +if __name__ == "__main__": + print("\n------------------------------------------------------------") + print("Using external DBPA encryption in Parquet Arrow") + print("------------------------------------------------------------\n") + round_trip_parquet_file_encryption() + +# ################################################################################################ \ No newline at end of file