diff --git a/cpp/src/generated/parquet_types.cpp b/cpp/src/generated/parquet_types.cpp index 0ee973f2a2d6..e125a911ea0c 100644 --- a/cpp/src/generated/parquet_types.cpp +++ b/cpp/src/generated/parquet_types.cpp @@ -5286,6 +5286,50 @@ void AesGcmCtrV1::printTo(std::ostream& out) const { } +ExternalDBPAV1::~ExternalDBPAV1() noexcept { +} + +ExternalDBPAV1::ExternalDBPAV1() noexcept { +} +std::ostream& operator<<(std::ostream& out, const ExternalDBPAV1& obj) +{ + obj.printTo(out); + return out; +} + + +void swap(ExternalDBPAV1 &a, ExternalDBPAV1 &b) { + using ::std::swap; + (void) a; + (void) b; +} + +bool ExternalDBPAV1::operator==(const ExternalDBPAV1 & /* rhs */) const +{ + return true; +} + +ExternalDBPAV1::ExternalDBPAV1(const ExternalDBPAV1& other344) noexcept { + (void) other344; +} +ExternalDBPAV1::ExternalDBPAV1(ExternalDBPAV1&& other345) noexcept { + (void) other345; +} +ExternalDBPAV1& ExternalDBPAV1::operator=(const ExternalDBPAV1& other346) noexcept { + (void) other346; + return *this; +} +ExternalDBPAV1& ExternalDBPAV1::operator=(ExternalDBPAV1&& other347) noexcept { + (void) other347; + return *this; +} +void ExternalDBPAV1::printTo(std::ostream& out) const { + using ::apache::thrift::to_string; + out << "ExternalDBPAV1("; + out << ")"; +} + + EncryptionAlgorithm::~EncryptionAlgorithm() noexcept { } @@ -5301,6 +5345,11 @@ void EncryptionAlgorithm::__set_AES_GCM_CTR_V1(const AesGcmCtrV1& val) { this->AES_GCM_CTR_V1 = val; __isset.AES_GCM_CTR_V1 = true; } + +void EncryptionAlgorithm::__set_EXTERNAL_DBPA_V1(const ExternalDBPAV1& val) { + this->EXTERNAL_DBPA_V1 = val; +__isset.EXTERNAL_DBPA_V1 = true; +} std::ostream& operator<<(std::ostream& out, const EncryptionAlgorithm& obj) { obj.printTo(out); @@ -5312,6 +5361,7 @@ void swap(EncryptionAlgorithm &a, EncryptionAlgorithm &b) { using ::std::swap; swap(a.AES_GCM_V1, b.AES_GCM_V1); swap(a.AES_GCM_CTR_V1, b.AES_GCM_CTR_V1); + swap(a.EXTERNAL_DBPA_V1, b.EXTERNAL_DBPA_V1); swap(a.__isset, b.__isset); } @@ -5325,29 +5375,37 @@ bool EncryptionAlgorithm::operator==(const EncryptionAlgorithm & rhs) const return false; else if (__isset.AES_GCM_CTR_V1 && !(AES_GCM_CTR_V1 == rhs.AES_GCM_CTR_V1)) return false; + if (__isset.EXTERNAL_DBPA_V1 != rhs.__isset.EXTERNAL_DBPA_V1) + return false; + else if (__isset.EXTERNAL_DBPA_V1 && !(EXTERNAL_DBPA_V1 == rhs.EXTERNAL_DBPA_V1)) + return false; return true; } -EncryptionAlgorithm::EncryptionAlgorithm(const EncryptionAlgorithm& other344) { - AES_GCM_V1 = other344.AES_GCM_V1; - AES_GCM_CTR_V1 = other344.AES_GCM_CTR_V1; - __isset = other344.__isset; +EncryptionAlgorithm::EncryptionAlgorithm(const EncryptionAlgorithm& other348) { + AES_GCM_V1 = other348.AES_GCM_V1; + AES_GCM_CTR_V1 = other348.AES_GCM_CTR_V1; + EXTERNAL_DBPA_V1 = other348.EXTERNAL_DBPA_V1; + __isset = other348.__isset; } -EncryptionAlgorithm::EncryptionAlgorithm(EncryptionAlgorithm&& other345) noexcept { - AES_GCM_V1 = std::move(other345.AES_GCM_V1); - AES_GCM_CTR_V1 = std::move(other345.AES_GCM_CTR_V1); - __isset = other345.__isset; +EncryptionAlgorithm::EncryptionAlgorithm(EncryptionAlgorithm&& other349) noexcept { + AES_GCM_V1 = std::move(other349.AES_GCM_V1); + AES_GCM_CTR_V1 = std::move(other349.AES_GCM_CTR_V1); + EXTERNAL_DBPA_V1 = std::move(other349.EXTERNAL_DBPA_V1); + __isset = other349.__isset; } -EncryptionAlgorithm& EncryptionAlgorithm::operator=(const EncryptionAlgorithm& other346) { - AES_GCM_V1 = other346.AES_GCM_V1; - AES_GCM_CTR_V1 = other346.AES_GCM_CTR_V1; - __isset = other346.__isset; +EncryptionAlgorithm& EncryptionAlgorithm::operator=(const EncryptionAlgorithm& other350) { + AES_GCM_V1 = other350.AES_GCM_V1; + AES_GCM_CTR_V1 = other350.AES_GCM_CTR_V1; + EXTERNAL_DBPA_V1 = other350.EXTERNAL_DBPA_V1; + __isset = other350.__isset; return *this; } -EncryptionAlgorithm& EncryptionAlgorithm::operator=(EncryptionAlgorithm&& other347) noexcept { - AES_GCM_V1 = std::move(other347.AES_GCM_V1); - AES_GCM_CTR_V1 = std::move(other347.AES_GCM_CTR_V1); - __isset = other347.__isset; +EncryptionAlgorithm& EncryptionAlgorithm::operator=(EncryptionAlgorithm&& other351) noexcept { + AES_GCM_V1 = std::move(other351.AES_GCM_V1); + AES_GCM_CTR_V1 = std::move(other351.AES_GCM_CTR_V1); + EXTERNAL_DBPA_V1 = std::move(other351.EXTERNAL_DBPA_V1); + __isset = other351.__isset; return *this; } void EncryptionAlgorithm::printTo(std::ostream& out) const { @@ -5355,6 +5413,7 @@ void EncryptionAlgorithm::printTo(std::ostream& out) const { out << "EncryptionAlgorithm("; out << "AES_GCM_V1="; (__isset.AES_GCM_V1 ? (out << to_string(AES_GCM_V1)) : (out << "")); out << ", " << "AES_GCM_CTR_V1="; (__isset.AES_GCM_CTR_V1 ? (out << to_string(AES_GCM_CTR_V1)) : (out << "")); + out << ", " << "EXTERNAL_DBPA_V1="; (__isset.EXTERNAL_DBPA_V1 ? (out << to_string(EXTERNAL_DBPA_V1)) : (out << "")); out << ")"; } @@ -5463,54 +5522,54 @@ bool FileMetaData::operator==(const FileMetaData & rhs) const return true; } -FileMetaData::FileMetaData(const FileMetaData& other372) { - version = other372.version; - schema = other372.schema; - num_rows = other372.num_rows; - row_groups = other372.row_groups; - key_value_metadata = other372.key_value_metadata; - created_by = other372.created_by; - column_orders = other372.column_orders; - encryption_algorithm = other372.encryption_algorithm; - footer_signing_key_metadata = other372.footer_signing_key_metadata; - __isset = other372.__isset; -} -FileMetaData::FileMetaData(FileMetaData&& other373) noexcept { - version = other373.version; - schema = std::move(other373.schema); - num_rows = other373.num_rows; - row_groups = std::move(other373.row_groups); - key_value_metadata = std::move(other373.key_value_metadata); - created_by = std::move(other373.created_by); - column_orders = std::move(other373.column_orders); - encryption_algorithm = std::move(other373.encryption_algorithm); - footer_signing_key_metadata = std::move(other373.footer_signing_key_metadata); - __isset = other373.__isset; -} -FileMetaData& FileMetaData::operator=(const FileMetaData& other374) { - version = other374.version; - schema = other374.schema; - num_rows = other374.num_rows; - row_groups = other374.row_groups; - key_value_metadata = other374.key_value_metadata; - created_by = other374.created_by; - column_orders = other374.column_orders; - encryption_algorithm = other374.encryption_algorithm; - footer_signing_key_metadata = other374.footer_signing_key_metadata; - __isset = other374.__isset; +FileMetaData::FileMetaData(const FileMetaData& other376) { + version = other376.version; + schema = other376.schema; + num_rows = other376.num_rows; + row_groups = other376.row_groups; + key_value_metadata = other376.key_value_metadata; + created_by = other376.created_by; + column_orders = other376.column_orders; + encryption_algorithm = other376.encryption_algorithm; + footer_signing_key_metadata = other376.footer_signing_key_metadata; + __isset = other376.__isset; +} +FileMetaData::FileMetaData(FileMetaData&& other377) noexcept { + version = other377.version; + schema = std::move(other377.schema); + num_rows = other377.num_rows; + row_groups = std::move(other377.row_groups); + key_value_metadata = std::move(other377.key_value_metadata); + created_by = std::move(other377.created_by); + column_orders = std::move(other377.column_orders); + encryption_algorithm = std::move(other377.encryption_algorithm); + footer_signing_key_metadata = std::move(other377.footer_signing_key_metadata); + __isset = other377.__isset; +} +FileMetaData& FileMetaData::operator=(const FileMetaData& other378) { + version = other378.version; + schema = other378.schema; + num_rows = other378.num_rows; + row_groups = other378.row_groups; + key_value_metadata = other378.key_value_metadata; + created_by = other378.created_by; + column_orders = other378.column_orders; + encryption_algorithm = other378.encryption_algorithm; + footer_signing_key_metadata = other378.footer_signing_key_metadata; + __isset = other378.__isset; return *this; } -FileMetaData& FileMetaData::operator=(FileMetaData&& other375) noexcept { - version = other375.version; - schema = std::move(other375.schema); - num_rows = other375.num_rows; - row_groups = std::move(other375.row_groups); - key_value_metadata = std::move(other375.key_value_metadata); - created_by = std::move(other375.created_by); - column_orders = std::move(other375.column_orders); - encryption_algorithm = std::move(other375.encryption_algorithm); - footer_signing_key_metadata = std::move(other375.footer_signing_key_metadata); - __isset = other375.__isset; +FileMetaData& FileMetaData::operator=(FileMetaData&& other379) noexcept { + version = other379.version; + schema = std::move(other379.schema); + num_rows = other379.num_rows; + row_groups = std::move(other379.row_groups); + key_value_metadata = std::move(other379.key_value_metadata); + created_by = std::move(other379.created_by); + column_orders = std::move(other379.column_orders); + encryption_algorithm = std::move(other379.encryption_algorithm); + footer_signing_key_metadata = std::move(other379.footer_signing_key_metadata); + __isset = other379.__isset; return *this; } void FileMetaData::printTo(std::ostream& out) const { @@ -5569,26 +5628,26 @@ bool FileCryptoMetaData::operator==(const FileCryptoMetaData & rhs) const return true; } -FileCryptoMetaData::FileCryptoMetaData(const FileCryptoMetaData& other376) { - encryption_algorithm = other376.encryption_algorithm; - key_metadata = other376.key_metadata; - __isset = other376.__isset; +FileCryptoMetaData::FileCryptoMetaData(const FileCryptoMetaData& other380) { + encryption_algorithm = other380.encryption_algorithm; + key_metadata = other380.key_metadata; + __isset = other380.__isset; } -FileCryptoMetaData::FileCryptoMetaData(FileCryptoMetaData&& other377) noexcept { - encryption_algorithm = std::move(other377.encryption_algorithm); - key_metadata = std::move(other377.key_metadata); - __isset = other377.__isset; +FileCryptoMetaData::FileCryptoMetaData(FileCryptoMetaData&& other381) noexcept { + encryption_algorithm = std::move(other381.encryption_algorithm); + key_metadata = std::move(other381.key_metadata); + __isset = other381.__isset; } -FileCryptoMetaData& FileCryptoMetaData::operator=(const FileCryptoMetaData& other378) { - encryption_algorithm = other378.encryption_algorithm; - key_metadata = other378.key_metadata; - __isset = other378.__isset; +FileCryptoMetaData& FileCryptoMetaData::operator=(const FileCryptoMetaData& other382) { + encryption_algorithm = other382.encryption_algorithm; + key_metadata = other382.key_metadata; + __isset = other382.__isset; return *this; } -FileCryptoMetaData& FileCryptoMetaData::operator=(FileCryptoMetaData&& other379) noexcept { - encryption_algorithm = std::move(other379.encryption_algorithm); - key_metadata = std::move(other379.key_metadata); - __isset = other379.__isset; +FileCryptoMetaData& FileCryptoMetaData::operator=(FileCryptoMetaData&& other383) noexcept { + encryption_algorithm = std::move(other383.encryption_algorithm); + key_metadata = std::move(other383.key_metadata); + __isset = other383.__isset; return *this; } void FileCryptoMetaData::printTo(std::ostream& out) const { diff --git a/cpp/src/generated/parquet_types.h b/cpp/src/generated/parquet_types.h index 1f1e254f5cf2..31ca1cf11577 100644 --- a/cpp/src/generated/parquet_types.h +++ b/cpp/src/generated/parquet_types.h @@ -477,6 +477,8 @@ class AesGcmV1; class AesGcmCtrV1; +class ExternalDBPAV1; + class EncryptionAlgorithm; class FileMetaData; @@ -3663,10 +3665,42 @@ void swap(AesGcmCtrV1 &a, AesGcmCtrV1 &b); std::ostream& operator<<(std::ostream& out, const AesGcmCtrV1& obj); + +class ExternalDBPAV1 { + public: + + ExternalDBPAV1(const ExternalDBPAV1&) noexcept; + ExternalDBPAV1(ExternalDBPAV1&&) noexcept; + ExternalDBPAV1& operator=(const ExternalDBPAV1&) noexcept; + ExternalDBPAV1& operator=(ExternalDBPAV1&&) noexcept; + ExternalDBPAV1() noexcept; + + virtual ~ExternalDBPAV1() noexcept; + + bool operator == (const ExternalDBPAV1 & /* rhs */) const; + bool operator != (const ExternalDBPAV1 &rhs) const { + return !(*this == rhs); + } + + bool operator < (const ExternalDBPAV1 & ) const; + + template + uint32_t read(Protocol_* iprot); + template + uint32_t write(Protocol_* oprot) const; + + virtual void printTo(std::ostream& out) const; +}; + +void swap(ExternalDBPAV1 &a, ExternalDBPAV1 &b); + +std::ostream& operator<<(std::ostream& out, const ExternalDBPAV1& obj); + typedef struct _EncryptionAlgorithm__isset { - _EncryptionAlgorithm__isset() : AES_GCM_V1(false), AES_GCM_CTR_V1(false) {} + _EncryptionAlgorithm__isset() : AES_GCM_V1(false), AES_GCM_CTR_V1(false), EXTERNAL_DBPA_V1(false) {} bool AES_GCM_V1 :1; bool AES_GCM_CTR_V1 :1; + bool EXTERNAL_DBPA_V1 :1; } _EncryptionAlgorithm__isset; class EncryptionAlgorithm { @@ -3681,6 +3715,7 @@ class EncryptionAlgorithm { virtual ~EncryptionAlgorithm() noexcept; AesGcmV1 AES_GCM_V1; AesGcmCtrV1 AES_GCM_CTR_V1; + ExternalDBPAV1 EXTERNAL_DBPA_V1; _EncryptionAlgorithm__isset __isset; @@ -3688,6 +3723,8 @@ class EncryptionAlgorithm { void __set_AES_GCM_CTR_V1(const AesGcmCtrV1& val); + void __set_EXTERNAL_DBPA_V1(const ExternalDBPAV1& val); + bool operator == (const EncryptionAlgorithm & rhs) const; bool operator != (const EncryptionAlgorithm &rhs) const { return !(*this == rhs); diff --git a/cpp/src/generated/parquet_types.tcc b/cpp/src/generated/parquet_types.tcc index 78e3e2549394..37b45881c22e 100644 --- a/cpp/src/generated/parquet_types.tcc +++ b/cpp/src/generated/parquet_types.tcc @@ -4957,6 +4957,46 @@ uint32_t AesGcmCtrV1::write(Protocol_* oprot) const { return xfer; } +template +uint32_t ExternalDBPAV1::read(Protocol_* iprot) { + + ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + xfer += iprot->skip(ftype); + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +template +uint32_t ExternalDBPAV1::write(Protocol_* oprot) const { + uint32_t xfer = 0; + ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); + xfer += oprot->writeStructBegin("ExternalDBPAV1"); + + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + template uint32_t EncryptionAlgorithm::read(Protocol_* iprot) { @@ -4995,6 +5035,14 @@ uint32_t EncryptionAlgorithm::read(Protocol_* iprot) { xfer += iprot->skip(ftype); } break; + case 3: + if (ftype == ::apache::thrift::protocol::T_STRUCT) { + xfer += this->EXTERNAL_DBPA_V1.read(iprot); + this->__isset.EXTERNAL_DBPA_V1 = true; + } else { + xfer += iprot->skip(ftype); + } + break; default: xfer += iprot->skip(ftype); break; @@ -5023,6 +5071,11 @@ uint32_t EncryptionAlgorithm::write(Protocol_* oprot) const { xfer += this->AES_GCM_CTR_V1.write(oprot); xfer += oprot->writeFieldEnd(); } + if (this->__isset.EXTERNAL_DBPA_V1) { + xfer += oprot->writeFieldBegin("EXTERNAL_DBPA_V1", ::apache::thrift::protocol::T_STRUCT, 3); + xfer += this->EXTERNAL_DBPA_V1.write(oprot); + xfer += oprot->writeFieldEnd(); + } xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; @@ -5066,14 +5119,14 @@ uint32_t FileMetaData::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->schema.clear(); - uint32_t _size348; - ::apache::thrift::protocol::TType _etype351; - xfer += iprot->readListBegin(_etype351, _size348); - this->schema.resize(_size348); - uint32_t _i352; - for (_i352 = 0; _i352 < _size348; ++_i352) + uint32_t _size352; + ::apache::thrift::protocol::TType _etype355; + xfer += iprot->readListBegin(_etype355, _size352); + this->schema.resize(_size352); + uint32_t _i356; + for (_i356 = 0; _i356 < _size352; ++_i356) { - xfer += this->schema[_i352].read(iprot); + xfer += this->schema[_i356].read(iprot); } xfer += iprot->readListEnd(); } @@ -5094,14 +5147,14 @@ uint32_t FileMetaData::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->row_groups.clear(); - uint32_t _size353; - ::apache::thrift::protocol::TType _etype356; - xfer += iprot->readListBegin(_etype356, _size353); - this->row_groups.resize(_size353); - uint32_t _i357; - for (_i357 = 0; _i357 < _size353; ++_i357) + uint32_t _size357; + ::apache::thrift::protocol::TType _etype360; + xfer += iprot->readListBegin(_etype360, _size357); + this->row_groups.resize(_size357); + uint32_t _i361; + for (_i361 = 0; _i361 < _size357; ++_i361) { - xfer += this->row_groups[_i357].read(iprot); + xfer += this->row_groups[_i361].read(iprot); } xfer += iprot->readListEnd(); } @@ -5114,14 +5167,14 @@ uint32_t FileMetaData::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->key_value_metadata.clear(); - uint32_t _size358; - ::apache::thrift::protocol::TType _etype361; - xfer += iprot->readListBegin(_etype361, _size358); - this->key_value_metadata.resize(_size358); - uint32_t _i362; - for (_i362 = 0; _i362 < _size358; ++_i362) + uint32_t _size362; + ::apache::thrift::protocol::TType _etype365; + xfer += iprot->readListBegin(_etype365, _size362); + this->key_value_metadata.resize(_size362); + uint32_t _i366; + for (_i366 = 0; _i366 < _size362; ++_i366) { - xfer += this->key_value_metadata[_i362].read(iprot); + xfer += this->key_value_metadata[_i366].read(iprot); } xfer += iprot->readListEnd(); } @@ -5142,14 +5195,14 @@ uint32_t FileMetaData::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->column_orders.clear(); - uint32_t _size363; - ::apache::thrift::protocol::TType _etype366; - xfer += iprot->readListBegin(_etype366, _size363); - this->column_orders.resize(_size363); - uint32_t _i367; - for (_i367 = 0; _i367 < _size363; ++_i367) + uint32_t _size367; + ::apache::thrift::protocol::TType _etype370; + xfer += iprot->readListBegin(_etype370, _size367); + this->column_orders.resize(_size367); + uint32_t _i371; + for (_i371 = 0; _i371 < _size367; ++_i371) { - xfer += this->column_orders[_i367].read(iprot); + xfer += this->column_orders[_i371].read(iprot); } xfer += iprot->readListEnd(); } @@ -5207,10 +5260,10 @@ uint32_t FileMetaData::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("schema", ::apache::thrift::protocol::T_LIST, 2); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->schema.size())); - std::vector ::const_iterator _iter368; - for (_iter368 = this->schema.begin(); _iter368 != this->schema.end(); ++_iter368) + std::vector ::const_iterator _iter372; + for (_iter372 = this->schema.begin(); _iter372 != this->schema.end(); ++_iter372) { - xfer += (*_iter368).write(oprot); + xfer += (*_iter372).write(oprot); } xfer += oprot->writeListEnd(); } @@ -5223,10 +5276,10 @@ uint32_t FileMetaData::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("row_groups", ::apache::thrift::protocol::T_LIST, 4); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->row_groups.size())); - std::vector ::const_iterator _iter369; - for (_iter369 = this->row_groups.begin(); _iter369 != this->row_groups.end(); ++_iter369) + std::vector ::const_iterator _iter373; + for (_iter373 = this->row_groups.begin(); _iter373 != this->row_groups.end(); ++_iter373) { - xfer += (*_iter369).write(oprot); + xfer += (*_iter373).write(oprot); } xfer += oprot->writeListEnd(); } @@ -5236,10 +5289,10 @@ uint32_t FileMetaData::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("key_value_metadata", ::apache::thrift::protocol::T_LIST, 5); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->key_value_metadata.size())); - std::vector ::const_iterator _iter370; - for (_iter370 = this->key_value_metadata.begin(); _iter370 != this->key_value_metadata.end(); ++_iter370) + std::vector ::const_iterator _iter374; + for (_iter374 = this->key_value_metadata.begin(); _iter374 != this->key_value_metadata.end(); ++_iter374) { - xfer += (*_iter370).write(oprot); + xfer += (*_iter374).write(oprot); } xfer += oprot->writeListEnd(); } @@ -5254,10 +5307,10 @@ uint32_t FileMetaData::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("column_orders", ::apache::thrift::protocol::T_LIST, 7); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->column_orders.size())); - std::vector ::const_iterator _iter371; - for (_iter371 = this->column_orders.begin(); _iter371 != this->column_orders.end(); ++_iter371) + std::vector ::const_iterator _iter375; + for (_iter375 = this->column_orders.begin(); _iter375 != this->column_orders.end(); ++_iter375) { - xfer += (*_iter371).write(oprot); + xfer += (*_iter375).write(oprot); } xfer += oprot->writeListEnd(); } diff --git a/cpp/src/parquet/CMakeLists.txt b/cpp/src/parquet/CMakeLists.txt index 465704331fe3..7c8301931a4b 100644 --- a/cpp/src/parquet/CMakeLists.txt +++ b/cpp/src/parquet/CMakeLists.txt @@ -413,7 +413,9 @@ if(PARQUET_REQUIRE_ENCRYPTION) encryption/write_configurations_test.cc encryption/read_configurations_test.cc encryption/properties_test.cc - encryption/test_encryption_util.cc) + encryption/crypto_factory_test.cc + encryption/test_encryption_util.cc + encryption/test_in_memory_kms.cc) add_parquet_test(encryption-key-management-test SOURCES encryption/key_management_test.cc diff --git a/cpp/src/parquet/encryption/crypto_factory.cc b/cpp/src/parquet/encryption/crypto_factory.cc index 56069d559771..604e4eecba22 100644 --- a/cpp/src/parquet/encryption/crypto_factory.cc +++ b/cpp/src/parquet/encryption/crypto_factory.cc @@ -28,24 +28,21 @@ namespace parquet::encryption { -void CryptoFactory::RegisterKmsClientFactory( - std::shared_ptr kms_client_factory) { - key_toolkit_->RegisterKmsClientFactory(std::move(kms_client_factory)); -} +/// Extracting functionality common to both GetFileEncryptionProperties and +/// GetExternalFileEncryptionProperties here for reuse. +namespace { -std::shared_ptr CryptoFactory::GetFileEncryptionProperties( - const KmsConnectionConfig& kms_connection_config, - const EncryptionConfiguration& encryption_config, const std::string& file_path, - const std::shared_ptr<::arrow::fs::FileSystem>& file_system) { - if (!encryption_config.uniform_encryption && encryption_config.column_keys.empty()) { - throw ParquetException("Either column_keys or uniform_encryption must be set"); - } else if (encryption_config.uniform_encryption && - !encryption_config.column_keys.empty()) { - throw ParquetException("Cannot set both column_keys and uniform_encryption"); - } - const std::string& footer_key_id = encryption_config.footer_key; - const std::string& column_key_str = encryption_config.column_keys; +// Struct to simplify the returned objects in GetFileKeyUtils. +struct FileKeyUtils { + std::shared_ptr key_material_store; + FileKeyWrapper key_wrapper; +}; +FileKeyUtils GetFileKeyUtils( + const std::shared_ptr& key_toolkit, + const KmsConnectionConfig& kms_connection_config, + const EncryptionConfiguration& encryption_config, + const std::string& file_path, const std::shared_ptr<::arrow::fs::FileSystem>& file_system) { std::shared_ptr key_material_store = nullptr; if (!encryption_config.internal_key_material) { try { @@ -58,18 +55,46 @@ std::shared_ptr CryptoFactory::GetFileEncryptionProper } } - FileKeyWrapper key_wrapper(key_toolkit_.get(), kms_connection_config, + FileKeyWrapper key_wrapper(key_toolkit.get(), kms_connection_config, key_material_store, encryption_config.cache_lifetime_seconds, encryption_config.double_wrapping); - int32_t dek_length_bits = encryption_config.data_key_length_bits; + return {key_material_store, std::move(key_wrapper)}; +} + +int ValidateAndGetKeyLength(int32_t dek_length_bits) { if (!internal::ValidateKeyLength(dek_length_bits)) { std::ostringstream ss; ss << "Wrong data key length : " << dek_length_bits; throw ParquetException(ss.str()); } + return dek_length_bits / 8; +} - int dek_length = dek_length_bits / 8; +} // Anonymous namespace + +void CryptoFactory::RegisterKmsClientFactory( + std::shared_ptr kms_client_factory) { + key_toolkit_->RegisterKmsClientFactory(std::move(kms_client_factory)); +} + +std::shared_ptr CryptoFactory::GetFileEncryptionProperties( + const KmsConnectionConfig& kms_connection_config, + const EncryptionConfiguration& encryption_config, const std::string& file_path, + const std::shared_ptr<::arrow::fs::FileSystem>& file_system) { + if (!encryption_config.uniform_encryption && encryption_config.column_keys.empty()) { + throw ParquetException("Either column_keys or uniform_encryption must be set"); + } else if (encryption_config.uniform_encryption && + !encryption_config.column_keys.empty()) { + throw ParquetException("Cannot set both column_keys and uniform_encryption"); + } + const std::string& footer_key_id = encryption_config.footer_key; + const std::string& column_key_str = encryption_config.column_keys; + + auto [key_material_store, key_wrapper] = GetFileKeyUtils( + key_toolkit_, kms_connection_config, encryption_config, file_path, file_system); + + int dek_length = ValidateAndGetKeyLength(encryption_config.data_key_length_bits); std::string footer_key(dek_length, '\0'); RandBytes(reinterpret_cast(footer_key.data()), footer_key.size()); @@ -98,6 +123,92 @@ std::shared_ptr CryptoFactory::GetFileEncryptionProper return properties_builder.build(); } +std::shared_ptr +CryptoFactory::GetExternalFileEncryptionProperties( + const KmsConnectionConfig& kms_connection_config, + const ExternalEncryptionConfiguration& external_encryption_config, + const std::string& file_path, const std::shared_ptr<::arrow::fs::FileSystem>& file_system) { + // Validate the same rules as FileEncryptionProperties but considering per_column_encryption too. + // If uniform_encryption is not set then either column_keys or per_column_encryption must have + // values. + // If uniform_encryption is set, then both column_keys and per_column_encryption must be empty. + bool no_columns_encrypted = external_encryption_config.column_keys.empty() && + external_encryption_config.per_column_encryption.empty(); + if (!external_encryption_config.uniform_encryption && no_columns_encrypted) { + throw ParquetException( + "Either uniform_encryption must be set or column encryption must be specified in either " + "column_keys or per_column_encryption"); + } else if (external_encryption_config.uniform_encryption && !no_columns_encrypted) { + throw ParquetException("Cannot set both column encryption and uniform_encryption"); + } + + auto [key_material_store, key_wrapper] = GetFileKeyUtils( + key_toolkit_, kms_connection_config, external_encryption_config, file_path, file_system); + + int dek_length = ValidateAndGetKeyLength(external_encryption_config.data_key_length_bits); + + std::string footer_key(dek_length, '\0'); + RandBytes(reinterpret_cast(footer_key.data()), footer_key.size()); + + std::string footer_key_metadata = + key_wrapper.GetEncryptionKeyMetadata(footer_key, external_encryption_config.footer_key, true); + + ExternalFileEncryptionProperties::Builder external_properties_builder = + ExternalFileEncryptionProperties::Builder(external_encryption_config.footer_key); + external_properties_builder.footer_key_metadata(footer_key_metadata); + external_properties_builder.algorithm(external_encryption_config.encryption_algorithm); + + if (!external_encryption_config.uniform_encryption && + external_encryption_config.plaintext_footer) { + external_properties_builder.set_plaintext_footer(); + } + + ColumnPathToEncryptionPropertiesMap encrypted_columns; + if (!external_encryption_config.column_keys.empty()) { + encrypted_columns = GetColumnEncryptionProperties( + dek_length, external_encryption_config.column_keys, &key_wrapper); + } + if (!external_encryption_config.per_column_encryption.empty()) { + for (const auto& pair : external_encryption_config.per_column_encryption) { + const std::string& column_name = pair.first; + const ColumnEncryptionAttributes& attributes = pair.second; + + // Validate column names are not in both column_keys and per_column_encryption maps. + if (encrypted_columns.find(column_name) != encrypted_columns.end()) { + std::stringstream string_stream; + string_stream << "Multiple keys defined for column [" << column_name << "]. "; + string_stream << "Keys found in column_keys and in per_column_encryption."; + throw ParquetException(string_stream.str()); + } + + // TODO(sbrenes): Check whether the attributes.parquet_cipher == EXTERNAL. + // If so, do not use KMS to resolve the column_key, just forward it. + std::string column_key(dek_length, '\0'); + RandBytes(reinterpret_cast(column_key.data()), column_key.size()); + std::string column_key_metadata = + key_wrapper.GetEncryptionKeyMetadata(column_key, attributes.key_id, false); + + std::shared_ptr column_properties = + ColumnEncryptionProperties::Builder(column_name) + .key(column_key) + ->key_metadata(column_key_metadata) + ->parquet_cipher(attributes.parquet_cipher) + ->build(); + + encrypted_columns.insert({column_name, column_properties}); + } + } + if (!encrypted_columns.empty()) { + external_properties_builder.encrypted_columns(encrypted_columns); + } + + if (key_material_store != nullptr) { + key_material_store->SaveMaterial(); + } + + return external_properties_builder.build_external(); +} + ColumnPathToEncryptionPropertiesMap CryptoFactory::GetColumnEncryptionProperties( int dek_length, const std::string& column_keys, FileKeyWrapper* key_wrapper) { ColumnPathToEncryptionPropertiesMap encrypted_columns; diff --git a/cpp/src/parquet/encryption/crypto_factory.h b/cpp/src/parquet/encryption/crypto_factory.h index 3c6a5f29ea88..e4ff213b7e27 100644 --- a/cpp/src/parquet/encryption/crypto_factory.h +++ b/cpp/src/parquet/encryption/crypto_factory.h @@ -18,12 +18,14 @@ #pragma once #include +#include #include "parquet/encryption/encryption.h" #include "parquet/encryption/file_key_wrapper.h" #include "parquet/encryption/key_toolkit.h" #include "parquet/encryption/kms_client_factory.h" #include "parquet/platform.h" +#include "parquet/types.h" namespace parquet::encryption { @@ -86,6 +88,56 @@ struct PARQUET_EXPORT EncryptionConfiguration { int32_t data_key_length_bits = kDefaultDataKeyLengthBits; }; +/// Helper struct for use in per column encryption specification. +/// The ExternalEncryptionConfiguration will use this to send encryption parameters per column. +struct PARQUET_EXPORT ColumnEncryptionAttributes { + /// Which type of encryptor to use. + ParquetCipher::type parquet_cipher; + + /// The key_id to use for this column. + std::string key_id; +}; + +/// Encryption Configuration for use with External Encryption services. +/// Extends the already existing EncryptionConfiguration with more context and with +/// the capability of specifying encryption algorithm per column. +struct PARQUET_EXPORT ExternalEncryptionConfiguration : public EncryptionConfiguration { + explicit ExternalEncryptionConfiguration(const std::string& footer_key) + : EncryptionConfiguration(footer_key) {} + + /// Map of the columns to encrypt to their associated encryption parameters. The id of the map + /// is the column name, and the value is a ColumnEncryptionAttributes struct that can be + /// used to construct the ColumnEncryptionProperties in the CryptoFactory. + /// As with the EncryptionConfiguration, either: + /// (1) uniform_encryption = true + /// or + /// (2) column_keys and/or per_column_encryption is set + /// If none of (1) and (2) are true, or if both are true, an exception will be thrown. + /// If a column name appears in the original column_keys list, it will be encrypted with the + /// algorithm specified in the encryption_algorithm field. + /// If a column name appears in the new per_column_encryption map, it will be encrypted using the + /// per column specific algorithm and key. + /// If a column name appears in both, an exception will be thrown. + std::unordered_map per_column_encryption; + + /// External encryption services may use additional context provided by the application to + /// enforce robust access control. The values sent to the external service depend on each + /// implementation. + /// This value must be a valid JSON-formatted string. + /// Validation of the string will be done by the external encryption service, Arrow will only + /// forward this value. + /// Format: "{\"user_id\": \"abc123\", \"location\": {\"lat\": 9.7489, \"lon\": -83.7534}}" + std::string app_context; + + /// Key/value map of the location of configuration files needed by the external + /// encryption service. This may include location of a dynamically-linked library, or the + /// location of a file where the external service can find urls, certificates, and parameters + /// needed to make a remote service call. + /// For security, these values should never be sent in this config, only the locations of + /// the files that the external service will know how to access. + std::unordered_map connection_config; +}; + struct PARQUET_EXPORT DecryptionConfiguration { /// Lifetime of cached entities (key encryption keys, local wrapping keys, KMS client /// objects). @@ -93,6 +145,26 @@ struct PARQUET_EXPORT DecryptionConfiguration { double cache_lifetime_seconds = kDefaultCacheLifetimeSeconds; }; +struct PARQUET_EXPORT ExternalDecryptionConfiguration : public DecryptionConfiguration { + /// External decryption services may use additional context provided by the application to + /// enforce robust access control. The values sent to the external service depend on each + /// implementation. + /// This value must be a valid JSON-formatted string. + /// Validation of the string will be done by the external decryption service, Arrow will only + /// forward this value. + /// Format: "{\"user_id\": \"abc123\", \"location\": {\"lat\": 9.7489, \"lon\": -83.7534}}" + std::string app_context; + + /// Map of the encryption algorithms to the key/value map of the location of configuration files + /// needed by the external decryption service. This may include location of a dynamically-linked + /// library, or the location of a file where the external service can find urls, certificates, + /// and parameters needed to make a remote service call. + /// For security, these values should never be sent in this config, only the locations of + /// the files that the external service will know how to access. + std::unordered_map> + connection_config; +}; + /// This is a core class, that translates the parameters of high level encryption (like /// the names of encrypted columns, names of master keys, etc), into parameters of low /// level encryption (like the key metadata, DEK, etc). A factory that produces the low @@ -105,13 +177,21 @@ class PARQUET_EXPORT CryptoFactory { void RegisterKmsClientFactory(std::shared_ptr kms_client_factory); /// Get the encryption properties for a Parquet file. - /// If external key material is used then a file system and path to the + /// If key material from outside the file is used, then a file system and path to the /// parquet file must be provided. std::shared_ptr GetFileEncryptionProperties( const KmsConnectionConfig& kms_connection_config, const EncryptionConfiguration& encryption_config, const std::string& file_path = "", const std::shared_ptr<::arrow::fs::FileSystem>& file_system = NULLPTR); + /// Get the external encryption properties for a Parquet file. Used when encryption + /// will be provided by an external service. + std::shared_ptr GetExternalFileEncryptionProperties( + const KmsConnectionConfig& kms_connection_config, + const ExternalEncryptionConfiguration& external_encryption_config, + const std::string& file_path = "", + const std::shared_ptr<::arrow::fs::FileSystem>& file_system = NULLPTR); + /// Get decryption properties for a Parquet file. /// If external key material is used then a file system and path to the /// parquet file must be provided. diff --git a/cpp/src/parquet/encryption/crypto_factory_test.cc b/cpp/src/parquet/encryption/crypto_factory_test.cc new file mode 100644 index 000000000000..b749b311cc37 --- /dev/null +++ b/cpp/src/parquet/encryption/crypto_factory_test.cc @@ -0,0 +1,185 @@ +#include +#include + +#include "parquet/encryption/crypto_factory.h" +#include "parquet/encryption/file_key_material_store.h" +#include "parquet/encryption/test_encryption_util.h" +#include "parquet/encryption/test_in_memory_kms.h" + +using ::testing::_; +using ::testing::HasSubstr; +using ::testing::Return; +using ::testing::StrEq; + +namespace parquet::encryption::test { + +class CryptoFactoryTest : public ::testing::Test { + + void SetUp() { + key_list_ = BuildKeyMap(kColumnMasterKeyIds, kColumnMasterKeys, kFooterMasterKeyId, + kFooterMasterKey); + crypto_factory_.RegisterKmsClientFactory(std::make_shared( + true, key_list_)); + } + + protected: + std::unordered_map key_list_; + KmsConnectionConfig kms_config_; + CryptoFactory crypto_factory_; +}; + +TEST_F(CryptoFactoryTest, UniformEncryptionAndColumnKeysThrowsException) { + ExternalEncryptionConfiguration config("kf"); + config.uniform_encryption = true; + config.column_keys = "kc1:col1,col2"; + + try { + auto properties = crypto_factory_.GetExternalFileEncryptionProperties(kms_config_, config); + FAIL() << "ParquetException should have been raised"; + } catch (const ParquetException& xcp) { + EXPECT_THAT(xcp.what(), HasSubstr("Cannot set both column encryption and uniform")); + } catch (...) { + FAIL() << "Caught unexpected exception type"; + } +} + +TEST_F(CryptoFactoryTest, UniformEncryptionAndPerColumnEncryptionThrowsException) { + ExternalEncryptionConfiguration config("kf"); + config.uniform_encryption = true; + + std::unordered_map per_column_encryption; + ColumnEncryptionAttributes attributes; + attributes.parquet_cipher = ParquetCipher::AES_GCM_CTR_V1; + attributes.key_id = "kc1"; + per_column_encryption["col1"] = attributes; + config.per_column_encryption = per_column_encryption; + + try { + auto properties = crypto_factory_.GetExternalFileEncryptionProperties(kms_config_, config); + FAIL() << "ParquetException should have been raised"; + } catch (const ParquetException& xcp) { + EXPECT_THAT(xcp.what(), HasSubstr("Cannot set both column encryption and uniform")); + } catch (...) { + FAIL() << "Caught unexpected exception type"; + } +} + +TEST_F(CryptoFactoryTest, NoUniformEncryptionAndNoColumnsThrowsException) { + ExternalEncryptionConfiguration config("kf"); + + try { + auto properties = crypto_factory_.GetExternalFileEncryptionProperties(kms_config_, config); + FAIL() << "ParquetException should have been raised"; + } catch (const ParquetException& xcp) { + EXPECT_THAT(xcp.what(), HasSubstr( + "uniform_encryption must be set or column encryption must be specified in either")); + } catch (...) { + FAIL() << "Caught unexpected exception type"; + } +} + +TEST_F(CryptoFactoryTest, BasicEncryptionConfig) { + ExternalEncryptionConfiguration config("kf"); + config.plaintext_footer = true; + config.column_keys = "kc1:col1,col2;kc2:col3,col4"; + + auto properties = crypto_factory_.GetExternalFileEncryptionProperties(kms_config_, config); + EXPECT_EQ("kf", properties->footer_key()); + EXPECT_TRUE(properties->footer_key_metadata().size() > 0); + EXPECT_EQ(ParquetCipher::AES_GCM_V1, properties->algorithm().algorithm); + EXPECT_FALSE(properties->encrypted_footer()); + EXPECT_TRUE(properties->encrypted_columns().size() == 4); + + auto column_properties_1 = properties->column_encryption_properties("col1"); + EXPECT_EQ("col1", column_properties_1->column_path()); + EXPECT_TRUE(column_properties_1->is_encrypted()); + EXPECT_FALSE(column_properties_1->is_encrypted_with_footer_key()); + EXPECT_THAT(column_properties_1->key_metadata(), HasSubstr("kc1")); + EXPECT_FALSE(column_properties_1->parquet_cipher().has_value()); + + auto column_properties_2 = properties->column_encryption_properties("col4"); + EXPECT_EQ("col4", column_properties_2->column_path()); + EXPECT_TRUE(column_properties_2->is_encrypted()); + EXPECT_FALSE(column_properties_2->is_encrypted_with_footer_key()); + EXPECT_THAT(column_properties_2->key_metadata(), HasSubstr("kc2")); + EXPECT_FALSE(column_properties_2->parquet_cipher().has_value()); +} + +TEST_F(CryptoFactoryTest, ExternalEncryptionConfig) { + ExternalEncryptionConfiguration config("kf"); + config.plaintext_footer = true; + config.column_keys = "kc3:col3,col4"; + + std::unordered_map per_column_encryption; + ColumnEncryptionAttributes attributes_1; + attributes_1.parquet_cipher = ParquetCipher::AES_GCM_CTR_V1; + attributes_1.key_id = "kc1"; + per_column_encryption["col1"] = attributes_1; + + ColumnEncryptionAttributes attributes_2; + attributes_2.parquet_cipher = ParquetCipher::AES_GCM_V1; + attributes_2.key_id = "kc2"; + per_column_encryption["col2"] = attributes_2; + + config.per_column_encryption = per_column_encryption; + + auto properties = crypto_factory_.GetExternalFileEncryptionProperties(kms_config_, config); + EXPECT_EQ("kf", properties->footer_key()); + EXPECT_TRUE(properties->footer_key_metadata().size() > 0); + EXPECT_EQ(ParquetCipher::AES_GCM_V1, properties->algorithm().algorithm); + EXPECT_FALSE(properties->encrypted_footer()); + EXPECT_TRUE(properties->encrypted_columns().size() == 4); + + auto column_properties_1 = properties->column_encryption_properties("col1"); + EXPECT_EQ("col1", column_properties_1->column_path()); + EXPECT_TRUE(column_properties_1->is_encrypted()); + EXPECT_FALSE(column_properties_1->is_encrypted_with_footer_key()); + EXPECT_THAT(column_properties_1->key_metadata(), HasSubstr("kc1")); + EXPECT_TRUE(column_properties_1->parquet_cipher().has_value()); + EXPECT_EQ(ParquetCipher::AES_GCM_CTR_V1, column_properties_1->parquet_cipher().value()); + + auto column_properties_2 = properties->column_encryption_properties("col2"); + EXPECT_EQ("col2", column_properties_2->column_path()); + EXPECT_TRUE(column_properties_2->is_encrypted()); + EXPECT_FALSE(column_properties_2->is_encrypted_with_footer_key()); + EXPECT_THAT(column_properties_2->key_metadata(), HasSubstr("kc2")); + EXPECT_TRUE(column_properties_2->parquet_cipher().has_value()); + EXPECT_EQ(ParquetCipher::AES_GCM_V1, column_properties_2->parquet_cipher().value()); + + auto column_properties_3 = properties->column_encryption_properties("col3"); + EXPECT_EQ("col3", column_properties_3->column_path()); + EXPECT_TRUE(column_properties_3->is_encrypted()); + EXPECT_FALSE(column_properties_3->is_encrypted_with_footer_key()); + EXPECT_THAT(column_properties_3->key_metadata(), HasSubstr("kc3")); + EXPECT_FALSE(column_properties_3->parquet_cipher().has_value()); +} + +TEST_F(CryptoFactoryTest, ColumnRepeatedInMapsThrowsException) { + ExternalEncryptionConfiguration config("kf"); + config.plaintext_footer = true; + config.column_keys = "kc3:col3,col2"; + + std::unordered_map per_column_encryption; + ColumnEncryptionAttributes attributes_1; + attributes_1.parquet_cipher = ParquetCipher::AES_GCM_CTR_V1; + attributes_1.key_id = "kc1"; + per_column_encryption["col1"] = attributes_1; + + ColumnEncryptionAttributes attributes_2; + attributes_2.parquet_cipher = ParquetCipher::AES_GCM_V1; + attributes_2.key_id = "kc2"; + per_column_encryption["col2"] = attributes_2; + + config.per_column_encryption = per_column_encryption; + + try { + auto properties = crypto_factory_.GetExternalFileEncryptionProperties(kms_config_, config); + FAIL() << "ParquetException should have been raised"; + } catch (const ParquetException& xcp) { + EXPECT_THAT(xcp.what(), HasSubstr("Multiple keys defined for column [col2]")); + } catch (...) { + FAIL() << "Caught unexpected exception type"; + } +} + +} // namespace parquet::encryption::test diff --git a/cpp/src/parquet/encryption/encryption.cc b/cpp/src/parquet/encryption/encryption.cc index 95a717c5e3c7..a1cd009cfca7 100644 --- a/cpp/src/parquet/encryption/encryption.cc +++ b/cpp/src/parquet/encryption/encryption.cc @@ -20,6 +20,7 @@ #include #include +#include #include #include "arrow/util/logging_internal.h" @@ -80,6 +81,12 @@ ColumnEncryptionProperties::Builder* ColumnEncryptionProperties::Builder::key_id return this; } +ColumnEncryptionProperties::Builder* ColumnEncryptionProperties::Builder::parquet_cipher( + ParquetCipher::type parquet_cipher) { + this->parquet_cipher_ = parquet_cipher; + return this; +} + FileDecryptionProperties::Builder* FileDecryptionProperties::Builder::column_keys( const ColumnPathToDecryptionPropertiesMap& column_decryption_properties) { if (column_decryption_properties.size() == 0) return this; @@ -129,6 +136,58 @@ FileDecryptionProperties::Builder* FileDecryptionProperties::Builder::aad_prefix return this; } +ExternalFileDecryptionProperties::Builder* ExternalFileDecryptionProperties::Builder::app_context( + const std::string& context) { + if (!app_context_.empty()) { + throw ParquetException("App context already set"); + } + + if (context.empty()) { + return this; + } + + app_context_ = context; + return this; +} + +ExternalFileDecryptionProperties::Builder* +ExternalFileDecryptionProperties::Builder::connection_config( + const std::map>& config) { + if (connection_config_.size() != 0) { + throw ParquetException("Connection config already set"); + } + + if (config.size() == 0) { + return this; + } + + connection_config_ = config; + return this; +} + +std::shared_ptr +ExternalFileDecryptionProperties::Builder::build_external() { + return std::shared_ptr(new ExternalFileDecryptionProperties( + footer_key_, key_retriever_, check_plaintext_footer_integrity_, aad_prefix_, + aad_prefix_verifier_, column_decryption_properties_, plaintext_files_allowed_, + app_context_, connection_config_)); +} + +ExternalFileDecryptionProperties::ExternalFileDecryptionProperties( + const std::string& footer_key, + std::shared_ptr key_retriever, + bool check_plaintext_footer_integrity, const std::string& aad_prefix, + std::shared_ptr aad_prefix_verifier, + const ColumnPathToDecryptionPropertiesMap& column_decryption_properties, + bool plaintext_files_allowed, + const std::string& app_context, + const std::map>& connection_config) + : FileDecryptionProperties(footer_key, key_retriever, check_plaintext_footer_integrity, + aad_prefix, aad_prefix_verifier, column_decryption_properties, + plaintext_files_allowed), + app_context_(app_context), + connection_config_(connection_config) {} + ColumnDecryptionProperties::Builder* ColumnDecryptionProperties::Builder::key( const std::string& key) { if (key.empty()) return this; @@ -138,9 +197,15 @@ ColumnDecryptionProperties::Builder* ColumnDecryptionProperties::Builder::key( return this; } +ColumnDecryptionProperties::Builder* ColumnDecryptionProperties::Builder::parquet_cipher( + ParquetCipher::type parquet_cipher) { + parquet_cipher_ = parquet_cipher; + return this; +} + std::shared_ptr ColumnDecryptionProperties::Builder::build() { return std::shared_ptr( - new ColumnDecryptionProperties(column_path_, key_)); + new ColumnDecryptionProperties(column_path_, key_, parquet_cipher_)); } FileEncryptionProperties::Builder* FileEncryptionProperties::Builder::footer_key_metadata( @@ -181,11 +246,10 @@ FileEncryptionProperties::Builder::disable_aad_prefix_storage() { return this; } -ColumnEncryptionProperties::ColumnEncryptionProperties(bool encrypted, - const std::string& column_path, - const std::string& key, - const std::string& key_metadata) - : column_path_(column_path) { +ColumnEncryptionProperties::ColumnEncryptionProperties( + bool encrypted, const std::string& column_path, const std::string& key, + const std::string& key_metadata, std::optional parquet_cipher) + : column_path_(column_path), parquet_cipher_(parquet_cipher) { DCHECK(!column_path.empty()); if (!encrypted) { DCHECK(key.empty() && key_metadata.empty()); @@ -205,9 +269,10 @@ ColumnEncryptionProperties::ColumnEncryptionProperties(bool encrypted, key_ = key; } -ColumnDecryptionProperties::ColumnDecryptionProperties(const std::string& column_path, - const std::string& key) - : column_path_(column_path) { +ColumnDecryptionProperties::ColumnDecryptionProperties( + const std::string& column_path, const std::string& key, + std::optional parquet_cipher) + : column_path_(column_path), parquet_cipher_(parquet_cipher) { DCHECK(!column_path.empty()); if (!key.empty()) { @@ -318,4 +383,52 @@ FileEncryptionProperties::FileEncryptionProperties( } } +ExternalFileEncryptionProperties::Builder* ExternalFileEncryptionProperties::Builder::app_context( + const std::string& context) { + if (!app_context_.empty()) { + throw ParquetException("App context already set"); + } + + if (context.empty()) { + return this; + } + + app_context_ = context; + return this; +} + +ExternalFileEncryptionProperties::Builder* +ExternalFileEncryptionProperties::Builder::connection_config( + const std::map& config) { + if (connection_config_.size() != 0) { + throw ParquetException("Connection config already set"); + } + + if (config.size() == 0) { + return this; + } + + connection_config_ = config; + return this; +} + +std::shared_ptr +ExternalFileEncryptionProperties::Builder::build_external() { + return std::shared_ptr(new ExternalFileEncryptionProperties( + parquet_cipher_, footer_key_, footer_key_metadata_, encrypted_footer_, aad_prefix_, + store_aad_prefix_in_file_, encrypted_columns_, app_context_, connection_config_)); +} + +ExternalFileEncryptionProperties::ExternalFileEncryptionProperties( + ParquetCipher::type cipher, const std::string& footer_key, + const std::string& footer_key_metadata, bool encrypted_footer, + const std::string& aad_prefix, bool store_aad_prefix_in_file, + const ColumnPathToEncryptionPropertiesMap& encrypted_columns, + const std::string& app_context, + const std::map& connection_config) + : FileEncryptionProperties(cipher, footer_key, footer_key_metadata, encrypted_footer, + aad_prefix, store_aad_prefix_in_file, encrypted_columns), + app_context_(app_context), + connection_config_(connection_config) {} + } // namespace parquet diff --git a/cpp/src/parquet/encryption/encryption.h b/cpp/src/parquet/encryption/encryption.h index 6604e329911a..defa156c024c 100644 --- a/cpp/src/parquet/encryption/encryption.h +++ b/cpp/src/parquet/encryption/encryption.h @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -125,9 +126,14 @@ class PARQUET_EXPORT ColumnEncryptionProperties { /// key_id will be converted to metadata (UTF-8 array). Builder* key_id(const std::string& key_id); + /// Set ParquetCipher type to use. + /// This field is declared as optional. If the value is not set, then the ParquetCipher + /// declared in the FileEncryptionProperties will be used. + Builder* parquet_cipher(ParquetCipher::type parquet_cipher); + std::shared_ptr build() { - return std::shared_ptr( - new ColumnEncryptionProperties(encrypted_, column_path_, key_, key_metadata_)); + return std::shared_ptr(new ColumnEncryptionProperties( + encrypted_, column_path_, key_, key_metadata_, parquet_cipher_)); } private: @@ -135,6 +141,7 @@ class PARQUET_EXPORT ColumnEncryptionProperties { bool encrypted_; std::string key_; std::string key_metadata_; + std::optional parquet_cipher_; Builder(const std::string path, bool encrypted) : column_path_(path), encrypted_(encrypted) {} @@ -146,6 +153,9 @@ class PARQUET_EXPORT ColumnEncryptionProperties { std::string key() const { return key_; } std::string key_metadata() const { return key_metadata_; } + /// Check whether the optional has a value before using. + std::optional parquet_cipher() const { return parquet_cipher_; } + ColumnEncryptionProperties() = default; ColumnEncryptionProperties(const ColumnEncryptionProperties& other) = default; ColumnEncryptionProperties(ColumnEncryptionProperties&& other) = default; @@ -158,9 +168,10 @@ class PARQUET_EXPORT ColumnEncryptionProperties { bool encrypted_with_footer_key_; std::string key_; std::string key_metadata_; + std::optional parquet_cipher_; explicit ColumnEncryptionProperties(bool encrypted, const std::string& column_path, - const std::string& key, - const std::string& key_metadata); + const std::string& key, const std::string& key_metadata, + std::optional parquet_cipher); }; class PARQUET_EXPORT ColumnDecryptionProperties { @@ -178,11 +189,17 @@ class PARQUET_EXPORT ColumnDecryptionProperties { /// key length must be either 16, 24 or 32 bytes. Builder* key(const std::string& key); + /// Set ParquetCipher type to use. + /// This field is declared as optional. If the value is not set, then the ParquetCipher + /// declared in the InternalFileDecryptor will be used. + Builder* parquet_cipher(ParquetCipher::type parquet_cipher); + std::shared_ptr build(); private: const std::string column_path_; std::string key_; + std::optional parquet_cipher_; }; ColumnDecryptionProperties() = default; @@ -194,15 +211,20 @@ class PARQUET_EXPORT ColumnDecryptionProperties { std::string column_path() const { return column_path_; } std::string key() const { return key_; } + /// Check whether the optional has a value before using. + std::optional parquet_cipher() const { return parquet_cipher_; } + private: const std::string column_path_; std::string key_; + std::optional parquet_cipher_; /// This class is only required for setting explicit column decryption keys - /// to override key retriever (or to provide keys when key metadata and/or /// key retriever are not available) explicit ColumnDecryptionProperties(const std::string& column_path, - const std::string& key); + const std::string& key, + std::optional parquet_cipher); }; class PARQUET_EXPORT AADPrefixVerifier { @@ -293,7 +315,7 @@ class PARQUET_EXPORT FileDecryptionProperties { aad_prefix_verifier_, column_decryption_properties_, plaintext_files_allowed_)); } - private: + protected: std::string footer_key_; std::string aad_prefix_; std::shared_ptr aad_prefix_verifier_; @@ -338,6 +360,7 @@ class PARQUET_EXPORT FileDecryptionProperties { bool check_plaintext_footer_integrity_; bool plaintext_files_allowed_; + protected: FileDecryptionProperties( const std::string& footer_key, std::shared_ptr key_retriever, @@ -347,6 +370,47 @@ class PARQUET_EXPORT FileDecryptionProperties { bool plaintext_files_allowed); }; +class PARQUET_EXPORT ExternalFileDecryptionProperties : public FileDecryptionProperties { + public: + class PARQUET_EXPORT Builder : public FileDecryptionProperties::Builder { + public: + Builder() : FileDecryptionProperties::Builder() {} + + Builder* app_context(const std::string& context); + + Builder* connection_config( + const std::map>& config); + + std::shared_ptr build_external(); + + private: + std::string app_context_; + std::map> connection_config_; + }; + + const std::string& app_context() const { + return app_context_; + } + + const std::map>& connection_config() const { + return connection_config_; + } + + private: + std::string app_context_; + std::map> connection_config_; + + ExternalFileDecryptionProperties( + const std::string& footer_key, + std::shared_ptr key_retriever, + bool check_plaintext_footer_integrity, const std::string& aad_prefix, + std::shared_ptr aad_prefix_verifier, + const ColumnPathToDecryptionPropertiesMap& column_decryption_properties, + bool plaintext_files_allowed, + const std::string& app_context, + const std::map>& connection_config); +}; + class PARQUET_EXPORT FileEncryptionProperties { public: class PARQUET_EXPORT Builder { @@ -399,7 +463,7 @@ class PARQUET_EXPORT FileEncryptionProperties { aad_prefix_, store_aad_prefix_in_file_, encrypted_columns_)); } - private: + protected: ParquetCipher::type parquet_cipher_; bool encrypted_footer_; std::string footer_key_; @@ -439,10 +503,93 @@ class PARQUET_EXPORT FileEncryptionProperties { bool store_aad_prefix_in_file_; ColumnPathToEncryptionPropertiesMap encrypted_columns_; + protected: FileEncryptionProperties(ParquetCipher::type cipher, const std::string& footer_key, const std::string& footer_key_metadata, bool encrypted_footer, const std::string& aad_prefix, bool store_aad_prefix_in_file, const ColumnPathToEncryptionPropertiesMap& encrypted_columns); }; +class PARQUET_EXPORT ExternalFileEncryptionProperties : public FileEncryptionProperties { + public: + + class PARQUET_EXPORT Builder : public FileEncryptionProperties::Builder { + public: + + explicit Builder(const std::string& footer_key) + : FileEncryptionProperties::Builder(footer_key) {} + + /// Valid JSON string with additional application context needed for security checks. + Builder* app_context(const std::string& context); + + /// Key/value map of the location of configuration files needed by the external + /// encryption service, including location of a dynamically-linked library, or config files + /// where the external service can find urls, certificates, and parameters needed to make a + /// remote service call. + Builder* connection_config(const std::map& config); + + /// Forward all base class property methods to the base class Builder so we can return the + /// correct Builder type. + Builder* set_plaintext_footer() { + FileEncryptionProperties::Builder::set_plaintext_footer(); + return this; + } + + Builder* algorithm(ParquetCipher::type parquet_cipher) { + FileEncryptionProperties::Builder::algorithm(parquet_cipher); + return this; + } + + Builder* footer_key_id(const std::string& key_id) { + FileEncryptionProperties::Builder::footer_key_id(key_id); + return this; + } + + Builder* footer_key_metadata(const std::string& footer_key_metadata) { + FileEncryptionProperties::Builder::footer_key_metadata(footer_key_metadata); + return this; + } + + Builder* aad_prefix(const std::string& aad_prefix) { + FileEncryptionProperties::Builder::aad_prefix(aad_prefix); + return this; + } + + Builder* disable_aad_prefix_storage() { + FileEncryptionProperties::Builder::disable_aad_prefix_storage(); + return this; + } + + Builder* encrypted_columns(const ColumnPathToEncryptionPropertiesMap& encrypted_columns) { + FileEncryptionProperties::Builder::encrypted_columns(encrypted_columns); + return this; + } + + std::shared_ptr build_external(); + + private: + std::string app_context_; + std::map connection_config_; + }; + + const std::string& app_context() const { + return app_context_; + } + + const std::map& connection_config() const { + return connection_config_; + } + + private: + std::string app_context_; + std::map connection_config_; + + ExternalFileEncryptionProperties(ParquetCipher::type cipher, const std::string& footer_key, + const std::string& footer_key_metadata, bool encrypted_footer, + const std::string& aad_prefix, bool store_aad_prefix_in_file, + const ColumnPathToEncryptionPropertiesMap& encrypted_columns, + const std::string& app_context, + const std::map& connection_config); +}; + } // namespace parquet diff --git a/cpp/src/parquet/encryption/encryption_internal.cc b/cpp/src/parquet/encryption/encryption_internal.cc index 657198f1701a..a833941681cc 100644 --- a/cpp/src/parquet/encryption/encryption_internal.cc +++ b/cpp/src/parquet/encryption/encryption_internal.cc @@ -61,6 +61,11 @@ class AesCryptoContext { length_buffer_length_ = include_length ? kBufferSizeLength : 0; ciphertext_size_delta_ = length_buffer_length_ + kNonceLength; + if (ParquetCipher::EXTERNAL_DBPA_V1 == alg_id) { + std::stringstream ss; + ss << "ExternalDataBatchProtectionAgent (DBPA) algorithm is not yet implemented"; + throw ParquetException(ss.str()); + } if (ParquetCipher::AES_GCM_V1 != alg_id && ParquetCipher::AES_GCM_CTR_V1 != alg_id) { std::stringstream ss; ss << "Crypto algorithm " << alg_id << " is not supported"; diff --git a/cpp/src/parquet/encryption/properties_test.cc b/cpp/src/parquet/encryption/properties_test.cc index 895cf6c63431..7bbced067192 100644 --- a/cpp/src/parquet/encryption/properties_test.cc +++ b/cpp/src/parquet/encryption/properties_test.cc @@ -48,6 +48,46 @@ TEST(TestColumnEncryptionProperties, ColumnEncryptedWithFooterKey) { ASSERT_EQ(true, column_props_1->is_encrypted_with_footer_key()); } +TEST(TestColumnEncryptionProperties, ColumnParquetCipherNotSpecified) { + std::string column_path = "column_path"; + ColumnEncryptionProperties::Builder column_builder(column_path); + std::shared_ptr properties = column_builder.build(); + + ASSERT_EQ(column_path, properties->column_path()); + ASSERT_EQ(false, properties->parquet_cipher().has_value()); +} + +TEST(TestColumnEncryptionProperties, ColumnParquetCipherSpecified) { + std::string column_path = "column_path"; + ColumnEncryptionProperties::Builder column_builder(column_path); + column_builder.parquet_cipher(ParquetCipher::AES_GCM_CTR_V1); + std::shared_ptr properties = column_builder.build(); + + ASSERT_EQ(column_path, properties->column_path()); + ASSERT_EQ(true, properties->parquet_cipher().has_value()); + ASSERT_EQ(ParquetCipher::AES_GCM_CTR_V1, properties->parquet_cipher().value()); +} + +TEST(TestColumnDecryptionProperties, ColumnParquetCipherNotSpecified) { + std::string column_path = "column_path"; + ColumnDecryptionProperties::Builder column_builder(column_path); + std::shared_ptr properties = column_builder.build(); + + ASSERT_EQ(column_path, properties->column_path()); + ASSERT_EQ(false, properties->parquet_cipher().has_value()); +} + +TEST(TestColumnDecryptionProperties, ColumnParquetCipherSpecified) { + std::string column_path = "column_path"; + ColumnDecryptionProperties::Builder column_builder(column_path); + column_builder.parquet_cipher(ParquetCipher::AES_GCM_CTR_V1); + std::shared_ptr properties = column_builder.build(); + + ASSERT_EQ(column_path, properties->column_path()); + ASSERT_EQ(true, properties->parquet_cipher().has_value()); + ASSERT_EQ(ParquetCipher::AES_GCM_CTR_V1, properties->parquet_cipher().value()); +} + // Encrypt all columns and the footer with the same key. // (uniform encryption) TEST(TestEncryptionProperties, UniformEncryption) { @@ -210,6 +250,128 @@ TEST(TestEncryptionProperties, UseAES_GCM_CTR_V1Algorithm) { ASSERT_EQ(ParquetCipher::AES_GCM_CTR_V1, props->algorithm().algorithm); } +TEST(TestExternalFileEncryptionProperties, SuperClassFieldsSetCorrectly) { + std::string column_name_1 = "column_1"; + ColumnEncryptionProperties::Builder column_builder_1(column_name_1); + column_builder_1.key(kColumnEncryptionKey1); + column_builder_1.key_id("kc1"); + + std::string column_name_2 = "column_2"; + ColumnEncryptionProperties::Builder column_builder_2(column_name_2); + column_builder_2.key(kColumnEncryptionKey2); + column_builder_2.key_id("kc2"); + + std::map> encrypted_columns; + encrypted_columns[column_name_1] = column_builder_1.build(); + encrypted_columns[column_name_2] = column_builder_2.build(); + + ExternalFileEncryptionProperties::Builder builder(kFooterEncryptionKey); + builder.footer_key_metadata("kf"); + builder.encrypted_columns(encrypted_columns); + std::shared_ptr props = builder.build_external(); + + ASSERT_EQ(true, props->encrypted_footer()); + ASSERT_EQ(kDefaultEncryptionAlgorithm, props->algorithm().algorithm); + ASSERT_EQ(kFooterEncryptionKey, props->footer_key()); + + std::shared_ptr out_col_props_1 = + props->column_encryption_properties(column_name_1); + + ASSERT_EQ(column_name_1, out_col_props_1->column_path()); + ASSERT_EQ(true, out_col_props_1->is_encrypted()); + ASSERT_EQ(false, out_col_props_1->is_encrypted_with_footer_key()); + ASSERT_EQ(kColumnEncryptionKey1, out_col_props_1->key()); + ASSERT_EQ("kc1", out_col_props_1->key_metadata()); + + std::shared_ptr out_col_props_2 = + props->column_encryption_properties(column_name_2); + + ASSERT_EQ(column_name_2, out_col_props_2->column_path()); + ASSERT_EQ(true, out_col_props_2->is_encrypted()); + ASSERT_EQ(false, out_col_props_2->is_encrypted_with_footer_key()); + ASSERT_EQ(kColumnEncryptionKey2, out_col_props_2->key()); + ASSERT_EQ("kc2", out_col_props_2->key_metadata()); + + std::string column_name_3 = "column_3"; + std::shared_ptr out_col_props_3 = + props->column_encryption_properties(column_name_3); + + ASSERT_EQ(NULLPTR, out_col_props_3); + ASSERT_EQ(true, props->app_context().empty()); + ASSERT_EQ(true, props->connection_config().size() == 0); +} + +// The subclass adds two additional fields +TEST(TestExternalFileEncryptionProperties, SetExternalContextAndConfig) { + std::string app_context = "{\n" + " \"user_id\": \"abc123\",\n" + " \"location\": {\n" + " \"lat\": 10.0,\n" + " \"lon\": -84.0\n" + " }\n" + "}"; + std::map connection_config; + connection_config["lib_location"] = "path/to/lib.so"; + connection_config["config_file"] = "path/to/config/file"; + + ExternalFileEncryptionProperties::Builder builder(kFooterEncryptionKey); + builder.app_context(app_context); + builder.connection_config(connection_config); + std::shared_ptr props = builder.build_external(); + + ASSERT_EQ(false, props->app_context().empty()); + ASSERT_EQ(app_context, props->app_context()); + ASSERT_EQ(false, props->connection_config().size() == 0); + ASSERT_EQ(connection_config, props->connection_config()); +} + +TEST(TestExternalFileEncryptionProperties, EncryptTwoColumnsWithDifferentAlgorithms) { + std::string column_name_1 = "column_1"; + ColumnEncryptionProperties::Builder column_builder_1(column_name_1); + column_builder_1.key(kColumnEncryptionKey1); + column_builder_1.key_id("kc1"); + column_builder_1.parquet_cipher(ParquetCipher::AES_GCM_V1); + + std::string column_name_2 = "column_2"; + ColumnEncryptionProperties::Builder column_builder_2(column_name_2); + column_builder_2.key(kColumnEncryptionKey2); + column_builder_2.key_id("kc2"); + column_builder_2.parquet_cipher(ParquetCipher::AES_GCM_CTR_V1); + + std::map> encrypted_columns; + encrypted_columns[column_name_1] = column_builder_1.build(); + encrypted_columns[column_name_2] = column_builder_2.build(); + + ExternalFileEncryptionProperties::Builder builder(kFooterEncryptionKey); + builder.footer_key_metadata("kf"); + builder.encrypted_columns(encrypted_columns); + std::shared_ptr props = builder.build_external(); + + ASSERT_EQ(true, props->encrypted_footer()); + ASSERT_EQ(kDefaultEncryptionAlgorithm, props->algorithm().algorithm); + ASSERT_EQ(kFooterEncryptionKey, props->footer_key()); + + std::shared_ptr out_col_props_1 = + props->column_encryption_properties(column_name_1); + + ASSERT_EQ(column_name_1, out_col_props_1->column_path()); + ASSERT_EQ(true, out_col_props_1->is_encrypted()); + ASSERT_EQ(false, out_col_props_1->is_encrypted_with_footer_key()); + ASSERT_EQ(kColumnEncryptionKey1, out_col_props_1->key()); + ASSERT_EQ(ParquetCipher::AES_GCM_V1, out_col_props_1->parquet_cipher()); + ASSERT_EQ("kc1", out_col_props_1->key_metadata()); + + std::shared_ptr out_col_props_2 = + props->column_encryption_properties(column_name_2); + + ASSERT_EQ(column_name_2, out_col_props_2->column_path()); + ASSERT_EQ(true, out_col_props_2->is_encrypted()); + ASSERT_EQ(false, out_col_props_2->is_encrypted_with_footer_key()); + ASSERT_EQ(kColumnEncryptionKey2, out_col_props_2->key()); + ASSERT_EQ(ParquetCipher::AES_GCM_CTR_V1, out_col_props_2->parquet_cipher()); + ASSERT_EQ("kc2", out_col_props_2->key_metadata()); +} + TEST(TestDecryptionProperties, UseKeyRetriever) { std::shared_ptr string_kr1 = std::make_shared(); diff --git a/cpp/src/parquet/encryption/write_configurations_test.cc b/cpp/src/parquet/encryption/write_configurations_test.cc index f27da8269487..b7f1d1a17158 100644 --- a/cpp/src/parquet/encryption/write_configurations_test.cc +++ b/cpp/src/parquet/encryption/write_configurations_test.cc @@ -223,6 +223,31 @@ TEST_F(TestEncryptionConfiguration, EncryptTwoColumnsAndFooterUseAES_GCM_CTR) { "tmp_encrypt_columns_and_footer_ctr.parquet.encrypted")); } +TEST_F(TestEncryptionConfiguration, EncryptOneColumnAndUseExternalDBPA) { + std::map> + encryption_cols; + parquet::ColumnEncryptionProperties::Builder encryption_col_builder( + path_to_double_field_); + encryption_col_builder.key(kColumnEncryptionKey1_)->key_id("kc1"); + + encryption_cols[path_to_double_field_] = encryption_col_builder.build(); + + parquet::FileEncryptionProperties::Builder file_encryption_builder( + kFooterEncryptionKey_); + + try { + this->EncryptFile(file_encryption_builder.footer_key_metadata("kf") + ->encrypted_columns(encryption_cols) + ->algorithm(parquet::ParquetCipher::EXTERNAL_DBPA_V1) + ->build(), + "tmp_encrypt_one_column_and_use_external_dbpa.parquet.encrypted"); + FAIL() << "Expected ParquetException was not thrown"; + } catch (const parquet::ParquetException& e) { + EXPECT_STREQ("ExternalDataBatchProtectionAgent (DBPA) algorithm is not yet implemented", + e.what()); + } +} + // Set temp_dir before running the write/read tests. The encrypted files will // be written/read from this directory. void TestEncryptionConfiguration::SetUpTestCase() { diff --git a/cpp/src/parquet/parquet.thrift b/cpp/src/parquet/parquet.thrift index e3cc5adb9648..1037b938a25a 100644 --- a/cpp/src/parquet/parquet.thrift +++ b/cpp/src/parquet/parquet.thrift @@ -1237,9 +1237,12 @@ struct AesGcmCtrV1 { 3: optional bool supply_aad_prefix } +struct ExternalDBPAV1 {} + union EncryptionAlgorithm { 1: AesGcmV1 AES_GCM_V1 2: AesGcmCtrV1 AES_GCM_CTR_V1 + 3: ExternalDBPAV1 EXTERNAL_DBPA_V1 } /** diff --git a/cpp/src/parquet/thrift_internal.h b/cpp/src/parquet/thrift_internal.h index e18fb36c02d5..5fe6b69846bc 100644 --- a/cpp/src/parquet/thrift_internal.h +++ b/cpp/src/parquet/thrift_internal.h @@ -241,6 +241,8 @@ static inline EncryptionAlgorithm FromThrift(format::EncryptionAlgorithm encrypt } else if (encryption.__isset.AES_GCM_CTR_V1) { encryption_algorithm.algorithm = ParquetCipher::AES_GCM_CTR_V1; encryption_algorithm.aad = FromThrift(encryption.AES_GCM_CTR_V1); + } else if (encryption.__isset.EXTERNAL_DBPA_V1) { + encryption_algorithm.algorithm = ParquetCipher::EXTERNAL_DBPA_V1; } else { throw ParquetException("Unsupported algorithm"); } @@ -382,12 +384,19 @@ static inline format::AesGcmCtrV1 ToAesGcmCtrV1Thrift(AadMetadata aad) { return aesGcmCtrV1; } +static inline format::ExternalDBPAV1 ToExternalDBPAV1Thrift() { + format::ExternalDBPAV1 externalDBPAV1; + return externalDBPAV1; +} + static inline format::EncryptionAlgorithm ToThrift(EncryptionAlgorithm encryption) { format::EncryptionAlgorithm encryption_algorithm; if (encryption.algorithm == ParquetCipher::AES_GCM_V1) { encryption_algorithm.__set_AES_GCM_V1(ToAesGcmV1Thrift(encryption.aad)); - } else { + } else if (encryption.algorithm == ParquetCipher::AES_GCM_CTR_V1) { encryption_algorithm.__set_AES_GCM_CTR_V1(ToAesGcmCtrV1Thrift(encryption.aad)); + } else { + encryption_algorithm.__set_EXTERNAL_DBPA_V1(ToExternalDBPAV1Thrift()); } return encryption_algorithm; } diff --git a/cpp/src/parquet/types.h b/cpp/src/parquet/types.h index 1ad6dc962861..61b35cfb46c2 100644 --- a/cpp/src/parquet/types.h +++ b/cpp/src/parquet/types.h @@ -518,7 +518,7 @@ PARQUET_EXPORT std::unique_ptr GetCodec(Compression::type codec, int compression_level); struct ParquetCipher { - enum type { AES_GCM_V1 = 0, AES_GCM_CTR_V1 = 1 }; + enum type { AES_GCM_V1 = 0, AES_GCM_CTR_V1 = 1, EXTERNAL_DBPA_V1 = 2 }; }; struct AadMetadata { diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd index f7df8a052a1f..f134b9d7fc0b 100644 --- a/python/pyarrow/_parquet.pxd +++ b/python/pyarrow/_parquet.pxd @@ -195,6 +195,7 @@ cdef extern from "parquet/api/schema.h" namespace "parquet" nogil: enum ParquetCipher" parquet::ParquetCipher::type": ParquetCipher_AES_GCM_V1" parquet::ParquetCipher::AES_GCM_V1" ParquetCipher_AES_GCM_CTR_V1" parquet::ParquetCipher::AES_GCM_CTR_V1" + ParquetCipher_EXTERNAL_DBPA_V1" parquet::ParquetCipher::EXTERNAL_DBPA_V1" struct AadMetadata: c_string aad_prefix @@ -579,6 +580,21 @@ cdef class FileEncryptionProperties: cdef inline shared_ptr[CFileEncryptionProperties] unwrap(self): return self.properties +cdef class ExternalFileEncryptionProperties(FileEncryptionProperties): + cdef: + shared_ptr[CExternalFileEncryptionProperties] properties + + @staticmethod + cdef inline ExternalFileEncryptionProperties wrap_external( + shared_ptr[CExternalFileEncryptionProperties] properties): + + result = ExternalFileEncryptionProperties() + result.properties = properties + return result + + cdef inline shared_ptr[CExternalFileEncryptionProperties] unwrap_external(self): + return self.properties + cdef shared_ptr[WriterProperties] _create_writer_properties( use_dictionary=*, compression=*, @@ -664,11 +680,18 @@ cdef extern from "parquet/encryption/encryption.h" namespace "parquet" nogil: cdef cppclass CFileDecryptionProperties\ " parquet::FileDecryptionProperties": pass - + + cdef cppclass CExternalFileDecryptionProperties\ + " parquet::ExternalFileDecryptionProperties": + pass cdef cppclass CFileEncryptionProperties\ " parquet::FileEncryptionProperties": pass + cdef cppclass CExternalFileEncryptionProperties\ + " parquet::ExternalFileEncryptionProperties": + pass + cdef class FileDecryptionProperties: """File-level decryption properties for the low-level API""" cdef: @@ -684,3 +707,19 @@ cdef class FileDecryptionProperties: cdef inline shared_ptr[CFileDecryptionProperties] unwrap(self): return self.properties + +cdef class ExternalFileDecryptionProperties: + """File-level decryption properties for the low-level API""" + cdef: + shared_ptr[CExternalFileDecryptionProperties] properties + + @staticmethod + cdef inline ExternalFileDecryptionProperties wrap( + shared_ptr[CExternalFileDecryptionProperties] properties): + + result = ExternalFileDecryptionProperties() + result.properties = properties + return result + + cdef inline shared_ptr[CExternalFileDecryptionProperties] unwrap_external(self): + return self.properties \ No newline at end of file diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx index 55c2866243b0..f92d2f27fcbf 100644 --- a/python/pyarrow/_parquet.pyx +++ b/python/pyarrow/_parquet.pyx @@ -45,7 +45,10 @@ from pyarrow.lib import (ArrowException, NativeFile, BufferOutputStream, tobytes, frombytes, is_threading_enabled) cimport cpython as cp +from libcpp.memory cimport shared_ptr +cdef extern from "memory" namespace "std": + shared_ptr[T] static_pointer_cast[T, U](shared_ptr[U] r) _DEFAULT_ROW_GROUP_SIZE = 1024*1024 _MAX_ROW_GROUP_SIZE = 64*1024*1024 @@ -2005,9 +2008,14 @@ cdef shared_ptr[WriterProperties] _create_writer_properties( # encryption if encryption_properties is not None: - props.encryption( - (encryption_properties).unwrap()) - + if isinstance(encryption_properties, ExternalFileEncryptionProperties): + props.encryption( + static_pointer_cast[CFileEncryptionProperties, CExternalFileEncryptionProperties] ( + (encryption_properties).unwrap_external())) + else: + + props.encryption((encryption_properties).unwrap()) + # For backwards compatibility reasons we cap the maximum row group size # at 64Mi rows. This could be changed in the future, though it would be # a breaking change. diff --git a/python/pyarrow/_parquet_encryption.pxd b/python/pyarrow/_parquet_encryption.pxd index d52669501a40..09f24a61ab98 100644 --- a/python/pyarrow/_parquet_encryption.pxd +++ b/python/pyarrow/_parquet_encryption.pxd @@ -22,11 +22,16 @@ from pyarrow.includes.common cimport * from pyarrow.includes.libparquet_encryption cimport * from pyarrow._parquet cimport (ParquetCipher, CFileEncryptionProperties, + CExternalFileEncryptionProperties, CFileDecryptionProperties, + CExternalFileDecryptionProperties, FileEncryptionProperties, + ExternalFileEncryptionProperties, FileDecryptionProperties, + ExternalFileDecryptionProperties, ParquetCipher_AES_GCM_V1, - ParquetCipher_AES_GCM_CTR_V1) + ParquetCipher_AES_GCM_CTR_V1, + ParquetCipher_EXTERNAL_DBPA_V1) from pyarrow.lib cimport _Weakrefable cdef class CryptoFactory(_Weakrefable): @@ -54,3 +59,16 @@ cdef shared_ptr[CCryptoFactory] pyarrow_unwrap_cryptofactory(object crypto_facto cdef shared_ptr[CKmsConnectionConfig] pyarrow_unwrap_kmsconnectionconfig(object kmsconnectionconfig) except * cdef shared_ptr[CEncryptionConfiguration] pyarrow_unwrap_encryptionconfig(object encryptionconfig) except * cdef shared_ptr[CDecryptionConfiguration] pyarrow_unwrap_decryptionconfig(object decryptionconfig) except * +cdef shared_ptr[CExternalDecryptionConfiguration] pyarrow_unwrap_external_decryptionconfig(object decryptionconfig) except * +cdef shared_ptr[CExternalEncryptionConfiguration] pyarrow_unwrap_external_encryptionconfig(object encryptionconfig) except * + + +cdef class ExternalEncryptionConfiguration(EncryptionConfiguration): + cdef shared_ptr[CExternalEncryptionConfiguration] external_configuration + cdef inline shared_ptr[CExternalEncryptionConfiguration] unwrap_external(self) nogil +cdef shared_ptr[CExternalEncryptionConfiguration] pyarrow_unwrap_external_encryptionconfig(object externalencryptionconfig) except * + +cdef class ExternalDecryptionConfiguration(DecryptionConfiguration): + cdef shared_ptr[CExternalDecryptionConfiguration] external_configuration + cdef inline shared_ptr[CExternalDecryptionConfiguration] unwrap_external(self) nogil +cdef shared_ptr[CExternalDecryptionConfiguration] pyarrow_unwrap_external_decryptionconfig(object externaldecryptionconfig) except * \ No newline at end of file diff --git a/python/pyarrow/_parquet_encryption.pyx b/python/pyarrow/_parquet_encryption.pyx index 81bd421dcbce..e9c2fc151a4c 100644 --- a/python/pyarrow/_parquet_encryption.pyx +++ b/python/pyarrow/_parquet_encryption.pyx @@ -27,6 +27,8 @@ from pyarrow.includes.libarrow cimport * from pyarrow.lib cimport _Weakrefable from pyarrow.lib import tobytes, frombytes +import json + cdef ParquetCipher cipher_from_name(name): name = name.upper() @@ -34,6 +36,8 @@ cdef ParquetCipher cipher_from_name(name): return ParquetCipher_AES_GCM_V1 elif name == 'AES_GCM_CTR_V1': return ParquetCipher_AES_GCM_CTR_V1 + elif name == 'EXTERNAL_DBPA_V1': + return ParquetCipher_EXTERNAL_DBPA_V1 else: raise ValueError(f'Invalid cipher name: {name!r}') @@ -43,6 +47,8 @@ cdef cipher_to_name(ParquetCipher cipher): return 'AES_GCM_V1' elif ParquetCipher_AES_GCM_CTR_V1 == cipher: return 'AES_GCM_CTR_V1' + elif ParquetCipher_EXTERNAL_DBPA_V1 == cipher: + return 'EXTERNAL_DBPA_V1' else: raise ValueError('Invalid cipher value: {0}'.format(cipher)) @@ -174,6 +180,130 @@ cdef class EncryptionConfiguration(_Weakrefable): cdef inline shared_ptr[CEncryptionConfiguration] unwrap(self) nogil: return self.configuration +cdef class ExternalEncryptionConfiguration(EncryptionConfiguration): + """ExternalEncryptionConfiguration is a Cython extension class that inherits from EncryptionConfiguration.""" + __slots__ = () + + def __init__(self, footer_key, *, column_keys=None, + encryption_algorithm=None, + plaintext_footer=None, double_wrapping=None, + cache_lifetime=None, internal_key_material=None, + data_key_length_bits=None, per_column_encryption=None, + app_context=None, connection_config=None): + + super().__init__(footer_key, + column_keys=column_keys, + encryption_algorithm=encryption_algorithm, + plaintext_footer=plaintext_footer, + double_wrapping=double_wrapping, + cache_lifetime=cache_lifetime, + internal_key_material=internal_key_material, + data_key_length_bits=data_key_length_bits) + + self.external_configuration.reset( + new CExternalEncryptionConfiguration(tobytes(footer_key))) + + if app_context is not None: + self.app_context = app_context + if connection_config is not None: + self.connection_config = connection_config + if per_column_encryption is not None: + self.per_column_encryption = per_column_encryption + + @property + def app_context(self): + """Get the application context as a dictionary.""" + app_context_str = frombytes(self.external_configuration.get().app_context) + if not app_context_str: + return {} + try: + return json.loads(app_context_str) + except Exception: + raise ValueError(f"Invalid JSON stored in app_context: {app_context_str}") + + @app_context.setter + def app_context(self, dict value): + """Set the application context from a dictionary.""" + if value is None: + raise ValueError("app_context must be JSON-serializable") + + try: + serialized = json.dumps(value) + self.external_configuration.get().app_context = tobytes(serialized) + except Exception: + raise TypeError(f"Failed to serialize app_context: {repr(value)}") + + @property + def connection_config(self): + """Get the connection configuration as a Python dictionary.""" + result = {} + + for pair in self.external_configuration.get().connection_config: + result[frombytes(pair.first)] = frombytes(pair.second) + + return result + + @connection_config.setter + def connection_config(self, dict value): + """Set the connection configuration from a Python dictionary.""" + if value is None: + raise ValueError("Connection config value cannot be None") + + cdef unordered_map[c_string, c_string] cpp_map + for k, v in value.items(): + if not isinstance(k, str): + raise TypeError(f"Connection config key must be str, got {type(k).__name__}") + if not isinstance(v, str): + raise TypeError(f"Connection config value must be str, got {type(v).__name__}") + cpp_map[tobytes(k)] = tobytes(v) + + self.external_configuration.get().connection_config = cpp_map + + @property + def per_column_encryption(self): + """Get the per_column_encryption as a Python dictionary.""" + + py_dict = {} + + for pair in self.external_configuration.get().per_column_encryption: + py_dict[frombytes(pair.first)] = { + "encryption_algorithm": cipher_to_name(pair.second.parquet_cipher), + "encryption_key": frombytes(pair.second.key_id) + } + + return py_dict + + @per_column_encryption.setter + def per_column_encryption(self, dict py_column_encryption): + """Set the per_column_encryption from a Python dictionary.""" + if py_column_encryption is None: + raise TypeError("per_column_encryption cannot be None") + + # Clear the existing C++ map first + self.external_configuration.get().per_column_encryption.clear() + + cdef CColumnEncryptionAttributes cpp_attrs + # Iterate over the Python dictionary + for py_key, py_attrs in py_column_encryption.items(): + if not isinstance(py_key, str) or not isinstance(py_attrs, dict): + raise TypeError("column_encryption keys must be strings and values must be dictionaries.") + + # Convert encryption_algorithm string to C++ ParquetCipher enum + if "encryption_algorithm" not in py_attrs or not isinstance(py_attrs["encryption_algorithm"], str): + raise ValueError("Each column must have 'encryption_algorithm' (string).") + + # Convert encryption_key string to C++ c_string + if "encryption_key" not in py_attrs or not isinstance(py_attrs["encryption_key"], str): + raise ValueError("Each column must have 'encryption_key' (string).") + + cpp_attrs.parquet_cipher = cipher_from_name(py_attrs["encryption_algorithm"]) + cpp_attrs.key_id = tobytes(py_attrs["encryption_key"]) + + self.external_configuration.get().per_column_encryption[tobytes(py_key)] = cpp_attrs + + cdef inline shared_ptr[CExternalEncryptionConfiguration] unwrap_external(self) nogil: + return self.external_configuration + cdef class DecryptionConfiguration(_Weakrefable): """Configuration of the decryption, such as cache timeout.""" @@ -197,6 +327,73 @@ cdef class DecryptionConfiguration(_Weakrefable): cdef inline shared_ptr[CDecryptionConfiguration] unwrap(self) nogil: return self.configuration +cdef class ExternalDecryptionConfiguration(DecryptionConfiguration): + """Configuration of the external decryption""" + # Avoid mistakingly creating attributes + __slots__ = () + + def __init__(self, *, cache_lifetime=None, app_context=None, connection_config=None): + super().__init__(cache_lifetime) + self.external_configuration.reset(new CExternalDecryptionConfiguration()) + + if app_context is not None: + self.app_context = app_context + if connection_config is not None: + self.connection_config = connection_config + + + @property + def app_context(self): + """Get the application context as a dictionary.""" + app_context_str = frombytes(self.external_configuration.get().app_context) + if not app_context_str: + return {} + try: + return json.loads(app_context_str) + except Exception: + raise ValueError(f"Invalid JSON stored in app_context: {app_context_str}") + + @app_context.setter + def app_context(self, dict value): + """Set the application context from a dictionary.""" + if value is None: + raise ValueError("app_context must be JSON-serializable") + + try: + serialized = json.dumps(value) + self.external_configuration.get().app_context = tobytes(serialized) + except Exception: + raise TypeError(f"Failed to serialize app_context: {repr(value)}") + + @property + def connection_config(self): + """Get the connection configuration as a Python dictionary.""" + result = {} + + for pair in self.external_configuration.get().connection_config: + result[frombytes(pair.first)] = frombytes(pair.second) + + return result + + @connection_config.setter + def connection_config(self, dict value): + """Set the connection configuration from a Python dictionary.""" + if value is None: + raise ValueError("Connection config value cannot be None") + + cdef unordered_map[c_string, c_string] cpp_map + for k, v in value.items(): + if not isinstance(k, str): + raise TypeError(f"Connection config key must be str, got {type(k).__name__}") + if not isinstance(v, str): + raise TypeError(f"Connection config value must be str, got {type(v).__name__}") + cpp_map[tobytes(k)] = tobytes(v) + + self.external_configuration.get().connection_config = cpp_map + + + cdef inline shared_ptr[CExternalDecryptionConfiguration] unwrap_external(self) nogil: + return self.external_configuration cdef class KmsConnectionConfig(_Weakrefable): """Configuration of the connection to the Key Management Service (KMS)""" @@ -412,6 +609,21 @@ cdef class CryptoFactory(_Weakrefable): file_encryption_properties_result) return FileEncryptionProperties.wrap(file_encryption_properties) + def external_file_encryption_properties(self, + KmsConnectionConfig kms_connection_config, + ExternalEncryptionConfiguration external_encryption_config): + cdef: + CResult[shared_ptr[CExternalFileEncryptionProperties]] \ + external_file_encryption_properties_result + with nogil: + external_file_encryption_properties_result = \ + self.factory.get().SafeGetExternalFileEncryptionProperties( + deref(kms_connection_config.unwrap().get()), + deref(external_encryption_config.unwrap_external().get())) + external_file_encryption_properties = GetResultValue( + external_file_encryption_properties_result) + return ExternalFileEncryptionProperties.wrap_external(external_file_encryption_properties) + def file_decryption_properties( self, KmsConnectionConfig kms_connection_config, @@ -449,6 +661,43 @@ cdef class CryptoFactory(_Weakrefable): c_file_decryption_properties) return FileDecryptionProperties.wrap(file_decryption_properties) + def external_file_decryption_properties( + self, + KmsConnectionConfig kms_connection_config, + ExternalDecryptionConfiguration decryption_config=None): + """Create file decryption properties. + + Parameters + ---------- + kms_connection_config : KmsConnectionConfig + Configuration of connection to KMS + + decryption_config : ExternalDecryptionConfiguration, default None + Configuration of the decryption, such as cache timeout. + Can be None. + + Returns + ------- + file_decryption_properties : ExternalFileDecryptionProperties + File decryption properties. + """ + cdef: + CExternalDecryptionConfiguration c_decryption_config + CResult[shared_ptr[CExternalFileDecryptionProperties]] \ + c_file_decryption_properties + if decryption_config is None: + c_decryption_config = CExternalDecryptionConfiguration() + else: + c_decryption_config = deref(decryption_config.unwrap().get()) + with nogil: + c_file_decryption_properties = \ + self.factory.get().SafeGetExternalFileDecryptionProperties( + deref(kms_connection_config.unwrap().get()), + c_decryption_config) + file_decryption_properties = GetResultValue( + c_file_decryption_properties) + return ExternalFileDecryptionProperties.wrap(file_decryption_properties) + def remove_cache_entries_for_token(self, access_token): self.factory.get().RemoveCacheEntriesForToken(tobytes(access_token)) @@ -477,8 +726,16 @@ cdef shared_ptr[CEncryptionConfiguration] pyarrow_unwrap_encryptionconfig(object return ( encryptionconfig).unwrap() raise TypeError("Expected EncryptionConfiguration, got %s" % type(encryptionconfig)) +cdef shared_ptr[CExternalEncryptionConfiguration] pyarrow_unwrap_external_encryptionconfig(object encryptionconfig) except *: + if isinstance(encryptionconfig, ExternalEncryptionConfiguration): + return ( encryptionconfig).unwrap_external() + raise TypeError("Expected ExternalEncryptionConfiguration, got %s" % type(encryptionconfig)) cdef shared_ptr[CDecryptionConfiguration] pyarrow_unwrap_decryptionconfig(object decryptionconfig) except *: if isinstance(decryptionconfig, DecryptionConfiguration): return ( decryptionconfig).unwrap() - raise TypeError("Expected DecryptionConfiguration, got %s" % type(decryptionconfig)) + +cdef shared_ptr[CExternalDecryptionConfiguration] pyarrow_unwrap_external_decryptionconfig(object decryptionconfig) except *: + if isinstance(decryptionconfig, ExternalDecryptionConfiguration): + return ( decryptionconfig).unwrap_external() + raise TypeError("Expected DecryptionConfiguration, got %s" % type(decryptionconfig)) \ No newline at end of file diff --git a/python/pyarrow/includes/libparquet_encryption.pxd b/python/pyarrow/includes/libparquet_encryption.pxd index 2b40414ce538..8fa54a749176 100644 --- a/python/pyarrow/includes/libparquet_encryption.pxd +++ b/python/pyarrow/includes/libparquet_encryption.pxd @@ -20,9 +20,12 @@ from pyarrow.includes.common cimport * from pyarrow._parquet cimport (ParquetCipher, CFileEncryptionProperties, + CExternalFileEncryptionProperties, CFileDecryptionProperties, + CExternalFileDecryptionProperties, ParquetCipher_AES_GCM_V1, - ParquetCipher_AES_GCM_CTR_V1) + ParquetCipher_AES_GCM_CTR_V1, + ParquetCipher_EXTERNAL_DBPA_V1) cdef extern from "parquet/encryption/kms_client.h" \ @@ -79,20 +82,52 @@ cdef extern from "parquet/encryption/crypto_factory.h" \ c_bool internal_key_material int32_t data_key_length_bits + cdef cppclass CColumnEncryptionAttributes\ + " parquet::encryption::ColumnEncryptionAttributes": + CColumnEncryptionAttributes() except + + ParquetCipher parquet_cipher + c_string key_id + + cdef cppclass CExternalEncryptionConfiguration\ + " parquet::encryption::ExternalEncryptionConfiguration": + CExternalEncryptionConfiguration(const c_string& footer_key) except + + c_string footer_key + c_string column_keys + ParquetCipher encryption_algorithm + c_bool plaintext_footer + c_bool double_wrapping + double cache_lifetime_seconds + c_bool internal_key_material + int32_t data_key_length_bits + unordered_map[c_string, CColumnEncryptionAttributes] per_column_encryption + c_string app_context + unordered_map[c_string, c_string] connection_config + cdef cppclass CDecryptionConfiguration\ " parquet::encryption::DecryptionConfiguration": CDecryptionConfiguration() except + double cache_lifetime_seconds + cdef cppclass CExternalDecryptionConfiguration\ + " parquet::encryption::ExternalDecryptionConfiguration": + CExternalDecryptionConfiguration() except + + double cache_lifetime_seconds + cdef cppclass CCryptoFactory" parquet::encryption::CryptoFactory": void RegisterKmsClientFactory( shared_ptr[CKmsClientFactory] kms_client_factory) except + shared_ptr[CFileEncryptionProperties] GetFileEncryptionProperties( const CKmsConnectionConfig& kms_connection_config, const CEncryptionConfiguration& encryption_config) except +* + shared_ptr[CExternalFileEncryptionProperties] GetExternalFileEncryptionProperties( + const CKmsConnectionConfig& kms_connection_config, + const CExternalEncryptionConfiguration& external_encryption_config) except +* shared_ptr[CFileDecryptionProperties] GetFileDecryptionProperties( const CKmsConnectionConfig& kms_connection_config, const CDecryptionConfiguration& decryption_config) except +* + shared_ptr[CExternalFileDecryptionProperties] GetExternalFileDecryptionProperties( + const CKmsConnectionConfig& kms_connection_config, + const CExternalDecryptionConfiguration& decryption_config) except +* void RemoveCacheEntriesForToken(const c_string& access_token) except + void RemoveCacheEntriesForAllTokens() except + @@ -124,7 +159,15 @@ cdef extern from "arrow/python/parquet_encryption.h" \ SafeGetFileEncryptionProperties( const CKmsConnectionConfig& kms_connection_config, const CEncryptionConfiguration& encryption_config) + CResult[shared_ptr[CExternalFileEncryptionProperties]] \ + SafeGetExternalFileEncryptionProperties( + const CKmsConnectionConfig& kms_connection_config, + const CExternalEncryptionConfiguration& external_encryption_config) CResult[shared_ptr[CFileDecryptionProperties]] \ SafeGetFileDecryptionProperties( const CKmsConnectionConfig& kms_connection_config, const CDecryptionConfiguration& decryption_config) + CResult[shared_ptr[CExternalFileDecryptionProperties]] \ + SafeExternalGetFileDecryptionProperties( + const CKmsConnectionConfig& kms_connection_config, + const CExternalDecryptionConfiguration& decryption_config) \ No newline at end of file diff --git a/python/pyarrow/parquet/encryption.py b/python/pyarrow/parquet/encryption.py index df6eed913fa5..a9633710c719 100644 --- a/python/pyarrow/parquet/encryption.py +++ b/python/pyarrow/parquet/encryption.py @@ -17,7 +17,9 @@ # specific language governing permissions and limitations # under the License. from pyarrow._parquet_encryption import (CryptoFactory, # noqa + ExternalEncryptionConfiguration, EncryptionConfiguration, DecryptionConfiguration, + ExternalDecryptionConfiguration, KmsConnectionConfig, KmsClient) diff --git a/python/pyarrow/src/arrow/python/parquet_encryption.cc b/python/pyarrow/src/arrow/python/parquet_encryption.cc index a5f924bce783..bd575689dc37 100644 --- a/python/pyarrow/src/arrow/python/parquet_encryption.cc +++ b/python/pyarrow/src/arrow/python/parquet_encryption.cc @@ -84,6 +84,14 @@ PyCryptoFactory::SafeGetFileEncryptionProperties( this->GetFileEncryptionProperties(kms_connection_config, encryption_config)); } +arrow::Result> +PyCryptoFactory::SafeGetExternalFileEncryptionProperties( + const ::parquet::encryption::KmsConnectionConfig& kms_connection_config, + const ::parquet::encryption::ExternalEncryptionConfiguration& external_encryption_config) { + PARQUET_CATCH_AND_RETURN( + this->GetExternalFileEncryptionProperties(kms_connection_config, external_encryption_config)); +} + arrow::Result> PyCryptoFactory::SafeGetFileDecryptionProperties( const ::parquet::encryption::KmsConnectionConfig& kms_connection_config, diff --git a/python/pyarrow/src/arrow/python/parquet_encryption.h b/python/pyarrow/src/arrow/python/parquet_encryption.h index 7a107c89f0bd..f9dc89d8a442 100644 --- a/python/pyarrow/src/arrow/python/parquet_encryption.h +++ b/python/pyarrow/src/arrow/python/parquet_encryption.h @@ -116,6 +116,11 @@ class ARROW_PYTHON_PARQUET_ENCRYPTION_EXPORT PyCryptoFactory const ::parquet::encryption::KmsConnectionConfig& kms_connection_config, const ::parquet::encryption::EncryptionConfiguration& encryption_config); + arrow::Result> + SafeGetExternalFileEncryptionProperties( + const ::parquet::encryption::KmsConnectionConfig& kms_connection_config, + const ::parquet::encryption::ExternalEncryptionConfiguration& external_encryption_config); + /// The returned FileDecryptionProperties object will use the cache inside this /// CryptoFactory object, so please keep this /// CryptoFactory object alive along with the returned diff --git a/python/pyarrow/tests/parquet/test_external_encryption.py b/python/pyarrow/tests/parquet/test_external_encryption.py new file mode 100644 index 000000000000..d6af033e6527 --- /dev/null +++ b/python/pyarrow/tests/parquet/test_external_encryption.py @@ -0,0 +1,219 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import pytest +from datetime import timedelta +import pyarrow.parquet as pq +import pyarrow.parquet.encryption as pe +from pyarrow._parquet_encryption import CryptoFactory, KmsConnectionConfig, ExternalEncryptionConfiguration + + +@pytest.fixture +def external_encryption_config(): + return pe.ExternalEncryptionConfiguration( + footer_key=b"0123456789abcdef", # exactly 16 bytes + column_keys={ + "col-key-id": ["a", "b"], + }, + encryption_algorithm="AES_GCM_V1", + plaintext_footer=True, + double_wrapping=True, + cache_lifetime=timedelta(minutes=5.0), + internal_key_material=True, + data_key_length_bits=256, + per_column_encryption={ + "a": { + "encryption_algorithm": "AES_GCM_V1", + "encryption_key": "key_1" + }, + "b": { + "encryption_algorithm": "AES_GCM_CTR_V1", + "encryption_key": "key_n" + } + }, + app_context={ + "user_id": "Picard1701", + "location": "Presidio" + }, + connection_config={ + "config_file": "path/to/config/file", + "config_file_decryption_key": "some_key" + } + ) + +def test_encryption_configuration_properties(): + """Test the standard EncryptionConfiguration properties to avoid regressions.""" + + config = pe.EncryptionConfiguration( + footer_key="footer-key-name", + column_keys={ + "col-key-id": ["a", "b"], + }, + encryption_algorithm="AES_GCM_V1", + plaintext_footer=True, + double_wrapping=True, + cache_lifetime=timedelta(minutes=5.0), + internal_key_material=True, + data_key_length_bits=256 + ) + + assert isinstance(config, pe.EncryptionConfiguration) + + assert config.footer_key == "footer-key-name" + assert config.column_keys == { + "col-key-id": ["a", "b"] + } + assert config.encryption_algorithm == "AES_GCM_V1" + assert config.plaintext_footer is True + assert config.double_wrapping is True + assert config.cache_lifetime == timedelta(minutes=5.0) + assert config.internal_key_material is True + assert config.data_key_length_bits == 256 + +def test_external_encryption_configuration_properties(external_encryption_config): + """Test the ExternalEncryptionConfiguration properties including external-specific fields.""" + + assert isinstance(external_encryption_config, pe.ExternalEncryptionConfiguration) + + assert external_encryption_config.footer_key == "0123456789abcdef" + assert external_encryption_config.column_keys == { + "col-key-id": ["a", "b"] + } + assert external_encryption_config.encryption_algorithm == "AES_GCM_V1" + assert external_encryption_config.plaintext_footer is True + assert external_encryption_config.double_wrapping is True + assert external_encryption_config.cache_lifetime == timedelta(minutes=5.0) + assert external_encryption_config.internal_key_material is True + assert external_encryption_config.data_key_length_bits == 256 + + assert external_encryption_config.app_context == { + "user_id": "Picard1701", + "location": "Presidio" + } + + assert external_encryption_config.connection_config == { + "config_file": "path/to/config/file", + "config_file_decryption_key": "some_key" + } + + assert external_encryption_config.per_column_encryption == { + "a": { + "encryption_algorithm": "AES_GCM_V1", + "encryption_key": "key_1" + }, + "b": { + "encryption_algorithm": "AES_GCM_CTR_V1", + "encryption_key": "key_n" + } + } + +def test_external_encryption_app_context_invalid_json(): + """Ensure app_context raises TypeError for non-JSON-serializable input.""" + with pytest.raises(TypeError, match="Failed to serialize app_context: {'invalid': {1, 2, 3}}"): + pe.ExternalEncryptionConfiguration( + footer_key="key", + app_context={"invalid": set([1, 2, 3])} # sets are not JSON-serializable + ) + +def test_external_encryption_per_column_encryption_invalid_algorithm(): + """Ensure invalid encryption_algorithm raises a ValueError or is rejected.""" + + with pytest.raises(ValueError, match="Invalid cipher name: 'INVALID'"): + pe.ExternalEncryptionConfiguration( + footer_key="key", + per_column_encryption={ + "a": { + "encryption_algorithm": "INVALID", + "encryption_key": "some_key" + } + } + ) + +def test_external_encryption_per_column_encryption_new_algorithm(): + """Ensure new encryption_algorithm is accepted.""" + + pe.ExternalEncryptionConfiguration( + footer_key="key", + per_column_encryption={ + "a": { + "encryption_algorithm": "EXTERNAL_DBPA_V1", + "encryption_key": "some_key" + } + } + ) + +def test_external_encryption_connection_config_invalid_types(): + """Ensure connection_config rejects non-string keys or values.""" + with pytest.raises(TypeError, match="Connection config key must be str, got int"): + config=pe.ExternalEncryptionConfiguration( + footer_key="key" + ) + config.connection_config={ + "config_file": "path/to/file", + 123: "should-fail" # Invalid: key is not a string + } + + with pytest.raises(TypeError, match="Connection config value must be str, got list"): + config = pe.ExternalEncryptionConfiguration( + footer_key="key" + ) + config.connection_config={ + "config_file": ["not", "a", "string"] # Invalid: value is not a string + } + +def test_external_encryption_rejects_none_values(): + config = pe.ExternalEncryptionConfiguration(footer_key="key") + + # per_column_encryption: expect ValueError + with pytest.raises(TypeError, match="per_column_encryption cannot be None"): + config.per_column_encryption = None + + # app_context: expect ValueError due to None not being JSON-serializable + with pytest.raises(ValueError, match="app_context must be JSON-serializable"): + config.app_context = None + + # connection_config: expect ValueError due to None not being iterable + with pytest.raises(ValueError, match="Connection config value cannot be None"): + config.connection_config = None + +def test_external_file_encryption_properties_valid(external_encryption_config): + class DummyKmsClient(pe.KmsClient): + def __init__(self, kms_connection_config): + super().__init__() + + def wrap_key(self, key_bytes, master_key_identifier): + # dummy wrap just returns key_bytes + return key_bytes + + def unwrap_key(self, wrapped_key, master_key_identifier): + # dummy unwrap just returns wrapped_key + return wrapped_key + + kms_config = pe.KmsConnectionConfig( + custom_kms_conf={ + "footer_key": "012footer_secret", + "orderid_key": "column_secret001", + "productid_key": "column_secret002" + } + ) + + factory = pe.CryptoFactory(DummyKmsClient) + result = factory.external_file_encryption_properties(kms_config, external_encryption_config) + + # Instead of isinstance, check class name and module dynamically because ExternalFileEncryptionProperties is not visbile + assert result.__class__.__name__ == "ExternalFileEncryptionProperties" + assert result.__class__.__module__ == "pyarrow._parquet" + diff --git a/python/scripts/base_app.py b/python/scripts/base_app.py new file mode 100644 index 000000000000..a073b1e62b0f --- /dev/null +++ b/python/scripts/base_app.py @@ -0,0 +1,138 @@ +""" +base_app.py + +@author sbrenes +""" + +import base64 +import datetime +import pyarrow +import pyarrow.parquet as pp +import pyarrow.parquet.encryption as ppe + + +class FooKmsClient(ppe.KmsClient): + + def __init__(self, kms_connection_config): + ppe.KmsClient.__init__(self) + self.master_keys_map = kms_connection_config.custom_kms_conf + + def wrap_key(self, key_bytes, master_key_identifier): + master_key_bytes = self.master_keys_map[master_key_identifier].encode('utf-8') + joint_key = b"".join([master_key_bytes, key_bytes]) + return base64.b64encode(joint_key) + + def unwrap_key(self, wrapped_key, master_key_identifier): + expected_master = self.master_keys_map[master_key_identifier] + decoded_key = base64.b64decode(wrapped_key) + master_key_bytes = decoded_key[:16] + decrypted_key = decoded_key[16:] + if (expected_master == master_key_bytes.decode('utf-8')): + return decrypted_key + raise ValueError(f"Bad master key used [{master_key_bytes}] - [{decrypted_key}]") + + + +def kms_client_factory(kms_connection_config): + return FooKmsClient(kms_connection_config) + + +def write_parquet(table, location, encryption_config=None): + encryption_properties = None + + if encryption_config: + crypto_factory = ppe.CryptoFactory(kms_client_factory) + encryption_properties = crypto_factory.file_encryption_properties( + get_kms_connection_config(), encryption_config) + + writer = pp.ParquetWriter(location, table.schema, encryption_properties=encryption_properties) + writer.write_table(table) + + +def encrypted_data_and_footer_sample(data_table): + parquet_path = "sample.parquet" + encryption_config = get_encryption_config() + write_parquet(data_table, parquet_path, + encryption_config=encryption_config) + print(f"Written to [{parquet_path}]") + + +def create_and_encrypt_parquet(): + sample_data = { + "orderId": [1001, 1002, 1003], + "productId": [152, 268, 6548], + "price": [3.25, 6.48, 2.12], + "vat": [0.0, 0.2, 0.05] + } + data_table = pyarrow.Table.from_pydict(sample_data) + + print("\nPyarrow table created. Writing parquet.") + + encrypted_data_and_footer_sample(data_table) + + +def read_and_print_parquet(): + print("\n-----------------------------------------------\nNow reading parquet file") + parquet_path = "sample.parquet" + + metadata = pp.read_metadata(parquet_path) + print("\nMetadata:") + print(metadata) + print("\n") + + decryption_config = get_decryption_config() + read_data_table = read_parquet(parquet_path, + decryption_config=decryption_config) + data_frame = read_data_table.to_pandas() + print("\nData:") + print(data_frame.head()) + print("\n") + + +def read_parquet(location, decryption_config=None, read_metadata=False): + decryption_properties = None + + if decryption_config: + crypto_factory = ppe.CryptoFactory(kms_client_factory) + decryption_properties = crypto_factory.file_decryption_properties( + get_kms_connection_config(), decryption_config) + + if read_metadata: + metadata = pp.read_metadata(location, decryption_properties=decryption_properties) + return metadata + + data_table = pp.ParquetFile(location, decryption_properties=decryption_properties).read() + return data_table + + +def get_kms_connection_config(): + return ppe.KmsConnectionConfig( + custom_kms_conf={ + "footer_key": "012footer_secret", + "orderid_key": "column_secret001", + "productid_key": "column_secret002" + } + ) + + +def get_encryption_config(plaintext_footer=True): + return ppe.EncryptionConfiguration( + footer_key = "footer_key", + column_keys = { + "orderid_key": ["orderId"], + "productid_key": ["productId"] + }, + encryption_algorithm = "AES_GCM_V1", + cache_lifetime=datetime.timedelta(minutes=2.0), + data_key_length_bits = 128, + plaintext_footer=plaintext_footer + ) + +def get_decryption_config(): + return ppe.DecryptionConfiguration(cache_lifetime=datetime.timedelta(minutes=2.0)) + + +if __name__ == "__main__": + create_and_encrypt_parquet() + read_and_print_parquet() + print("\nPlayground finished!\n") \ No newline at end of file