diff --git a/src/common/dbpa_local_test.cpp b/src/common/dbpa_local_test.cpp index 7942d04..c9209be 100644 --- a/src/common/dbpa_local_test.cpp +++ b/src/common/dbpa_local_test.cpp @@ -23,8 +23,10 @@ using namespace dbps::external; namespace { - // should rename to DBPS_ENCRYPTION_METADATA - const std::map DBPS_ENCRYPTION_METADATA = {{"dbps_agent_version", "v0.01_unittest"}}; + const std::map DBPS_ENCRYPTION_METADATA = { + {"dbps_agent_version", "v0.01_unittest"}, + {"encryption_mode", "per_block"} + }; } // Test fixture for LocalDataBatchProtectionAgent tests @@ -121,8 +123,8 @@ TEST_F(LocalDataBatchProtectionAgentTest, RoundTripEncryptDecrypt) { // Verify encryption_metadata is present in the result auto encryption_metadata = encrypt_result->encryption_metadata(); ASSERT_TRUE(encryption_metadata.has_value()); - ASSERT_EQ(1, encryption_metadata->size()); - ASSERT_TRUE(encryption_metadata->at("dbps_agent_version").length() > 0); // Non-empty string + ASSERT_TRUE(encryption_metadata->find("dbps_agent_version") != encryption_metadata->end()); + ASSERT_TRUE(encryption_metadata->find("encryption_mode") != encryption_metadata->end()); // Get the ciphertext auto ciphertext_span = encrypt_result->ciphertext(); diff --git a/src/common/dbpa_remote.cpp b/src/common/dbpa_remote.cpp index c125bcb..9f488a1 100644 --- a/src/common/dbpa_remote.cpp +++ b/src/common/dbpa_remote.cpp @@ -362,6 +362,10 @@ std::unique_ptr RemoteDataBatchProtectionAgent::Decrypt(span(std::make_unique(std::move(response))); } +void RemoteDataBatchProtectionAgent::UpdateEncryptionMetadata(std::optional> encryption_metadata) { + column_encryption_metadata_ = std::move(encryption_metadata); +} + std::optional RemoteDataBatchProtectionAgent::LoadConnectionConfigFile(const std::map& connection_config) const { const std::string error_trace = "ERROR: RemoteDataBatchProtectionAgent::LoadConnectionConfigFile() - "; auto config_file_it = connection_config.find(k_connection_config_key_); diff --git a/src/common/dbpa_remote.h b/src/common/dbpa_remote.h index 6849b79..71c159c 100644 --- a/src/common/dbpa_remote.h +++ b/src/common/dbpa_remote.h @@ -123,6 +123,10 @@ class DBPS_EXPORT RemoteDataBatchProtectionAgent : public DataBatchProtectionAge span ciphertext, std::map encoding_attributes) override; + // Updates the encryption metadata for this agent. Primary used to facilitate testing. + // This metadata is used during decryption operations to ensure compatibility with the encryption parameters. + void UpdateEncryptionMetadata(std::optional> encryption_metadata); + ~RemoteDataBatchProtectionAgent() override = default; protected: diff --git a/src/scripts/dbpa_remote_testapp.cpp b/src/scripts/dbpa_remote_testapp.cpp index 85c3be6..61220dd 100644 --- a/src/scripts/dbpa_remote_testapp.cpp +++ b/src/scripts/dbpa_remote_testapp.cpp @@ -39,7 +39,6 @@ template using span = tcb::span; namespace { - const std::map VALID_ENCRYPTION_METADATA = {{"dbps_agent_version", "v0.01_unittest"}}; const std::string SEQUENCER_ENCRYPTION_METADATA_VERSION = "v0.01"; } @@ -92,7 +91,7 @@ class DBPARemoteTestApp { Type::UNDEFINED, // datatype std::nullopt, // datatype_length (not needed for UNDEFINED) CompressionCodec::UNCOMPRESSED, // compression_type - VALID_ENCRYPTION_METADATA // column_encryption_metadata + std::nullopt // column_encryption_metadata ); std::cout << "OK: Main DBPA agent initialized successfully" << std::endl; @@ -119,7 +118,7 @@ class DBPARemoteTestApp { Type::FLOAT, // datatype std::nullopt, // datatype_length (not needed for FLOAT) CompressionCodec::UNCOMPRESSED, // compression_type - VALID_ENCRYPTION_METADATA // column_encryption_metadata + std::nullopt // column_encryption_metadata ); std::cout << "OK: Float DBPA agent initialized successfully" << std::endl; @@ -146,7 +145,7 @@ class DBPARemoteTestApp { Type::FIXED_LEN_BYTE_ARRAY, // datatype 8, // datatype_length (8 bytes per element) CompressionCodec::SNAPPY, // compression_type (input will be Snappy-compressed) - VALID_ENCRYPTION_METADATA // column_encryption_metadata + std::nullopt // column_encryption_metadata ); std::cout << "OK: Fixed-length DBPA agent initialized successfully" << std::endl; @@ -235,6 +234,7 @@ class DBPARemoteTestApp { std::cout << std::endl; // Then decrypt + agent_->UpdateEncryptionMetadata(encryption_metadata); auto decrypt_result = agent_->Decrypt(span(encrypt_result->ciphertext()), encoding_attributes); if (!decrypt_result || !decrypt_result->success()) { @@ -306,6 +306,7 @@ class DBPARemoteTestApp { } // Decrypt + agent_->UpdateEncryptionMetadata(encrypt_result->encryption_metadata()); auto decrypt_result = agent_->Decrypt(span(encrypt_result->ciphertext()), encoding_attributes); if (!decrypt_result || !decrypt_result->success()) { @@ -384,6 +385,7 @@ class DBPARemoteTestApp { std::cout << "OK: Float data encrypted successfully (" << encrypt_result->size() << " bytes)" << std::endl; // Decrypt the float data + float_agent_->UpdateEncryptionMetadata(encrypt_result->encryption_metadata()); auto decrypt_result = float_agent_->Decrypt(span(encrypt_result->ciphertext()), float_encoding_attributes); if (!decrypt_result || !decrypt_result->success()) { @@ -488,6 +490,7 @@ class DBPARemoteTestApp { std::cout << "OK: FIXED_LEN_BYTE_ARRAY encrypted successfully (" << encrypt_result->size() << " bytes)" << std::endl; // Test decryption + fixed_len_agent_->UpdateEncryptionMetadata(encrypt_result->encryption_metadata()); auto decrypt_result = fixed_len_agent_->Decrypt(span(encrypt_result->ciphertext()), fixed_len_encoding_attributes); if (!decrypt_result || !decrypt_result->success()) { diff --git a/src/server/decoding_utils.cpp b/src/server/decoding_utils.cpp index 27c4096..7e60e4e 100644 --- a/src/server/decoding_utils.cpp +++ b/src/server/decoding_utils.cpp @@ -222,6 +222,14 @@ TypedListValues ParseValueBytesIntoTypedList( } } +std::vector GetTypedListAsValueBytes( + const TypedListValues& list, + Type::type datatype, + const std::optional& datatype_length, + Format::type format) { + throw DBPSUnsupportedException("GetTypedListAsValueBytes not implemented"); +} + template const char* GetTypeName() { if constexpr (std::is_same_v>) return "INT32"; diff --git a/src/server/decoding_utils.h b/src/server/decoding_utils.h index ff5c771..a0149bd 100644 --- a/src/server/decoding_utils.h +++ b/src/server/decoding_utils.h @@ -83,6 +83,24 @@ TypedListValues ParseValueBytesIntoTypedList( const std::optional& datatype_length, Format::type format); +/** + * Convert a typed list back into value bytes based on the data type and format. + * This is the reverse operation of ParseValueBytesIntoTypedList. + * + * @param list The typed list to convert + * @param datatype The data type of the values + * @param datatype_length Optional length for fixed-length types (required for FIXED_LEN_BYTE_ARRAY) + * @param format The format of the data (currently only PLAIN is supported) + * @return std::vector containing the serialized value bytes + * @throws DBPSUnsupportedException if format or datatype is unsupported + * @throws InvalidInputException if the data is invalid or malformed + */ +std::vector GetTypedListAsValueBytes( + const TypedListValues& list, + Type::type datatype, + const std::optional& datatype_length, + Format::type format); + /** * Print a typed list in a human-readable format. * diff --git a/src/server/encryption_sequencer.cpp b/src/server/encryption_sequencer.cpp index f670d79..9718f02 100644 --- a/src/server/encryption_sequencer.cpp +++ b/src/server/encryption_sequencer.cpp @@ -19,6 +19,7 @@ #include "enum_utils.h" #include "decoding_utils.h" #include "compression_utils.h" +#include "exceptions.h" #include #include #include @@ -33,7 +34,10 @@ using namespace dbps::compression; namespace { constexpr const char* DBPS_VERSION_KEY = "dbps_agent_version"; - constexpr const char* DBPS_VERSION_VALUE = "v0.01"; + constexpr const char* DBPS_VERSION = "v0.01"; + constexpr const char* ENCRYPTION_MODE = "encryption_mode"; + constexpr const char* ENCRYPTION_PER_BLOCK = "per_block"; + constexpr const char* ENCRYPTION_PER_VALUE = "per_value"; } // Constructor implementation @@ -61,6 +65,8 @@ DataBatchEncryptionSequencer::DataBatchEncryptionSequencer( application_context_(application_context), encryption_metadata_(encryption_metadata) {} +// Top level encryption/decryption methods. + // TODO: Rename this method so it captures better the flow of decompress/format and encrypt/decrypt operations. bool DataBatchEncryptionSequencer::ConvertAndEncrypt(const std::vector& plaintext) { // Validate all parameters and key_id @@ -76,32 +82,106 @@ bool DataBatchEncryptionSequencer::ConvertAndEncrypt(const std::vector& } try { - auto level_and_value_bytes = DecompressAndSplit(plaintext); - auto typed_list = ParseValueBytesIntoTypedList( - level_and_value_bytes.value_bytes, datatype_, datatype_length_, format_); - auto encrypted_bytes = EncryptTypedList(typed_list, level_and_value_bytes.level_bytes); + // Decompress and split plaintext into level and value bytes + auto [level_bytes, value_bytes] = DecompressAndSplit(plaintext); + + // Parse value bytes into typed list + auto typed_list = ParseValueBytesIntoTypedList(value_bytes, datatype_, datatype_length_, format_); + + // Encrypt the typed list and level bytes + auto encrypted_bytes = EncryptTypedList(typed_list, level_bytes); + + // Compress the encrypted bytes encrypted_result_ = Compress(encrypted_bytes, encrypted_compression_); + } catch (const DBPSUnsupportedException& e) { - // If any stage is as of yet unsupported, default to whole payload - // (as opposed to per-value) XOR encryption + // If any stage is as of yet unsupported, default to whole payload (per-block) encryption + // (as opposed to per-value) encrypted_result_ = EncryptData(plaintext); if (encrypted_result_.empty()) { error_stage_ = "encryption"; error_message_ = "Failed to encrypt data"; return false; } - encryption_metadata_[DBPS_VERSION_KEY] = DBPS_VERSION_VALUE; + // If the sequence was interrupted by a DBPSUnsupportedException at any point, use per block encryption. + // Set the encryption type to per block. + encryption_metadata_[ENCRYPTION_MODE] = ENCRYPTION_PER_BLOCK; + encryption_metadata_[DBPS_VERSION_KEY] = DBPS_VERSION; return true; } catch (const InvalidInputException& e) { // Throw the exception so it can be caught by the caller. throw; } - encryption_metadata_[DBPS_VERSION_KEY] = DBPS_VERSION_VALUE; + // If the sequencer got here, it means the encryption of the values in the typed list finished successfully. + // Set the encryption type to per value + encryption_metadata_[ENCRYPTION_MODE] = ENCRYPTION_PER_VALUE; + encryption_metadata_[DBPS_VERSION_KEY] = DBPS_VERSION; + return true; +} + +// TODO: Rename this method so it captures better the flow of decompress/format and encrypt/decrypt operations. +bool DataBatchEncryptionSequencer::ConvertAndDecrypt(const std::vector& ciphertext) { + // Validate all parameters and key_id + if (!ValidateParameters()) { + return false; + } + + // Check that ciphertext is not null and not empty + if (ciphertext.empty()) { + error_stage_ = "validation"; + error_message_ = "ciphertext cannot be null or empty"; + return false; + } + + // Check encryption_metadata for dbps_agent_version + std::string version_error = ValidateDecryptionVersion(); + if (!version_error.empty()) { + error_stage_ = "decrypt_version_check"; + error_message_ = version_error; + return false; + } + + // Get encryption_mode from encryption_metadata + auto encryption_mode_opt = SafeGetEncryptionMode(); + if (!encryption_mode_opt.has_value()) { + error_stage_ = "decrypt_encryption_mode_validation"; + error_message_ = "Failed to get encryption_mode from encryption_metadata"; + return false; + } + std::string encryption_mode = encryption_mode_opt.value(); + + // Per-value encryption + if (encryption_mode == ENCRYPTION_PER_VALUE) { + // Decompress the encrypted bytes + auto decompressed_encrypted_bytes = Decompress(ciphertext, encrypted_compression_); + + // Decrypt the typed list and level bytes + auto [typed_list, level_bytes] = DecryptTypedList(decompressed_encrypted_bytes); + + // Convert typed list back to value bytes + auto value_bytes = GetTypedListAsValueBytes(typed_list, datatype_, datatype_length_, format_); + + // Merge level and value bytes and compress to get plaintext + decrypted_result_ = CompressAndMerge(level_bytes, value_bytes); + } + + // Per-block encryption + else if (encryption_mode == ENCRYPTION_PER_BLOCK) { + // Simple XOR decryption (same operation as encryption) for per-block encryption + decrypted_result_ = DecryptData(ciphertext); + if (decrypted_result_.empty()) { + error_stage_ = "decryption"; + error_message_ = "Failed to decrypt data"; + return false; + } + } + return true; } -// Main processing methods +// Functions to pack/unpack the plaintext into level and value bytes. +// Used during Encryption. Decompresses and splits the plaintext into level and value bytes. LevelAndValueBytes DataBatchEncryptionSequencer::DecompressAndSplit( const std::vector& plaintext) { LevelAndValueBytes result; @@ -144,14 +224,20 @@ LevelAndValueBytes DataBatchEncryptionSequencer::DecompressAndSplit( throw InvalidInputException("Unexpected page type: " + page_type); } -// This is the primary integration point for Protegrity to encrypt individual items from a typed list. +// Used during Decryption. Merges level and value bytes and compresses them back into the output format. +std::vector DataBatchEncryptionSequencer::CompressAndMerge( + const std::vector& level_bytes, const std::vector& value_bytes) { + throw DBPSUnsupportedException("CompressAndMerge not implemented"); +} + +// This is the primary integration point for Protegrity to encrypt individual values from a typed list. // // Also the context rich encryptor can use these additional parameters: column_name, user_id, key_id, application_context. // -// The current implementation prints out the list of items in the typed list and throws the DBPSUnsupportedException +// The current implementation prints out the list of values in the typed list and throws the DBPSUnsupportedException // so that the default encryption method is used. // -// Both the level_bytes AND elements need to be encrypted and combined as a single encrypted vector of bytes. +// Both the level_bytes AND list of values need to be encrypted and combined as a single encrypted vector of bytes. // std::vector DataBatchEncryptionSequencer::EncryptTypedList( const TypedListValues& typed_list, const std::vector& level_bytes) { @@ -176,50 +262,13 @@ std::vector DataBatchEncryptionSequencer::EncryptTypedList( throw DBPSUnsupportedException("EncryptTypedList not implemented"); } -// TODO: Rename this method so it captures better the flow of decompress/format and encrypt/decrypt operations. -bool DataBatchEncryptionSequencer::ConvertAndDecrypt(const std::vector& ciphertext) { - // Validate all parameters and key_id - if (!ValidateParameters()) { - return false; - } - - // Check that ciphertext is not null and not empty - if (ciphertext.empty()) { - error_stage_ = "validation"; - error_message_ = "ciphertext cannot be null or empty"; - return false; - } - - // Check encryption_metadata for dbps_agent_version - // - // The DBPS server version check during Decrypt is to future-proof against changes on the Encryption process. - // The Encryption process could change due to updates on the payload decoding, updates on fallback encryption methods, or other changes, - // and it is possible that it results on a mismatch with the Decryption implementation. This check helps to catch such mismatches. - auto it = encryption_metadata_.find(DBPS_VERSION_KEY); - if (it == encryption_metadata_.end()) { - std::cerr << "ERROR: EncryptionSequencer - encryption_metadata must contain key '" << DBPS_VERSION_KEY << "'" << std::endl; - error_stage_ = "decrypt_version_check"; - error_message_ = "encryption_metadata must contain key '" + std::string(DBPS_VERSION_KEY) + "'"; - return false; - } else if (it->second.find(DBPS_VERSION_VALUE) != 0) { - std::cerr << "ERROR: EncryptionSequencer - encryption_metadata['" << DBPS_VERSION_KEY << "'] must match '" - << DBPS_VERSION_VALUE << "', but got '" << it->second << "'" << std::endl; - error_stage_ = "decrypt_version_check"; - error_message_ = "encryption_metadata['" + std::string(DBPS_VERSION_KEY) + "'] must match '" + std::string(DBPS_VERSION_VALUE) + "'"; - return false; - } - - // Simple XOR decryption (same operation as encryption) - decrypted_result_ = DecryptData(ciphertext); - if (decrypted_result_.empty()) { - error_stage_ = "decryption"; - error_message_ = "Failed to decrypt data"; - return false; - } - - return true; +std::pair> DataBatchEncryptionSequencer::DecryptTypedList( + const std::vector& encrypted_bytes) { + throw DBPSUnsupportedException("DecryptTypedList not implemented"); } +// Helper methods to validate and basic parameter reading. + bool DataBatchEncryptionSequencer::ConvertEncodingAttributesToValues() { // Helper to find key and return value or null auto FindKey = [this](const std::string& key) -> const std::string* { @@ -332,6 +381,35 @@ bool DataBatchEncryptionSequencer::ValidateParameters() { return true; } +std::string DataBatchEncryptionSequencer::ValidateDecryptionVersion() { + auto it = encryption_metadata_.find(DBPS_VERSION_KEY); + if (it == encryption_metadata_.end()) { + std::cerr << "ERROR: EncryptionSequencer - encryption_metadata must contain key '" << DBPS_VERSION_KEY << "'" << std::endl; + return "encryption_metadata must contain key '" + std::string(DBPS_VERSION_KEY) + "'"; + } else if (it->second.find(DBPS_VERSION) != 0) { + std::cerr << "ERROR: EncryptionSequencer - encryption_metadata['" << DBPS_VERSION_KEY << "'] must match '" + << DBPS_VERSION << "', but got '" << it->second << "'" << std::endl; + return "encryption_metadata['" + std::string(DBPS_VERSION_KEY) + "'] must match '" + std::string(DBPS_VERSION) + "'"; + } + return ""; +} + +std::optional DataBatchEncryptionSequencer::SafeGetEncryptionMode() { + auto it = encryption_metadata_.find(ENCRYPTION_MODE); + if (it == encryption_metadata_.end()) { + // The metadata key for encryption mode is missing. + return std::nullopt; + } + const std::string& encryption_mode = it->second; + if (encryption_mode != ENCRYPTION_PER_BLOCK && encryption_mode != ENCRYPTION_PER_VALUE) { + // The value for encryption mode is not valid. + return std::nullopt; + } + return encryption_mode; +} + +// Simple encryption/decryption functions using XOR with key_id hash + std::vector DataBatchEncryptionSequencer::EncryptData(const std::vector& data) { if (data.empty()) { return std::vector(); diff --git a/src/server/encryption_sequencer.h b/src/server/encryption_sequencer.h index e3f2f2c..429b10c 100644 --- a/src/server/encryption_sequencer.h +++ b/src/server/encryption_sequencer.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -99,6 +100,39 @@ class DataBatchEncryptionSequencer { std::string user_id_; std::string application_context_; + /** + * Decompresses and splits the plaintext into level and value bytes. + * Returns the level and value bytes. + */ + LevelAndValueBytes DecompressAndSplit(const std::vector& plaintext); + + /** + * Merges level and value bytes and compresses them into plaintext. + * This is the reverse operation of DecompressAndSplit. + * Handles different page types (DATA_PAGE_V1, DATA_PAGE_V2, DICTIONARY_PAGE) appropriately. + * Returns the merged and compressed plaintext. + */ + std::vector CompressAndMerge(const std::vector& level_bytes, const std::vector& value_bytes); + + + /** + * Integration point: Encryption function based on list of values that will be the implemented by Protegrity. + * + * The level_bytes and the elements need to be encrypted and combined into a single encrypted + * vector of bytes. + */ + std::vector EncryptTypedList( + const TypedListValues& typed_list, const std::vector& level_bytes); + + /** + * Integration point: Decryption function based on encrypted bytes that will be the implemented by Protegrity. + * + * Takes encrypted bytes and decrypts them back into a typed list and level bytes. + * Returns a pair containing the decrypted TypedListValues and level_bytes. + */ + std::pair> DecryptTypedList( + const std::vector& encrypted_bytes); + // Converted encoding attributes values to corresponding types std::map> encoding_attributes_converted_; @@ -119,18 +153,20 @@ class DataBatchEncryptionSequencer { bool ValidateParameters(); /** - * Decompresses and splits the plaintext into level and value bytes. - * Returns the level and value bytes. + * Validates the DBPS version in encryption_metadata during decryption. + * The DBPS server version check during Decrypt is to future-proof against changes on the Encryption process. + * The Encryption process could change due to updates on the payload decoding, updates on fallback encryption methods, or other changes, + * and it is possible that it results on a mismatch with the Decryption implementation. This check helps to catch such mismatches. + * Returns empty string if validation passes, otherwise returns the error message. */ - LevelAndValueBytes DecompressAndSplit(const std::vector& plaintext); - + std::string ValidateDecryptionVersion(); + /** - * Data element encryption function that will be the integration point for Protegrity. - * The level_bytes and the elements need to be encrypted and combined into a single encrypted - * vector of bytes. + * Safely gets the encryption_mode value from encryption_metadata. + * Returns the encryption mode value ("per_block" or "per_value") if found and valid, + * otherwise returns empty string. */ - std::vector EncryptTypedList( - const TypedListValues& typed_list, const std::vector& level_bytes); + std::optional SafeGetEncryptionMode(); // Simple encryption/decryption using XOR with key_id hash std::vector EncryptData(const std::vector& data);