From 1ddbed9376fcfba68ebf561400e393fd93b4975c Mon Sep 17 00:00:00 2001 From: Alejandro Valerio Date: Tue, 10 Mar 2026 16:21:14 -0600 Subject: [PATCH 01/18] TEMPORARY BRANCH FOR ZERO-COPY VERSION PERFORMANCE DEBUGGING --- src/processing/encryption_sequencer.cpp | 51 +++++ .../encryptors/basic_xor_encryptor.cpp | 197 ++++++++++++++++-- 2 files changed, 231 insertions(+), 17 deletions(-) diff --git a/src/processing/encryption_sequencer.cpp b/src/processing/encryption_sequencer.cpp index c1d728e..889cc3c 100644 --- a/src/processing/encryption_sequencer.cpp +++ b/src/processing/encryption_sequencer.cpp @@ -22,6 +22,7 @@ #include "compression_utils.h" #include "../common/exceptions.h" #include "encryptors/basic_xor_encryptor.h" +#include #include #include #include @@ -42,6 +43,23 @@ namespace { constexpr const char* ENCRYPTION_MODE_KEY_DATA_PAGE = "encrypt_mode_data_page"; constexpr const char* ENCRYPTION_MODE_PER_BLOCK = "per_block"; constexpr const char* ENCRYPTION_MODE_PER_VALUE = "per_value"; + + void PrintDecodeAndEncryptTimings( + int64_t decompress_and_split_us, + int64_t parse_value_bytes_into_typed_list_us, + int64_t encrypt_value_list_us, + int64_t encrypt_block_level_bytes_us, + int64_t join_with_length_prefix_us, + int64_t compress_us) { + + std::cout << "+++++ DecodeAndEncrypt timings (microseconds) +++++" << std::endl; + std::cout << " DecompressAndSplit: " << decompress_and_split_us << std::endl; + std::cout << " ParseValueBytesIntoTypedList: " << parse_value_bytes_into_typed_list_us << std::endl; + std::cout << " EncryptValueList: " << encrypt_value_list_us << std::endl; + std::cout << " EncryptBlock(level_bytes): " << encrypt_block_level_bytes_us << std::endl; + std::cout << " JoinWithLengthPrefix: " << join_with_length_prefix_us << std::endl; + std::cout << " Compress: " << compress_us << std::endl; + } } // Helper function to create encryptor instance @@ -137,20 +155,53 @@ bool DataBatchEncryptionSequencer::DecodeAndEncrypt(tcb::span pla * - Once per-value encryption for all cases is complete, the try-catch block and the call to EncryptBlock must be removed. */ try { + int64_t decompress_and_split_us = 0; + int64_t parse_value_bytes_into_typed_list_us = 0; + int64_t encrypt_value_list_us = 0; + int64_t encrypt_block_level_bytes_us = 0; + int64_t join_with_length_prefix_us = 0; + int64_t compress_us = 0; + // Decompress and split plaintext into level and value bytes + auto stage_start = std::chrono::steady_clock::now(); auto [level_bytes, value_bytes] = DecompressAndSplit( plaintext, compression_, encoding_attributes_converted_); + decompress_and_split_us = std::chrono::duration_cast( + std::chrono::steady_clock::now() - stage_start).count(); // Parse value bytes into typed values buffer + stage_start = std::chrono::steady_clock::now(); auto typed_buffer = ReinterpretValueBytesAsTypedValuesBuffer(value_bytes, datatype_, datatype_length_, encoding_); + parse_value_bytes_into_typed_list_us = std::chrono::duration_cast( + std::chrono::steady_clock::now() - stage_start).count(); // Encrypt the typed values buffer and level bytes, then join them into a single encrypted byte vector. + stage_start = std::chrono::steady_clock::now(); auto encrypted_value_bytes = encryptor_->EncryptValueList(typed_buffer); + encrypt_value_list_us = std::chrono::duration_cast( + std::chrono::steady_clock::now() - stage_start).count(); + stage_start = std::chrono::steady_clock::now(); auto encrypted_level_bytes = encryptor_->EncryptBlock(level_bytes); + encrypt_block_level_bytes_us = std::chrono::duration_cast( + std::chrono::steady_clock::now() - stage_start).count(); + stage_start = std::chrono::steady_clock::now(); auto joined_encrypted_bytes = JoinWithLengthPrefix(encrypted_level_bytes, encrypted_value_bytes); + join_with_length_prefix_us = std::chrono::duration_cast( + std::chrono::steady_clock::now() - stage_start).count(); // Compress the joined encrypted bytes + stage_start = std::chrono::steady_clock::now(); encrypted_result_ = Compress(joined_encrypted_bytes, encrypted_compression_); + compress_us = std::chrono::duration_cast( + std::chrono::steady_clock::now() - stage_start).count(); + + PrintDecodeAndEncryptTimings( + decompress_and_split_us, + parse_value_bytes_into_typed_list_us, + encrypt_value_list_us, + encrypt_block_level_bytes_us, + join_with_length_prefix_us, + compress_us); // Set the encryption type to per-value encryption_metadata_[encryption_mode_key] = ENCRYPTION_MODE_PER_VALUE; diff --git a/src/processing/encryptors/basic_xor_encryptor.cpp b/src/processing/encryptors/basic_xor_encryptor.cpp index 44b8d18..10f0867 100644 --- a/src/processing/encryptors/basic_xor_encryptor.cpp +++ b/src/processing/encryptors/basic_xor_encryptor.cpp @@ -19,6 +19,7 @@ #include "encryptor_utils.h" #include "../../common/exceptions.h" #include "../../common/enum_utils.h" +#include #include #include #include @@ -27,6 +28,66 @@ using namespace dbps::processing; using namespace dbps::external; +namespace { + int64_t ElapsedMicrosecondsSince(const std::chrono::steady_clock::time_point& start) { + return std::chrono::duration_cast( + std::chrono::steady_clock::now() - start).count(); + } + + void PrintBasicXorBlockTimings(const char* operation, int64_t total_us) { + std::cout << "+++++ BasicXorEncryptor timings (microseconds) +++++" << std::endl; + std::cout << " " << operation << ": " << total_us << std::endl; + } + + void PrintBasicXorEncryptTypedElementsTimings( + bool is_fixed, + size_t num_elements, + size_t element_size, + int64_t encrypt_elements_loop_us, + int64_t get_raw_element_us, + int64_t xor_encrypt_us, + int64_t set_element_us, + int64_t finalize_buffer_us, + int64_t write_header_us, + int64_t total_us) { + std::cout << "+++++ BasicXorEncryptor EncryptTypedElements timings (microseconds) +++++" << std::endl; + std::cout << " mode: " << (is_fixed ? "fixed" : "variable") << std::endl; + std::cout << " num_elements: " << num_elements << std::endl; + std::cout << " element_size: " << element_size << std::endl; + std::cout << " EncryptElementsLoop: " << encrypt_elements_loop_us << std::endl; + std::cout << " GetRawElement(aggregated): " << get_raw_element_us << std::endl; + std::cout << " XorEncrypt(aggregated): " << xor_encrypt_us << std::endl; + std::cout << " SetElement(aggregated): " << set_element_us << std::endl; + std::cout << " LoopResidual(iterator+other): " + << (encrypt_elements_loop_us - get_raw_element_us - xor_encrypt_us - set_element_us) << std::endl; + std::cout << " FinalizeBuffer: " << finalize_buffer_us << std::endl; + std::cout << " WriteHeader: " << write_header_us << std::endl; + std::cout << " EncryptTypedElements(total): " << total_us << std::endl; + } + + void PrintBasicXorEncryptValueListTimings(int64_t visit_dispatch_us, int64_t total_us) { + std::cout << "+++++ BasicXorEncryptor EncryptValueList timings (microseconds) +++++" << std::endl; + std::cout << " VariantVisitAndEncrypt: " << visit_dispatch_us << std::endl; + std::cout << " EncryptValueList(total): " << total_us << std::endl; + } + + void PrintBasicXorDecryptValueListTimings( + bool is_fixed, + size_t num_elements, + int64_t read_header_us, + int64_t setup_buffer_us, + int64_t decrypt_elements_us, + int64_t total_us) { + std::cout << "+++++ BasicXorEncryptor DecryptValueList timings (microseconds) +++++" << std::endl; + std::cout << " mode: " << (is_fixed ? "fixed" : "variable") << std::endl; + std::cout << " num_elements: " << num_elements << std::endl; + std::cout << " ReadHeader: " << read_header_us << std::endl; + std::cout << " SetupEncryptedBuffer: " << setup_buffer_us << std::endl; + std::cout << " DecryptElements: " << decrypt_elements_us << std::endl; + std::cout << " DecryptValueList(total): " << total_us << std::endl; + } +} + // --------------------------------------------------------------------------- // Functions for encrypting and decrypting byte arrays. // --------------------------------------------------------------------------- @@ -57,11 +118,17 @@ std::vector BasicXorEncryptor::XorDecrypt(tcb::span data // --------------------------------------------------------------------------- std::vector BasicXorEncryptor::EncryptBlock(tcb::span data) { - return XorEncrypt(data, key_id_); + auto start = std::chrono::steady_clock::now(); + auto out = XorEncrypt(data, key_id_); + PrintBasicXorBlockTimings("EncryptBlock", ElapsedMicrosecondsSince(start)); + return out; } std::vector BasicXorEncryptor::DecryptBlock(tcb::span data) { - return XorDecrypt(data, key_id_); + auto start = std::chrono::steady_clock::now(); + auto out = XorDecrypt(data, key_id_); + PrintBasicXorBlockTimings("DecryptBlock", ElapsedMicrosecondsSince(start)); + return out; } // --------------------------------------------------------------------------- @@ -96,6 +163,7 @@ std::vector BasicXorEncryptor::DecryptBlock(tcb::span da template std::vector BasicXorEncryptor::EncryptTypedElements( const TypedBuffer& input_buffer, const std::string& key_id) { + auto total_start = std::chrono::steady_clock::now(); constexpr bool is_fixed = TypedBuffer::is_fixed_sized; constexpr size_t prefix_length = is_fixed ? kFixedHeaderLength : kVariableHeaderLength; const size_t num_elements = input_buffer.GetNumElements(); @@ -114,19 +182,33 @@ std::vector BasicXorEncryptor::EncryptTypedElements( std::vector final_buffer; size_t element_size = 0; + int64_t encrypt_elements_loop_us = 0; + int64_t get_raw_element_us = 0; + int64_t xor_encrypt_us = 0; + int64_t set_element_us = 0; + int64_t finalize_buffer_us = 0; // Encrypt fixed-size elements if constexpr (is_fixed) { element_size = input_buffer.GetElementSize(); TypedBufferRawBytesFixedSized output_buffer{ num_elements, prefix_length, RawBytesFixedSizedCodec{element_size}}; - size_t output_index = 0; - for (const auto raw_bytes : input_buffer.raw_elements()) { + auto stage_start = std::chrono::steady_clock::now(); + for (size_t i = 0; i < num_elements; ++i) { + auto op_start = std::chrono::steady_clock::now(); + auto raw_bytes = input_buffer.GetRawElement(i); + get_raw_element_us += ElapsedMicrosecondsSince(op_start); + op_start = std::chrono::steady_clock::now(); auto encrypted = XorEncrypt(raw_bytes, key_id); - output_buffer.SetElement(output_index, tcb::span(encrypted)); - output_index++; + xor_encrypt_us += ElapsedMicrosecondsSince(op_start); + op_start = std::chrono::steady_clock::now(); + output_buffer.SetElement(i, tcb::span(encrypted)); + set_element_us += ElapsedMicrosecondsSince(op_start); } + encrypt_elements_loop_us = ElapsedMicrosecondsSince(stage_start); + stage_start = std::chrono::steady_clock::now(); final_buffer = output_buffer.FinalizeAndTakeBuffer(); + finalize_buffer_us = ElapsedMicrosecondsSince(stage_start); } // Encrypt variable-size elements @@ -134,24 +216,52 @@ std::vector BasicXorEncryptor::EncryptTypedElements( auto reserved_bytes_hint = input_buffer.GetRawBufferSize(); TypedBufferRawBytesVariableSized output_buffer{ num_elements, reserved_bytes_hint, true, prefix_length}; - size_t output_index = 0; - for (const auto raw_bytes : input_buffer.raw_elements()) { + + auto stage_start = std::chrono::steady_clock::now(); + for (size_t i = 0; i < num_elements; ++i) { + + auto op_start = std::chrono::steady_clock::now(); + auto raw_bytes = input_buffer.GetRawElement(i); + get_raw_element_us += ElapsedMicrosecondsSince(op_start); + + op_start = std::chrono::steady_clock::now(); auto encrypted = XorEncrypt(raw_bytes, key_id); - output_buffer.SetElement(output_index, tcb::span(encrypted)); - output_index++; + xor_encrypt_us += ElapsedMicrosecondsSince(op_start); + + op_start = std::chrono::steady_clock::now(); + output_buffer.SetElement(i, tcb::span(encrypted)); + set_element_us += ElapsedMicrosecondsSince(op_start); } + encrypt_elements_loop_us = ElapsedMicrosecondsSince(stage_start); + stage_start = std::chrono::steady_clock::now(); final_buffer = output_buffer.FinalizeAndTakeBuffer(); + finalize_buffer_us = ElapsedMicrosecondsSince(stage_start); } // Write the header to the final buffer and return it. + auto header_start = std::chrono::steady_clock::now(); WriteHeader(final_buffer, {is_fixed, static_cast(num_elements), static_cast(element_size)}); + auto write_header_us = ElapsedMicrosecondsSince(header_start); + + PrintBasicXorEncryptTypedElementsTimings( + is_fixed, + num_elements, + element_size, + encrypt_elements_loop_us, + get_raw_element_us, + xor_encrypt_us, + set_element_us, + finalize_buffer_us, + write_header_us, + ElapsedMicrosecondsSince(total_start)); return final_buffer; } std::vector BasicXorEncryptor::EncryptValueList( const TypedValuesBuffer& typed_buffer) { + auto total_start = std::chrono::steady_clock::now(); // Printable context string for logging // std::string context_str = std::string("Context parameters:") @@ -166,9 +276,13 @@ std::vector BasicXorEncryptor::EncryptValueList( // std::visit extracts the concrete buffer type from the TypedValuesBuffer variant // and forwards it to EncryptTypedElements, which handles all buffer types generically. - return std::visit([&](const auto& input_buffer) { + auto visit_start = std::chrono::steady_clock::now(); + auto encrypted = std::visit([&](const auto& input_buffer) { return EncryptTypedElements(input_buffer, key_id_); }, typed_buffer); + auto visit_dispatch_us = ElapsedMicrosecondsSince(visit_start); + PrintBasicXorEncryptValueListTimings(visit_dispatch_us, ElapsedMicrosecondsSince(total_start)); + return encrypted; } // --------------------------------------------------------------------------- @@ -193,38 +307,80 @@ TypedBuffer BasicXorEncryptor::DecryptFixedSizedElementsIntoTypedBuffer( TypedValuesBuffer BasicXorEncryptor::DecryptValueList( tcb::span encrypted_bytes) { + auto total_start = std::chrono::steady_clock::now(); + auto stage_start = std::chrono::steady_clock::now(); auto header = ReadHeader(encrypted_bytes); + auto read_header_us = ElapsedMicrosecondsSince(stage_start); auto num_elements = static_cast(header.num_elements); // Decrypt fixed-size elements if (header.is_fixed) { + stage_start = std::chrono::steady_clock::now(); // Create a fixed-sized byte buffer for reading the encrypted elements. TypedBufferRawBytesFixedSized encrypted_buffer{ encrypted_bytes, kFixedHeaderLength, RawBytesFixedSizedCodec{header.element_size}}; + auto setup_buffer_us = ElapsedMicrosecondsSince(stage_start); // Populate a typed buffer with the decrypted elements in the corresponding type. + stage_start = std::chrono::steady_clock::now(); switch (datatype_) { case Type::INT32: - return DecryptFixedSizedElementsIntoTypedBuffer( + { + auto out = DecryptFixedSizedElementsIntoTypedBuffer( encrypted_buffer, key_id_, TypedBufferI32{num_elements}); + PrintBasicXorDecryptValueListTimings( + true, num_elements, read_header_us, setup_buffer_us, + ElapsedMicrosecondsSince(stage_start), ElapsedMicrosecondsSince(total_start)); + return out; + } case Type::INT64: - return DecryptFixedSizedElementsIntoTypedBuffer( + { + auto out = DecryptFixedSizedElementsIntoTypedBuffer( encrypted_buffer, key_id_, TypedBufferI64{num_elements}); + PrintBasicXorDecryptValueListTimings( + true, num_elements, read_header_us, setup_buffer_us, + ElapsedMicrosecondsSince(stage_start), ElapsedMicrosecondsSince(total_start)); + return out; + } case Type::INT96: - return DecryptFixedSizedElementsIntoTypedBuffer( + { + auto out = DecryptFixedSizedElementsIntoTypedBuffer( encrypted_buffer, key_id_, TypedBufferInt96{num_elements}); + PrintBasicXorDecryptValueListTimings( + true, num_elements, read_header_us, setup_buffer_us, + ElapsedMicrosecondsSince(stage_start), ElapsedMicrosecondsSince(total_start)); + return out; + } case Type::FLOAT: - return DecryptFixedSizedElementsIntoTypedBuffer( + { + auto out = DecryptFixedSizedElementsIntoTypedBuffer( encrypted_buffer, key_id_, TypedBufferFloat{num_elements}); + PrintBasicXorDecryptValueListTimings( + true, num_elements, read_header_us, setup_buffer_us, + ElapsedMicrosecondsSince(stage_start), ElapsedMicrosecondsSince(total_start)); + return out; + } case Type::DOUBLE: - return DecryptFixedSizedElementsIntoTypedBuffer( + { + auto out = DecryptFixedSizedElementsIntoTypedBuffer( encrypted_buffer, key_id_, TypedBufferDouble{num_elements}); + PrintBasicXorDecryptValueListTimings( + true, num_elements, read_header_us, setup_buffer_us, + ElapsedMicrosecondsSince(stage_start), ElapsedMicrosecondsSince(total_start)); + return out; + } case Type::FIXED_LEN_BYTE_ARRAY: - return DecryptFixedSizedElementsIntoTypedBuffer( + { + auto out = DecryptFixedSizedElementsIntoTypedBuffer( encrypted_buffer, key_id_, TypedBufferRawBytesFixedSized{num_elements, 0, RawBytesFixedSizedCodec{header.element_size}}); + PrintBasicXorDecryptValueListTimings( + true, num_elements, read_header_us, setup_buffer_us, + ElapsedMicrosecondsSince(stage_start), ElapsedMicrosecondsSince(total_start)); + return out; + } default: throw InvalidInputException( std::string("DecryptValueList: unsupported fixed-size datatype: ") @@ -234,8 +390,10 @@ TypedValuesBuffer BasicXorEncryptor::DecryptValueList( // Decrypt variable-size elements else { + stage_start = std::chrono::steady_clock::now(); // Create a variable-sized byte buffer for reading the encrypted elements. TypedBufferRawBytesVariableSized encrypted_buffer{ encrypted_bytes, kVariableHeaderLength}; + auto setup_buffer_us = ElapsedMicrosecondsSince(stage_start); switch (datatype_) { // Create a BYTE-ARRAY typed buffer for storing the decrypted elements. @@ -243,11 +401,16 @@ TypedValuesBuffer BasicXorEncryptor::DecryptValueList( auto reserved_bytes_hint = encrypted_buffer.GetRawBufferSize(); TypedBufferRawBytesVariableSized output_buffer{num_elements, reserved_bytes_hint, true}; size_t output_index = 0; + stage_start = std::chrono::steady_clock::now(); for (const auto element : encrypted_buffer) { auto decrypted_bytes = XorDecrypt(element, key_id_); output_buffer.SetElement(output_index, tcb::span(decrypted_bytes)); output_index++; } + auto decrypt_elements_us = ElapsedMicrosecondsSince(stage_start); + PrintBasicXorDecryptValueListTimings( + false, num_elements, read_header_us, setup_buffer_us, + decrypt_elements_us, ElapsedMicrosecondsSince(total_start)); return output_buffer; } default: From a997cce63c676ffaabac5043d58b8db23ccf0a14 Mon Sep 17 00:00:00 2001 From: Alejandro Valerio Date: Tue, 10 Mar 2026 16:33:32 -0600 Subject: [PATCH 02/18] - Timer switched to nanoseconds --- src/processing/encryption_sequencer.cpp | 74 ++++---- .../encryptors/basic_xor_encryptor.cpp | 171 ++++++++++-------- 2 files changed, 133 insertions(+), 112 deletions(-) diff --git a/src/processing/encryption_sequencer.cpp b/src/processing/encryption_sequencer.cpp index 889cc3c..506bab0 100644 --- a/src/processing/encryption_sequencer.cpp +++ b/src/processing/encryption_sequencer.cpp @@ -44,21 +44,31 @@ namespace { constexpr const char* ENCRYPTION_MODE_PER_BLOCK = "per_block"; constexpr const char* ENCRYPTION_MODE_PER_VALUE = "per_value"; + int64_t ToMicroseconds(int64_t nanoseconds) { + return nanoseconds / 1000; + } + + void PrintDurationLine(const char* label, int64_t nanoseconds) { + std::cout << " " << label << ": " + << ToMicroseconds(nanoseconds) << " us" + << " (" << nanoseconds << " ns)" << std::endl; + } + void PrintDecodeAndEncryptTimings( - int64_t decompress_and_split_us, - int64_t parse_value_bytes_into_typed_list_us, - int64_t encrypt_value_list_us, - int64_t encrypt_block_level_bytes_us, - int64_t join_with_length_prefix_us, - int64_t compress_us) { - - std::cout << "+++++ DecodeAndEncrypt timings (microseconds) +++++" << std::endl; - std::cout << " DecompressAndSplit: " << decompress_and_split_us << std::endl; - std::cout << " ParseValueBytesIntoTypedList: " << parse_value_bytes_into_typed_list_us << std::endl; - std::cout << " EncryptValueList: " << encrypt_value_list_us << std::endl; - std::cout << " EncryptBlock(level_bytes): " << encrypt_block_level_bytes_us << std::endl; - std::cout << " JoinWithLengthPrefix: " << join_with_length_prefix_us << std::endl; - std::cout << " Compress: " << compress_us << std::endl; + int64_t decompress_and_split_ns, + int64_t parse_value_bytes_into_typed_list_ns, + int64_t encrypt_value_list_ns, + int64_t encrypt_block_level_bytes_ns, + int64_t join_with_length_prefix_ns, + int64_t compress_ns) { + + std::cout << "+++++ DecodeAndEncrypt timings (microseconds + nanoseconds) +++++" << std::endl; + PrintDurationLine("DecompressAndSplit", decompress_and_split_ns); + PrintDurationLine("ParseValueBytesIntoTypedList", parse_value_bytes_into_typed_list_ns); + PrintDurationLine("EncryptValueList", encrypt_value_list_ns); + PrintDurationLine("EncryptBlock(level_bytes)", encrypt_block_level_bytes_ns); + PrintDurationLine("JoinWithLengthPrefix", join_with_length_prefix_ns); + PrintDurationLine("Compress", compress_ns); } } @@ -155,53 +165,53 @@ bool DataBatchEncryptionSequencer::DecodeAndEncrypt(tcb::span pla * - Once per-value encryption for all cases is complete, the try-catch block and the call to EncryptBlock must be removed. */ try { - int64_t decompress_and_split_us = 0; - int64_t parse_value_bytes_into_typed_list_us = 0; - int64_t encrypt_value_list_us = 0; - int64_t encrypt_block_level_bytes_us = 0; - int64_t join_with_length_prefix_us = 0; - int64_t compress_us = 0; + int64_t decompress_and_split_ns = 0; + int64_t parse_value_bytes_into_typed_list_ns = 0; + int64_t encrypt_value_list_ns = 0; + int64_t encrypt_block_level_bytes_ns = 0; + int64_t join_with_length_prefix_ns = 0; + int64_t compress_ns = 0; // Decompress and split plaintext into level and value bytes auto stage_start = std::chrono::steady_clock::now(); auto [level_bytes, value_bytes] = DecompressAndSplit( plaintext, compression_, encoding_attributes_converted_); - decompress_and_split_us = std::chrono::duration_cast( + decompress_and_split_ns = std::chrono::duration_cast( std::chrono::steady_clock::now() - stage_start).count(); // Parse value bytes into typed values buffer stage_start = std::chrono::steady_clock::now(); auto typed_buffer = ReinterpretValueBytesAsTypedValuesBuffer(value_bytes, datatype_, datatype_length_, encoding_); - parse_value_bytes_into_typed_list_us = std::chrono::duration_cast( + parse_value_bytes_into_typed_list_ns = std::chrono::duration_cast( std::chrono::steady_clock::now() - stage_start).count(); // Encrypt the typed values buffer and level bytes, then join them into a single encrypted byte vector. stage_start = std::chrono::steady_clock::now(); auto encrypted_value_bytes = encryptor_->EncryptValueList(typed_buffer); - encrypt_value_list_us = std::chrono::duration_cast( + encrypt_value_list_ns = std::chrono::duration_cast( std::chrono::steady_clock::now() - stage_start).count(); stage_start = std::chrono::steady_clock::now(); auto encrypted_level_bytes = encryptor_->EncryptBlock(level_bytes); - encrypt_block_level_bytes_us = std::chrono::duration_cast( + encrypt_block_level_bytes_ns = std::chrono::duration_cast( std::chrono::steady_clock::now() - stage_start).count(); stage_start = std::chrono::steady_clock::now(); auto joined_encrypted_bytes = JoinWithLengthPrefix(encrypted_level_bytes, encrypted_value_bytes); - join_with_length_prefix_us = std::chrono::duration_cast( + join_with_length_prefix_ns = std::chrono::duration_cast( std::chrono::steady_clock::now() - stage_start).count(); // Compress the joined encrypted bytes stage_start = std::chrono::steady_clock::now(); encrypted_result_ = Compress(joined_encrypted_bytes, encrypted_compression_); - compress_us = std::chrono::duration_cast( + compress_ns = std::chrono::duration_cast( std::chrono::steady_clock::now() - stage_start).count(); PrintDecodeAndEncryptTimings( - decompress_and_split_us, - parse_value_bytes_into_typed_list_us, - encrypt_value_list_us, - encrypt_block_level_bytes_us, - join_with_length_prefix_us, - compress_us); + decompress_and_split_ns, + parse_value_bytes_into_typed_list_ns, + encrypt_value_list_ns, + encrypt_block_level_bytes_ns, + join_with_length_prefix_ns, + compress_ns); // Set the encryption type to per-value encryption_metadata_[encryption_mode_key] = ENCRYPTION_MODE_PER_VALUE; diff --git a/src/processing/encryptors/basic_xor_encryptor.cpp b/src/processing/encryptors/basic_xor_encryptor.cpp index 10f0867..d1e807a 100644 --- a/src/processing/encryptors/basic_xor_encryptor.cpp +++ b/src/processing/encryptors/basic_xor_encryptor.cpp @@ -29,62 +29,73 @@ using namespace dbps::processing; using namespace dbps::external; namespace { - int64_t ElapsedMicrosecondsSince(const std::chrono::steady_clock::time_point& start) { - return std::chrono::duration_cast( + int64_t ElapsedNanosecondsSince(const std::chrono::steady_clock::time_point& start) { + return std::chrono::duration_cast( std::chrono::steady_clock::now() - start).count(); } - void PrintBasicXorBlockTimings(const char* operation, int64_t total_us) { - std::cout << "+++++ BasicXorEncryptor timings (microseconds) +++++" << std::endl; - std::cout << " " << operation << ": " << total_us << std::endl; + int64_t ToMicroseconds(int64_t nanoseconds) { + return nanoseconds / 1000; + } + + void PrintDurationLine(const char* label, int64_t nanoseconds) { + std::cout << " " << label << ": " + << ToMicroseconds(nanoseconds) << " us" + << " (" << nanoseconds << " ns)" << std::endl; + } + + void PrintBasicXorBlockTimings(const char* operation, int64_t total_ns) { + std::cout << "+++++ BasicXorEncryptor timings (microseconds + nanoseconds) +++++" << std::endl; + PrintDurationLine(operation, total_ns); } void PrintBasicXorEncryptTypedElementsTimings( bool is_fixed, size_t num_elements, size_t element_size, - int64_t encrypt_elements_loop_us, - int64_t get_raw_element_us, - int64_t xor_encrypt_us, - int64_t set_element_us, - int64_t finalize_buffer_us, - int64_t write_header_us, - int64_t total_us) { - std::cout << "+++++ BasicXorEncryptor EncryptTypedElements timings (microseconds) +++++" << std::endl; + int64_t encrypt_elements_loop_ns, + int64_t get_raw_element_ns, + int64_t xor_encrypt_ns, + int64_t set_element_ns, + int64_t finalize_buffer_ns, + int64_t write_header_ns, + int64_t total_ns) { + std::cout << "+++++ BasicXorEncryptor EncryptTypedElements timings (microseconds + nanoseconds) +++++" << std::endl; std::cout << " mode: " << (is_fixed ? "fixed" : "variable") << std::endl; std::cout << " num_elements: " << num_elements << std::endl; std::cout << " element_size: " << element_size << std::endl; - std::cout << " EncryptElementsLoop: " << encrypt_elements_loop_us << std::endl; - std::cout << " GetRawElement(aggregated): " << get_raw_element_us << std::endl; - std::cout << " XorEncrypt(aggregated): " << xor_encrypt_us << std::endl; - std::cout << " SetElement(aggregated): " << set_element_us << std::endl; - std::cout << " LoopResidual(iterator+other): " - << (encrypt_elements_loop_us - get_raw_element_us - xor_encrypt_us - set_element_us) << std::endl; - std::cout << " FinalizeBuffer: " << finalize_buffer_us << std::endl; - std::cout << " WriteHeader: " << write_header_us << std::endl; - std::cout << " EncryptTypedElements(total): " << total_us << std::endl; + PrintDurationLine("EncryptElementsLoop", encrypt_elements_loop_ns); + PrintDurationLine("GetRawElement(aggregated)", get_raw_element_ns); + PrintDurationLine("XorEncrypt(aggregated)", xor_encrypt_ns); + PrintDurationLine("SetElement(aggregated)", set_element_ns); + PrintDurationLine( + "LoopResidual(iterator+other)", + encrypt_elements_loop_ns - get_raw_element_ns - xor_encrypt_ns - set_element_ns); + PrintDurationLine("FinalizeBuffer", finalize_buffer_ns); + PrintDurationLine("WriteHeader", write_header_ns); + PrintDurationLine("EncryptTypedElements(total)", total_ns); } - void PrintBasicXorEncryptValueListTimings(int64_t visit_dispatch_us, int64_t total_us) { - std::cout << "+++++ BasicXorEncryptor EncryptValueList timings (microseconds) +++++" << std::endl; - std::cout << " VariantVisitAndEncrypt: " << visit_dispatch_us << std::endl; - std::cout << " EncryptValueList(total): " << total_us << std::endl; + void PrintBasicXorEncryptValueListTimings(int64_t visit_dispatch_ns, int64_t total_ns) { + std::cout << "+++++ BasicXorEncryptor EncryptValueList timings (microseconds + nanoseconds) +++++" << std::endl; + PrintDurationLine("VariantVisitAndEncrypt", visit_dispatch_ns); + PrintDurationLine("EncryptValueList(total)", total_ns); } void PrintBasicXorDecryptValueListTimings( bool is_fixed, size_t num_elements, - int64_t read_header_us, - int64_t setup_buffer_us, - int64_t decrypt_elements_us, - int64_t total_us) { - std::cout << "+++++ BasicXorEncryptor DecryptValueList timings (microseconds) +++++" << std::endl; + int64_t read_header_ns, + int64_t setup_buffer_ns, + int64_t decrypt_elements_ns, + int64_t total_ns) { + std::cout << "+++++ BasicXorEncryptor DecryptValueList timings (microseconds + nanoseconds) +++++" << std::endl; std::cout << " mode: " << (is_fixed ? "fixed" : "variable") << std::endl; std::cout << " num_elements: " << num_elements << std::endl; - std::cout << " ReadHeader: " << read_header_us << std::endl; - std::cout << " SetupEncryptedBuffer: " << setup_buffer_us << std::endl; - std::cout << " DecryptElements: " << decrypt_elements_us << std::endl; - std::cout << " DecryptValueList(total): " << total_us << std::endl; + PrintDurationLine("ReadHeader", read_header_ns); + PrintDurationLine("SetupEncryptedBuffer", setup_buffer_ns); + PrintDurationLine("DecryptElements", decrypt_elements_ns); + PrintDurationLine("DecryptValueList(total)", total_ns); } } @@ -120,14 +131,14 @@ std::vector BasicXorEncryptor::XorDecrypt(tcb::span data std::vector BasicXorEncryptor::EncryptBlock(tcb::span data) { auto start = std::chrono::steady_clock::now(); auto out = XorEncrypt(data, key_id_); - PrintBasicXorBlockTimings("EncryptBlock", ElapsedMicrosecondsSince(start)); + PrintBasicXorBlockTimings("EncryptBlock", ElapsedNanosecondsSince(start)); return out; } std::vector BasicXorEncryptor::DecryptBlock(tcb::span data) { auto start = std::chrono::steady_clock::now(); auto out = XorDecrypt(data, key_id_); - PrintBasicXorBlockTimings("DecryptBlock", ElapsedMicrosecondsSince(start)); + PrintBasicXorBlockTimings("DecryptBlock", ElapsedNanosecondsSince(start)); return out; } @@ -182,11 +193,11 @@ std::vector BasicXorEncryptor::EncryptTypedElements( std::vector final_buffer; size_t element_size = 0; - int64_t encrypt_elements_loop_us = 0; - int64_t get_raw_element_us = 0; - int64_t xor_encrypt_us = 0; - int64_t set_element_us = 0; - int64_t finalize_buffer_us = 0; + int64_t encrypt_elements_loop_ns = 0; + int64_t get_raw_element_ns = 0; + int64_t xor_encrypt_ns = 0; + int64_t set_element_ns = 0; + int64_t finalize_buffer_ns = 0; // Encrypt fixed-size elements if constexpr (is_fixed) { @@ -197,18 +208,18 @@ std::vector BasicXorEncryptor::EncryptTypedElements( for (size_t i = 0; i < num_elements; ++i) { auto op_start = std::chrono::steady_clock::now(); auto raw_bytes = input_buffer.GetRawElement(i); - get_raw_element_us += ElapsedMicrosecondsSince(op_start); + get_raw_element_ns += ElapsedNanosecondsSince(op_start); op_start = std::chrono::steady_clock::now(); auto encrypted = XorEncrypt(raw_bytes, key_id); - xor_encrypt_us += ElapsedMicrosecondsSince(op_start); + xor_encrypt_ns += ElapsedNanosecondsSince(op_start); op_start = std::chrono::steady_clock::now(); output_buffer.SetElement(i, tcb::span(encrypted)); - set_element_us += ElapsedMicrosecondsSince(op_start); + set_element_ns += ElapsedNanosecondsSince(op_start); } - encrypt_elements_loop_us = ElapsedMicrosecondsSince(stage_start); + encrypt_elements_loop_ns = ElapsedNanosecondsSince(stage_start); stage_start = std::chrono::steady_clock::now(); final_buffer = output_buffer.FinalizeAndTakeBuffer(); - finalize_buffer_us = ElapsedMicrosecondsSince(stage_start); + finalize_buffer_ns = ElapsedNanosecondsSince(stage_start); } // Encrypt variable-size elements @@ -222,20 +233,20 @@ std::vector BasicXorEncryptor::EncryptTypedElements( auto op_start = std::chrono::steady_clock::now(); auto raw_bytes = input_buffer.GetRawElement(i); - get_raw_element_us += ElapsedMicrosecondsSince(op_start); + get_raw_element_ns += ElapsedNanosecondsSince(op_start); op_start = std::chrono::steady_clock::now(); auto encrypted = XorEncrypt(raw_bytes, key_id); - xor_encrypt_us += ElapsedMicrosecondsSince(op_start); + xor_encrypt_ns += ElapsedNanosecondsSince(op_start); op_start = std::chrono::steady_clock::now(); output_buffer.SetElement(i, tcb::span(encrypted)); - set_element_us += ElapsedMicrosecondsSince(op_start); + set_element_ns += ElapsedNanosecondsSince(op_start); } - encrypt_elements_loop_us = ElapsedMicrosecondsSince(stage_start); + encrypt_elements_loop_ns = ElapsedNanosecondsSince(stage_start); stage_start = std::chrono::steady_clock::now(); final_buffer = output_buffer.FinalizeAndTakeBuffer(); - finalize_buffer_us = ElapsedMicrosecondsSince(stage_start); + finalize_buffer_ns = ElapsedNanosecondsSince(stage_start); } // Write the header to the final buffer and return it. @@ -243,19 +254,19 @@ std::vector BasicXorEncryptor::EncryptTypedElements( WriteHeader(final_buffer, {is_fixed, static_cast(num_elements), static_cast(element_size)}); - auto write_header_us = ElapsedMicrosecondsSince(header_start); + auto write_header_ns = ElapsedNanosecondsSince(header_start); PrintBasicXorEncryptTypedElementsTimings( is_fixed, num_elements, element_size, - encrypt_elements_loop_us, - get_raw_element_us, - xor_encrypt_us, - set_element_us, - finalize_buffer_us, - write_header_us, - ElapsedMicrosecondsSince(total_start)); + encrypt_elements_loop_ns, + get_raw_element_ns, + xor_encrypt_ns, + set_element_ns, + finalize_buffer_ns, + write_header_ns, + ElapsedNanosecondsSince(total_start)); return final_buffer; } @@ -280,8 +291,8 @@ std::vector BasicXorEncryptor::EncryptValueList( auto encrypted = std::visit([&](const auto& input_buffer) { return EncryptTypedElements(input_buffer, key_id_); }, typed_buffer); - auto visit_dispatch_us = ElapsedMicrosecondsSince(visit_start); - PrintBasicXorEncryptValueListTimings(visit_dispatch_us, ElapsedMicrosecondsSince(total_start)); + auto visit_dispatch_ns = ElapsedNanosecondsSince(visit_start); + PrintBasicXorEncryptValueListTimings(visit_dispatch_ns, ElapsedNanosecondsSince(total_start)); return encrypted; } @@ -311,7 +322,7 @@ TypedValuesBuffer BasicXorEncryptor::DecryptValueList( auto stage_start = std::chrono::steady_clock::now(); auto header = ReadHeader(encrypted_bytes); - auto read_header_us = ElapsedMicrosecondsSince(stage_start); + auto read_header_ns = ElapsedNanosecondsSince(stage_start); auto num_elements = static_cast(header.num_elements); // Decrypt fixed-size elements @@ -321,7 +332,7 @@ TypedValuesBuffer BasicXorEncryptor::DecryptValueList( // Create a fixed-sized byte buffer for reading the encrypted elements. TypedBufferRawBytesFixedSized encrypted_buffer{ encrypted_bytes, kFixedHeaderLength, RawBytesFixedSizedCodec{header.element_size}}; - auto setup_buffer_us = ElapsedMicrosecondsSince(stage_start); + auto setup_buffer_ns = ElapsedNanosecondsSince(stage_start); // Populate a typed buffer with the decrypted elements in the corresponding type. stage_start = std::chrono::steady_clock::now(); @@ -331,8 +342,8 @@ TypedValuesBuffer BasicXorEncryptor::DecryptValueList( auto out = DecryptFixedSizedElementsIntoTypedBuffer( encrypted_buffer, key_id_, TypedBufferI32{num_elements}); PrintBasicXorDecryptValueListTimings( - true, num_elements, read_header_us, setup_buffer_us, - ElapsedMicrosecondsSince(stage_start), ElapsedMicrosecondsSince(total_start)); + true, num_elements, read_header_ns, setup_buffer_ns, + ElapsedNanosecondsSince(stage_start), ElapsedNanosecondsSince(total_start)); return out; } case Type::INT64: @@ -340,8 +351,8 @@ TypedValuesBuffer BasicXorEncryptor::DecryptValueList( auto out = DecryptFixedSizedElementsIntoTypedBuffer( encrypted_buffer, key_id_, TypedBufferI64{num_elements}); PrintBasicXorDecryptValueListTimings( - true, num_elements, read_header_us, setup_buffer_us, - ElapsedMicrosecondsSince(stage_start), ElapsedMicrosecondsSince(total_start)); + true, num_elements, read_header_ns, setup_buffer_ns, + ElapsedNanosecondsSince(stage_start), ElapsedNanosecondsSince(total_start)); return out; } case Type::INT96: @@ -349,8 +360,8 @@ TypedValuesBuffer BasicXorEncryptor::DecryptValueList( auto out = DecryptFixedSizedElementsIntoTypedBuffer( encrypted_buffer, key_id_, TypedBufferInt96{num_elements}); PrintBasicXorDecryptValueListTimings( - true, num_elements, read_header_us, setup_buffer_us, - ElapsedMicrosecondsSince(stage_start), ElapsedMicrosecondsSince(total_start)); + true, num_elements, read_header_ns, setup_buffer_ns, + ElapsedNanosecondsSince(stage_start), ElapsedNanosecondsSince(total_start)); return out; } case Type::FLOAT: @@ -358,8 +369,8 @@ TypedValuesBuffer BasicXorEncryptor::DecryptValueList( auto out = DecryptFixedSizedElementsIntoTypedBuffer( encrypted_buffer, key_id_, TypedBufferFloat{num_elements}); PrintBasicXorDecryptValueListTimings( - true, num_elements, read_header_us, setup_buffer_us, - ElapsedMicrosecondsSince(stage_start), ElapsedMicrosecondsSince(total_start)); + true, num_elements, read_header_ns, setup_buffer_ns, + ElapsedNanosecondsSince(stage_start), ElapsedNanosecondsSince(total_start)); return out; } case Type::DOUBLE: @@ -367,8 +378,8 @@ TypedValuesBuffer BasicXorEncryptor::DecryptValueList( auto out = DecryptFixedSizedElementsIntoTypedBuffer( encrypted_buffer, key_id_, TypedBufferDouble{num_elements}); PrintBasicXorDecryptValueListTimings( - true, num_elements, read_header_us, setup_buffer_us, - ElapsedMicrosecondsSince(stage_start), ElapsedMicrosecondsSince(total_start)); + true, num_elements, read_header_ns, setup_buffer_ns, + ElapsedNanosecondsSince(stage_start), ElapsedNanosecondsSince(total_start)); return out; } case Type::FIXED_LEN_BYTE_ARRAY: @@ -377,8 +388,8 @@ TypedValuesBuffer BasicXorEncryptor::DecryptValueList( encrypted_buffer, key_id_, TypedBufferRawBytesFixedSized{num_elements, 0, RawBytesFixedSizedCodec{header.element_size}}); PrintBasicXorDecryptValueListTimings( - true, num_elements, read_header_us, setup_buffer_us, - ElapsedMicrosecondsSince(stage_start), ElapsedMicrosecondsSince(total_start)); + true, num_elements, read_header_ns, setup_buffer_ns, + ElapsedNanosecondsSince(stage_start), ElapsedNanosecondsSince(total_start)); return out; } default: @@ -393,7 +404,7 @@ TypedValuesBuffer BasicXorEncryptor::DecryptValueList( stage_start = std::chrono::steady_clock::now(); // Create a variable-sized byte buffer for reading the encrypted elements. TypedBufferRawBytesVariableSized encrypted_buffer{ encrypted_bytes, kVariableHeaderLength}; - auto setup_buffer_us = ElapsedMicrosecondsSince(stage_start); + auto setup_buffer_ns = ElapsedNanosecondsSince(stage_start); switch (datatype_) { // Create a BYTE-ARRAY typed buffer for storing the decrypted elements. @@ -407,10 +418,10 @@ TypedValuesBuffer BasicXorEncryptor::DecryptValueList( output_buffer.SetElement(output_index, tcb::span(decrypted_bytes)); output_index++; } - auto decrypt_elements_us = ElapsedMicrosecondsSince(stage_start); + auto decrypt_elements_ns = ElapsedNanosecondsSince(stage_start); PrintBasicXorDecryptValueListTimings( - false, num_elements, read_header_us, setup_buffer_us, - decrypt_elements_us, ElapsedMicrosecondsSince(total_start)); + false, num_elements, read_header_ns, setup_buffer_ns, + decrypt_elements_ns, ElapsedNanosecondsSince(total_start)); return output_buffer; } default: From f52a9e1ca0fa6fed2a27a06b302ab2c90efafeb6 Mon Sep 17 00:00:00 2001 From: Alejandro Valerio Date: Tue, 10 Mar 2026 20:58:29 -0600 Subject: [PATCH 03/18] - Small optimization in GetRawElement for fixed-size elements - Made various functions inline to avoid extra function calls --- src/processing/typed_buffer.h | 60 +++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/src/processing/typed_buffer.h b/src/processing/typed_buffer.h index b99d810..f0062c2 100644 --- a/src/processing/typed_buffer.h +++ b/src/processing/typed_buffer.h @@ -208,7 +208,7 @@ ByteBuffer::ByteBuffer( // Initializes `num_elements_` and `offsets_` from the span. // Called in a lazy manner when the buffer is accessed with GetElement or GetNumElements, avoiding unnecessary initialization. template -void ByteBuffer::InitializeFromSpan() const { +inline void ByteBuffer::InitializeFromSpan() const { if (elements_span_.size() < prefix_size_) { throw InvalidInputException("Malformed buffer: prefix_size exceeds span size"); } @@ -265,7 +265,7 @@ void ByteBuffer::InitializeFromSpan() const { } template -void ByteBuffer::EnsureInitializedFromSpan() const { +inline void ByteBuffer::EnsureInitializedFromSpan() const { // If the span is already initialized, skip it. if (is_initialized_from_span_) { return; @@ -280,7 +280,7 @@ void ByteBuffer::EnsureInitializedFromSpan() const { // For read-only buffers, gets the number of elements in the buffer and sets num_elements_ if not already set. // A lighter version to only get num_elements_ and avoid calling InitializeFromSpan that also builds offsets_. template -size_t ByteBuffer::GetNumElements() const { +inline size_t ByteBuffer::GetNumElements() const { if (num_elements_ != kUnsetSize) { return num_elements_; } @@ -291,7 +291,7 @@ size_t ByteBuffer::GetNumElements() const { } template -size_t ByteBuffer::EstimateOffsetsReserveCountFromSample(tcb::span bytes) { +inline size_t ByteBuffer::EstimateOffsetsReserveCountFromSample(tcb::span bytes) { if (bytes.empty()) return 0; @@ -338,7 +338,7 @@ size_t ByteBuffer::EstimateOffsetsReserveCountFromSample(tcb::span -size_t ByteBuffer::CalculateOffsetOfElement(size_t position) const { +inline size_t ByteBuffer::CalculateOffsetOfElement(size_t position) const { EnsureInitializedFromSpan(); if (position >= num_elements_) { throw InvalidInputException("Element position out of range during CalculateOffsetOfElement"); @@ -350,19 +350,23 @@ size_t ByteBuffer::CalculateOffsetOfElement(size_t position) const { } template -tcb::span ByteBuffer::GetRawElement(size_t position) const { +inline tcb::span ByteBuffer::GetRawElement(size_t position) const { EnsureInitializedFromSpan(); - if (position >= num_elements_) { - throw InvalidInputException("Element position out of range during GetRawElement"); - } - const size_t offset = CalculateOffsetOfElement(position); // For fixed-size elements are stored contiguously. if constexpr (is_fixed_sized) { + if (position >= num_elements_) { + throw InvalidInputException("Element position out of range during GetRawElement"); + } + const size_t offset = prefix_size_ + (position * element_size_); return elements_span_.subspan(offset, element_size_); } // For variable-size elements, we need to read the size first [u32 size][element]. + if (position >= num_elements_) { + throw InvalidInputException("Element position out of range during GetRawElement"); + } + const size_t offset = offsets_[position]; if (offset == kUnsetSize) { throw InvalidInputException("Element position has not been written yet"); } @@ -371,7 +375,7 @@ tcb::span ByteBuffer::GetRawElement(size_t position) const } template -typename ByteBuffer::value_type ByteBuffer::GetElement(size_t position) const { +inline typename ByteBuffer::value_type ByteBuffer::GetElement(size_t position) const { return codec_.Decode(GetRawElement(position)); } @@ -384,7 +388,7 @@ typename ByteBuffer::value_type ByteBuffer::GetElement(size_t posi // ----------------------------------------------------------------------------- template -ByteBuffer::ConstIterator::ConstIterator(const ByteBuffer* buffer, size_t cursor_offset) +inline ByteBuffer::ConstIterator::ConstIterator(const ByteBuffer* buffer, size_t cursor_offset) : buffer_(buffer), cursor_offset_(cursor_offset), elements_span_size_(buffer != nullptr ? buffer->elements_span_.size() : 0u), @@ -416,7 +420,7 @@ inline size_t ByteBuffer::ConstIterator::ReadAndValidateVariableElementSi } template -tcb::span ByteBuffer::ConstIterator::RawSpanAtCursor() const { +inline tcb::span ByteBuffer::ConstIterator::RawSpanAtCursor() const { if (buffer_ == nullptr || cursor_offset_ >= elements_span_size_) { throw InvalidInputException("Cannot dereference ByteBuffer iterator at end position"); } @@ -429,14 +433,14 @@ tcb::span ByteBuffer::ConstIterator::RawSpanAtCursor() con } template -typename ByteBuffer::ConstIterator::value_type ByteBuffer::ConstIterator::operator*() const { +inline typename ByteBuffer::ConstIterator::value_type ByteBuffer::ConstIterator::operator*() const { // Decode converts raw bytes into the codec's value_type (e.g. int32_t, float, string_view). // This keeps the iterator's return type consistent with GetElement across all codecs. return buffer_->codec_.Decode(RawSpanAtCursor()); } template -typename ByteBuffer::ConstIterator& ByteBuffer::ConstIterator::operator++() { +inline typename ByteBuffer::ConstIterator& ByteBuffer::ConstIterator::operator++() { if (buffer_ == nullptr || cursor_offset_ >= elements_span_size_) { return *this; } @@ -451,23 +455,23 @@ typename ByteBuffer::ConstIterator& ByteBuffer::ConstIterator::ope } template -bool ByteBuffer::ConstIterator::operator==(const ConstIterator& other) const { +inline bool ByteBuffer::ConstIterator::operator==(const ConstIterator& other) const { return buffer_ == other.buffer_ && cursor_offset_ == other.cursor_offset_; } template -bool ByteBuffer::ConstIterator::operator!=(const ConstIterator& other) const { +inline bool ByteBuffer::ConstIterator::operator!=(const ConstIterator& other) const { return !(*this == other); } template -tcb::span ByteBuffer::ConstRawIterator::operator*() const { +inline tcb::span ByteBuffer::ConstRawIterator::operator*() const { // Returns the raw bytes for the current element, consistent with GetRawElement. return this->RawSpanAtCursor(); } template -void ByteBuffer::ValidateIteratorReadPreconditions() const { +inline void ByteBuffer::ValidateIteratorReadPreconditions() const { if (is_write_buffer_initialized_) { throw InvalidInputException("Iterator is only available for read buffers"); } @@ -486,19 +490,19 @@ void ByteBuffer::ValidateIteratorReadPreconditions() const { } template -typename ByteBuffer::ConstIterator ByteBuffer::begin() const { +inline typename ByteBuffer::ConstIterator ByteBuffer::begin() const { ValidateIteratorReadPreconditions(); return ConstIterator(this, prefix_size_); } template -typename ByteBuffer::ConstIterator ByteBuffer::end() const { +inline typename ByteBuffer::ConstIterator ByteBuffer::end() const { ValidateIteratorReadPreconditions(); return ConstIterator(this, elements_span_.size()); } template -typename ByteBuffer::RawElementsView ByteBuffer::raw_elements() const { +inline typename ByteBuffer::RawElementsView ByteBuffer::raw_elements() const { ValidateIteratorReadPreconditions(); return RawElementsView{this}; } @@ -540,7 +544,7 @@ ByteBuffer::ByteBuffer( // Initializes `write_buffer_`, `offsets_` and `elements_span_` template -void ByteBuffer::InitializeForWriteBuffer(size_t variable_size_reserved_bytes_hint) { +inline void ByteBuffer::InitializeForWriteBuffer(size_t variable_size_reserved_bytes_hint) { // Fixed-size elements if constexpr (is_fixed_sized) { if (element_size_ <= 0) { @@ -592,7 +596,7 @@ void ByteBuffer::InitializeForWriteBuffer(size_t variable_size_reserved_b template -tcb::span ByteBuffer::GetWritableSpanForElement(size_t position, size_t payload_size) { +inline tcb::span ByteBuffer::GetWritableSpanForElement(size_t position, size_t payload_size) { if (!is_write_buffer_initialized_) { throw InvalidInputException("Cannot GetWriteSpanForElement: write buffer is not initialized."); } @@ -651,7 +655,7 @@ tcb::span ByteBuffer::GetWritableSpanForElement(size_t position, } template -void ByteBuffer::SetElement(size_t position, const value_type& element) { +inline void ByteBuffer::SetElement(size_t position, const value_type& element) { if constexpr (is_fixed_sized) { auto write_span = GetWritableSpanForElement(position, element_size_); codec_.Encode(element, write_span); @@ -662,13 +666,13 @@ void ByteBuffer::SetElement(size_t position, const value_type& element) { } template -void ByteBuffer::SetRawElement(size_t position, tcb::span raw) { +inline void ByteBuffer::SetRawElement(size_t position, tcb::span raw) { auto write_span = GetWritableSpanForElement(position, raw.size()); std::memcpy(write_span.data(), raw.data(), raw.size()); } template -std::vector ByteBuffer::FinalizeAndTakeBuffer() { +inline std::vector ByteBuffer::FinalizeAndTakeBuffer() { if (is_write_buffer_finalized_) { throw InvalidInputException("FinalizeAndTakeBuffer: write buffer has already been finalized"); } @@ -739,7 +743,7 @@ std::vector ByteBuffer::FinalizeAndTakeBuffer() { } template -void ByteBuffer::RebindSpanToWriteBuffer() { +inline void ByteBuffer::RebindSpanToWriteBuffer() { elements_span_ = tcb::span(write_buffer_.data(), write_buffer_.size()); } From 2d551a2f91a4dc5841c6a1176be99f99f450e552 Mon Sep 17 00:00:00 2001 From: Alejandro Valerio Date: Wed, 11 Mar 2026 07:28:47 -0600 Subject: [PATCH 04/18] - Updating to use GetWritableRawElement in the BasicXorEncryptor. - Marked various functions as inline - Switched to ues XorEncryptInto --- src/processing/encryption_sequencer.cpp | 6 +- .../encryptors/basic_xor_encryptor.cpp | 178 ++++-------------- .../encryptors/basic_xor_encryptor.h | 4 +- src/processing/typed_buffer.h | 8 +- 4 files changed, 52 insertions(+), 144 deletions(-) diff --git a/src/processing/encryption_sequencer.cpp b/src/processing/encryption_sequencer.cpp index 506bab0..91c32b6 100644 --- a/src/processing/encryption_sequencer.cpp +++ b/src/processing/encryption_sequencer.cpp @@ -49,9 +49,9 @@ namespace { } void PrintDurationLine(const char* label, int64_t nanoseconds) { - std::cout << " " << label << ": " - << ToMicroseconds(nanoseconds) << " us" - << " (" << nanoseconds << " ns)" << std::endl; + std::cout << " " << std::left << std::setw(34) << label + << " : " << std::right << std::setw(12) << ToMicroseconds(nanoseconds) << " us" + << " (" << std::setw(12) << nanoseconds << " ns)" << std::endl; } void PrintDecodeAndEncryptTimings( diff --git a/src/processing/encryptors/basic_xor_encryptor.cpp b/src/processing/encryptors/basic_xor_encryptor.cpp index 4cc2edb..ec34965 100644 --- a/src/processing/encryptors/basic_xor_encryptor.cpp +++ b/src/processing/encryptors/basic_xor_encryptor.cpp @@ -102,21 +102,19 @@ namespace { // Functions for encrypting and decrypting byte arrays. // --------------------------------------------------------------------------- -std::vector BasicXorEncryptor::XorEncrypt(tcb::span data) { - if (data.empty()) { - return {}; +void BasicXorEncryptor::XorEncryptInto(tcb::span data, tcb::span out) { + if (data.size() != out.size()) { + throw InvalidInputException("XorEncryptInto: input and output sizes must match"); } size_t key_hash = key_id_hash_; - std::vector out(data.size()); for (size_t i = 0; i < data.size(); ++i) { out[i] = data[i] ^ (key_hash & 0xFF); key_hash = (key_hash << 1) | (key_hash >> 31); } - return out; } -std::vector BasicXorEncryptor::XorDecrypt(tcb::span data) { - return XorEncrypt(data); +void BasicXorEncryptor::XorDecryptInto(tcb::span data, tcb::span out) { + XorEncryptInto(data, out); } // --------------------------------------------------------------------------- @@ -124,16 +122,20 @@ std::vector BasicXorEncryptor::XorDecrypt(tcb::span data // --------------------------------------------------------------------------- std::vector BasicXorEncryptor::EncryptBlock(tcb::span data) { - auto start = std::chrono::steady_clock::now(); - auto out = XorEncrypt(data, key_id_); - PrintBasicXorBlockTimings("EncryptBlock", ElapsedNanosecondsSince(start)); + if (data.empty()) { + return {}; + } + std::vector out(data.size()); + XorEncryptInto(data, tcb::span(out.data(), out.size())); return out; } std::vector BasicXorEncryptor::DecryptBlock(tcb::span data) { - auto start = std::chrono::steady_clock::now(); - auto out = XorDecrypt(data, key_id_); - PrintBasicXorBlockTimings("DecryptBlock", ElapsedNanosecondsSince(start)); + if (data.empty()) { + return {}; + } + std::vector out(data.size()); + XorDecryptInto(data, tcb::span(out.data(), out.size())); return out; } @@ -169,7 +171,6 @@ std::vector BasicXorEncryptor::DecryptBlock(tcb::span da template std::vector BasicXorEncryptor::EncryptTypedElements( const TypedBuffer& input_buffer) { - auto total_start = std::chrono::steady_clock::now(); constexpr bool is_fixed = TypedBuffer::is_fixed_sized; constexpr size_t prefix_length = is_fixed ? kFixedHeaderLength : kVariableHeaderLength; const size_t num_elements = input_buffer.GetNumElements(); @@ -188,33 +189,18 @@ std::vector BasicXorEncryptor::EncryptTypedElements( std::vector final_buffer; size_t element_size = 0; - int64_t encrypt_elements_loop_ns = 0; - int64_t get_raw_element_ns = 0; - int64_t xor_encrypt_ns = 0; - int64_t set_element_ns = 0; - int64_t finalize_buffer_ns = 0; // Encrypt fixed-size elements if constexpr (is_fixed) { element_size = input_buffer.GetElementSize(); TypedBufferRawBytesFixedSized output_buffer{ num_elements, prefix_length, RawBytesFixedSizedCodec{element_size}}; - auto stage_start = std::chrono::steady_clock::now(); for (size_t i = 0; i < num_elements; ++i) { - auto op_start = std::chrono::steady_clock::now(); auto raw_bytes = input_buffer.GetRawElement(i); - get_raw_element_ns += ElapsedNanosecondsSince(op_start); - op_start = std::chrono::steady_clock::now(); - auto encrypted = XorEncrypt(raw_bytes, key_id); - xor_encrypt_ns += ElapsedNanosecondsSince(op_start); - op_start = std::chrono::steady_clock::now(); - output_buffer.SetElement(i, tcb::span(encrypted)); - set_element_ns += ElapsedNanosecondsSince(op_start); + auto write_span = output_buffer.GetWritableRawElement(i, raw_bytes.size()); + XorEncryptInto(raw_bytes, write_span); } - encrypt_elements_loop_ns = ElapsedNanosecondsSince(stage_start); - stage_start = std::chrono::steady_clock::now(); final_buffer = output_buffer.FinalizeAndTakeBuffer(); - finalize_buffer_ns = ElapsedNanosecondsSince(stage_start); } // Encrypt variable-size elements @@ -222,46 +208,19 @@ std::vector BasicXorEncryptor::EncryptTypedElements( auto reserved_bytes_hint = input_buffer.GetRawBufferSize(); TypedBufferRawBytesVariableSized output_buffer{ num_elements, reserved_bytes_hint, true, prefix_length}; - - auto stage_start = std::chrono::steady_clock::now(); - for (size_t i = 0; i < num_elements; ++i) { - - auto op_start = std::chrono::steady_clock::now(); - auto raw_bytes = input_buffer.GetRawElement(i); - get_raw_element_ns += ElapsedNanosecondsSince(op_start); - - op_start = std::chrono::steady_clock::now(); - auto encrypted = XorEncrypt(raw_bytes, key_id); - xor_encrypt_ns += ElapsedNanosecondsSince(op_start); - - op_start = std::chrono::steady_clock::now(); - output_buffer.SetElement(i, tcb::span(encrypted)); - set_element_ns += ElapsedNanosecondsSince(op_start); + size_t output_index = 0; + for (const auto raw_bytes : input_buffer.raw_elements()) { + auto write_span = output_buffer.GetWritableRawElement(output_index, raw_bytes.size()); + XorEncryptInto(raw_bytes, write_span); + output_index++; } - encrypt_elements_loop_ns = ElapsedNanosecondsSince(stage_start); - stage_start = std::chrono::steady_clock::now(); final_buffer = output_buffer.FinalizeAndTakeBuffer(); - finalize_buffer_ns = ElapsedNanosecondsSince(stage_start); } // Write the header to the final buffer and return it. - auto header_start = std::chrono::steady_clock::now(); WriteHeader(final_buffer, {is_fixed, static_cast(num_elements), static_cast(element_size)}); - auto write_header_ns = ElapsedNanosecondsSince(header_start); - - PrintBasicXorEncryptTypedElementsTimings( - is_fixed, - num_elements, - element_size, - encrypt_elements_loop_ns, - get_raw_element_ns, - xor_encrypt_ns, - set_element_ns, - finalize_buffer_ns, - write_header_ns, - ElapsedNanosecondsSince(total_start)); return final_buffer; } @@ -282,13 +241,9 @@ std::vector BasicXorEncryptor::EncryptValueList( // std::visit extracts the concrete buffer type from the TypedValuesBuffer variant // and forwards it to EncryptTypedElements, which handles all buffer types generically. - auto visit_start = std::chrono::steady_clock::now(); - auto encrypted = std::visit([&](const auto& input_buffer) { - return EncryptTypedElements(input_buffer, key_id_); + return std::visit([&](const auto& input_buffer) { + return EncryptTypedElements(input_buffer); }, typed_buffer); - auto visit_dispatch_ns = ElapsedNanosecondsSince(visit_start); - PrintBasicXorEncryptValueListTimings(visit_dispatch_ns, ElapsedNanosecondsSince(total_start)); - return encrypted; } // --------------------------------------------------------------------------- @@ -304,8 +259,8 @@ TypedBuffer BasicXorEncryptor::DecryptFixedSizedElementsIntoTypedBuffer( const TypedBufferRawBytesFixedSized& encrypted_buffer, TypedBuffer output_buffer) { size_t output_index = 0; for (const auto raw_bytes : encrypted_buffer.raw_elements()) { - auto decrypted_bytes = XorDecrypt(raw_bytes); - output_buffer.SetRawElement(output_index, tcb::span(decrypted_bytes)); + auto write_span = output_buffer.GetWritableRawElement(output_index, raw_bytes.size()); + XorDecryptInto(raw_bytes, write_span); output_index++; } return output_buffer; @@ -313,80 +268,34 @@ TypedBuffer BasicXorEncryptor::DecryptFixedSizedElementsIntoTypedBuffer( TypedValuesBuffer BasicXorEncryptor::DecryptValueList( tcb::span encrypted_bytes) { - auto total_start = std::chrono::steady_clock::now(); - - auto stage_start = std::chrono::steady_clock::now(); auto header = ReadHeader(encrypted_bytes); - auto read_header_ns = ElapsedNanosecondsSince(stage_start); auto num_elements = static_cast(header.num_elements); // Decrypt fixed-size elements if (header.is_fixed) { - stage_start = std::chrono::steady_clock::now(); - // Create a fixed-sized byte buffer for reading the encrypted elements. TypedBufferRawBytesFixedSized encrypted_buffer{ encrypted_bytes, kFixedHeaderLength, RawBytesFixedSizedCodec{header.element_size}}; - auto setup_buffer_ns = ElapsedNanosecondsSince(stage_start); - - // Populate a typed buffer with the decrypted elements in the corresponding type. - stage_start = std::chrono::steady_clock::now(); switch (datatype_) { case Type::INT32: - { - auto out = DecryptFixedSizedElementsIntoTypedBuffer( - encrypted_buffer, key_id_, TypedBufferI32{num_elements}); - PrintBasicXorDecryptValueListTimings( - true, num_elements, read_header_ns, setup_buffer_ns, - ElapsedNanosecondsSince(stage_start), ElapsedNanosecondsSince(total_start)); - return out; - } + return DecryptFixedSizedElementsIntoTypedBuffer( + encrypted_buffer, TypedBufferI32{num_elements}); case Type::INT64: - { - auto out = DecryptFixedSizedElementsIntoTypedBuffer( - encrypted_buffer, key_id_, TypedBufferI64{num_elements}); - PrintBasicXorDecryptValueListTimings( - true, num_elements, read_header_ns, setup_buffer_ns, - ElapsedNanosecondsSince(stage_start), ElapsedNanosecondsSince(total_start)); - return out; - } + return DecryptFixedSizedElementsIntoTypedBuffer( + encrypted_buffer, TypedBufferI64{num_elements}); case Type::INT96: - { - auto out = DecryptFixedSizedElementsIntoTypedBuffer( - encrypted_buffer, key_id_, TypedBufferInt96{num_elements}); - PrintBasicXorDecryptValueListTimings( - true, num_elements, read_header_ns, setup_buffer_ns, - ElapsedNanosecondsSince(stage_start), ElapsedNanosecondsSince(total_start)); - return out; - } + return DecryptFixedSizedElementsIntoTypedBuffer( + encrypted_buffer, TypedBufferInt96{num_elements}); case Type::FLOAT: - { - auto out = DecryptFixedSizedElementsIntoTypedBuffer( - encrypted_buffer, key_id_, TypedBufferFloat{num_elements}); - PrintBasicXorDecryptValueListTimings( - true, num_elements, read_header_ns, setup_buffer_ns, - ElapsedNanosecondsSince(stage_start), ElapsedNanosecondsSince(total_start)); - return out; - } + return DecryptFixedSizedElementsIntoTypedBuffer( + encrypted_buffer, TypedBufferFloat{num_elements}); case Type::DOUBLE: - { - auto out = DecryptFixedSizedElementsIntoTypedBuffer( - encrypted_buffer, key_id_, TypedBufferDouble{num_elements}); - PrintBasicXorDecryptValueListTimings( - true, num_elements, read_header_ns, setup_buffer_ns, - ElapsedNanosecondsSince(stage_start), ElapsedNanosecondsSince(total_start)); - return out; - } + return DecryptFixedSizedElementsIntoTypedBuffer( + encrypted_buffer, TypedBufferDouble{num_elements}); case Type::FIXED_LEN_BYTE_ARRAY: - { - auto out = DecryptFixedSizedElementsIntoTypedBuffer( - encrypted_buffer, key_id_, + return DecryptFixedSizedElementsIntoTypedBuffer( + encrypted_buffer, TypedBufferRawBytesFixedSized{num_elements, 0, RawBytesFixedSizedCodec{header.element_size}}); - PrintBasicXorDecryptValueListTimings( - true, num_elements, read_header_ns, setup_buffer_ns, - ElapsedNanosecondsSince(stage_start), ElapsedNanosecondsSince(total_start)); - return out; - } default: throw InvalidInputException( std::string("DecryptValueList: unsupported fixed-size datatype: ") @@ -396,10 +305,8 @@ TypedValuesBuffer BasicXorEncryptor::DecryptValueList( // Decrypt variable-size elements else { - stage_start = std::chrono::steady_clock::now(); // Create a variable-sized byte buffer for reading the encrypted elements. TypedBufferRawBytesVariableSized encrypted_buffer{ encrypted_bytes, kVariableHeaderLength}; - auto setup_buffer_ns = ElapsedNanosecondsSince(stage_start); switch (datatype_) { // Create a BYTE-ARRAY typed buffer for storing the decrypted elements. @@ -407,16 +314,11 @@ TypedValuesBuffer BasicXorEncryptor::DecryptValueList( auto reserved_bytes_hint = encrypted_buffer.GetRawBufferSize(); TypedBufferRawBytesVariableSized output_buffer{num_elements, reserved_bytes_hint, true}; size_t output_index = 0; - stage_start = std::chrono::steady_clock::now(); for (const auto element : encrypted_buffer) { - auto decrypted_bytes = XorDecrypt(element); - output_buffer.SetElement(output_index, tcb::span(decrypted_bytes)); + auto write_span = output_buffer.GetWritableRawElement(output_index, element.size()); + XorDecryptInto(element, write_span); output_index++; } - auto decrypt_elements_ns = ElapsedNanosecondsSince(stage_start); - PrintBasicXorDecryptValueListTimings( - false, num_elements, read_header_ns, setup_buffer_ns, - decrypt_elements_ns, ElapsedNanosecondsSince(total_start)); return output_buffer; } default: diff --git a/src/processing/encryptors/basic_xor_encryptor.h b/src/processing/encryptors/basic_xor_encryptor.h index 2e30354..c2f55f0 100644 --- a/src/processing/encryptors/basic_xor_encryptor.h +++ b/src/processing/encryptors/basic_xor_encryptor.h @@ -66,8 +66,8 @@ class DBPS_EXPORT BasicXorEncryptor : public DBPSEncryptor { private: const size_t key_id_hash_; - std::vector XorEncrypt(tcb::span data); - std::vector XorDecrypt(tcb::span data); + void XorEncryptInto(tcb::span data, tcb::span out); + void XorDecryptInto(tcb::span data, tcb::span out); template std::vector EncryptTypedElements(const InputBuffer& input_buffer); diff --git a/src/processing/typed_buffer.h b/src/processing/typed_buffer.h index f0062c2..2750787 100644 --- a/src/processing/typed_buffer.h +++ b/src/processing/typed_buffer.h @@ -67,6 +67,7 @@ class ByteBuffer { // Get and set elements by position with type access from Codec value_type GetElement(size_t position) const; tcb::span GetRawElement(size_t position) const; + tcb::span GetWritableRawElement(size_t position, size_t payload_size); void SetElement(size_t position, const value_type& element); void SetRawElement(size_t position, tcb::span raw); @@ -614,7 +615,7 @@ inline tcb::span ByteBuffer::GetWritableSpanForElement(size_t po if (payload_size != element_size_) { throw InvalidInputException("GetWriteSpanForElement: payload does not match element_size"); } - const size_t offset = CalculateOffsetOfElement(position); + const size_t offset = prefix_size_ + (position * element_size_); auto write_span = tcb::span(write_buffer_.data() + offset, element_size_); return write_span; } @@ -654,6 +655,11 @@ inline tcb::span ByteBuffer::GetWritableSpanForElement(size_t po } } +template +inline tcb::span ByteBuffer::GetWritableRawElement(size_t position, size_t payload_size) { + return GetWritableSpanForElement(position, payload_size); +} + template inline void ByteBuffer::SetElement(size_t position, const value_type& element) { if constexpr (is_fixed_sized) { From 99d8c34fd550750770733edd25d8b817135230da Mon Sep 17 00:00:00 2001 From: Alejandro Valerio Date: Wed, 11 Mar 2026 07:45:04 -0600 Subject: [PATCH 05/18] - Restored one lost comment in the BasicXorEncryptor.cpp --- src/processing/encryptors/basic_xor_encryptor.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/processing/encryptors/basic_xor_encryptor.cpp b/src/processing/encryptors/basic_xor_encryptor.cpp index ec34965..8086b1e 100644 --- a/src/processing/encryptors/basic_xor_encryptor.cpp +++ b/src/processing/encryptors/basic_xor_encryptor.cpp @@ -268,14 +268,18 @@ TypedBuffer BasicXorEncryptor::DecryptFixedSizedElementsIntoTypedBuffer( TypedValuesBuffer BasicXorEncryptor::DecryptValueList( tcb::span encrypted_bytes) { + auto header = ReadHeader(encrypted_bytes); auto num_elements = static_cast(header.num_elements); // Decrypt fixed-size elements if (header.is_fixed) { + // Create a fixed-sized byte buffer for reading the encrypted elements. TypedBufferRawBytesFixedSized encrypted_buffer{ encrypted_bytes, kFixedHeaderLength, RawBytesFixedSizedCodec{header.element_size}}; + + // Populate a typed buffer with the decrypted elements in the corresponding type. switch (datatype_) { case Type::INT32: return DecryptFixedSizedElementsIntoTypedBuffer( From 91b76dba08a8468af18738e19e900bab4fc7d11e Mon Sep 17 00:00:00 2001 From: Alejandro Valerio Date: Wed, 11 Mar 2026 15:20:21 -0600 Subject: [PATCH 06/18] - Adding streamlined iterator for typed buffers. --- src/common/bytes_utils.h | 6 ++ .../encryptors/basic_xor_encryptor.cpp | 40 +++++--- src/processing/typed_buffer.h | 91 ++++++++++++++++--- src/processing/typed_buffer_test.cpp | 76 ++++++++++++++++ 4 files changed, 188 insertions(+), 25 deletions(-) diff --git a/src/common/bytes_utils.h b/src/common/bytes_utils.h index 6b08aeb..a9ddf45 100644 --- a/src/common/bytes_utils.h +++ b/src/common/bytes_utils.h @@ -95,6 +95,12 @@ inline uint32_t read_u32_le(tcb::span in, size_t offset) { (static_cast(in[offset + 3]) << 24); } +inline uint32_t read_u32_le(const uint8_t* p) { + uint32_t v; + std::memcpy(&v, p, sizeof(v)); + return v; +} + // Utility functions for splitting and joining byte vectors. struct BytesPair { diff --git a/src/processing/encryptors/basic_xor_encryptor.cpp b/src/processing/encryptors/basic_xor_encryptor.cpp index 8086b1e..eb191eb 100644 --- a/src/processing/encryptors/basic_xor_encryptor.cpp +++ b/src/processing/encryptors/basic_xor_encryptor.cpp @@ -44,7 +44,7 @@ namespace { } void PrintBasicXorBlockTimings(const char* operation, int64_t total_ns) { - std::cout << "+++++ BasicXorEncryptor timings (microseconds + nanoseconds) +++++" << std::endl; + std::cout << "----- BasicXorEncryptor timings (microseconds + nanoseconds) -----" << std::endl; PrintDurationLine(operation, total_ns); } @@ -59,7 +59,7 @@ namespace { int64_t finalize_buffer_ns, int64_t write_header_ns, int64_t total_ns) { - std::cout << "+++++ BasicXorEncryptor EncryptTypedElements timings (microseconds + nanoseconds) +++++" << std::endl; + std::cout << "----- BasicXorEncryptor EncryptTypedElements timings (microseconds + nanoseconds) -----" << std::endl; std::cout << " mode: " << (is_fixed ? "fixed" : "variable") << std::endl; std::cout << " num_elements: " << num_elements << std::endl; std::cout << " element_size: " << element_size << std::endl; @@ -76,7 +76,7 @@ namespace { } void PrintBasicXorEncryptValueListTimings(int64_t visit_dispatch_ns, int64_t total_ns) { - std::cout << "+++++ BasicXorEncryptor EncryptValueList timings (microseconds + nanoseconds) +++++" << std::endl; + std::cout << "----- BasicXorEncryptor EncryptValueList timings (microseconds + nanoseconds) -----" << std::endl; PrintDurationLine("VariantVisitAndEncrypt", visit_dispatch_ns); PrintDurationLine("EncryptValueList(total)", total_ns); } @@ -88,7 +88,7 @@ namespace { int64_t setup_buffer_ns, int64_t decrypt_elements_ns, int64_t total_ns) { - std::cout << "+++++ BasicXorEncryptor DecryptValueList timings (microseconds + nanoseconds) +++++" << std::endl; + std::cout << "----- BasicXorEncryptor DecryptValueList timings (microseconds + nanoseconds) -----" << std::endl; std::cout << " mode: " << (is_fixed ? "fixed" : "variable") << std::endl; std::cout << " num_elements: " << num_elements << std::endl; PrintDurationLine("ReadHeader", read_header_ns); @@ -173,7 +173,9 @@ std::vector BasicXorEncryptor::EncryptTypedElements( const TypedBuffer& input_buffer) { constexpr bool is_fixed = TypedBuffer::is_fixed_sized; constexpr size_t prefix_length = is_fixed ? kFixedHeaderLength : kVariableHeaderLength; - const size_t num_elements = input_buffer.GetNumElements(); + + const size_t num_elements = 500000; // +++++++++ input_buffer.GetNumElements(); + //const size_t num_elements = input_buffer.GetNumElements(); // If there are no elements, return an empty buffer with the header. if (num_elements == 0) { @@ -195,25 +197,30 @@ std::vector BasicXorEncryptor::EncryptTypedElements( element_size = input_buffer.GetElementSize(); TypedBufferRawBytesFixedSized output_buffer{ num_elements, prefix_length, RawBytesFixedSizedCodec{element_size}}; - for (size_t i = 0; i < num_elements; ++i) { - auto raw_bytes = input_buffer.GetRawElement(i); - auto write_span = output_buffer.GetWritableRawElement(i, raw_bytes.size()); + + size_t output_index = 0; + tcb::span raw_bytes; + while (input_buffer.ElementsIteratorNext(raw_bytes)) { + auto write_span = output_buffer.GetWritableRawElement(output_index, raw_bytes.size()); XorEncryptInto(raw_bytes, write_span); + output_index++; } final_buffer = output_buffer.FinalizeAndTakeBuffer(); - } + } // Encrypt variable-size elements else { auto reserved_bytes_hint = input_buffer.GetRawBufferSize(); TypedBufferRawBytesVariableSized output_buffer{ num_elements, reserved_bytes_hint, true, prefix_length}; + size_t output_index = 0; - for (const auto raw_bytes : input_buffer.raw_elements()) { + tcb::span raw_bytes; + while (input_buffer.ElementsIteratorNext(raw_bytes)) { auto write_span = output_buffer.GetWritableRawElement(output_index, raw_bytes.size()); XorEncryptInto(raw_bytes, write_span); output_index++; - } + } final_buffer = output_buffer.FinalizeAndTakeBuffer(); } @@ -258,7 +265,9 @@ template TypedBuffer BasicXorEncryptor::DecryptFixedSizedElementsIntoTypedBuffer( const TypedBufferRawBytesFixedSized& encrypted_buffer, TypedBuffer output_buffer) { size_t output_index = 0; - for (const auto raw_bytes : encrypted_buffer.raw_elements()) { + tcb::span raw_bytes; + for (auto raw_bytes : encrypted_buffer.raw_elements()) { + // +++++++ while (encrypted_buffer.ElementsIteratorNext(raw_bytes)) { auto write_span = output_buffer.GetWritableRawElement(output_index, raw_bytes.size()); XorDecryptInto(raw_bytes, write_span); output_index++; @@ -317,12 +326,15 @@ TypedValuesBuffer BasicXorEncryptor::DecryptValueList( case Type::BYTE_ARRAY: { auto reserved_bytes_hint = encrypted_buffer.GetRawBufferSize(); TypedBufferRawBytesVariableSized output_buffer{num_elements, reserved_bytes_hint, true}; + size_t output_index = 0; - for (const auto element : encrypted_buffer) { + tcb::span element; + for (auto element : encrypted_buffer.raw_elements()) { + // +++++++ while (encrypted_buffer.ElementsIteratorNext(element)) { auto write_span = output_buffer.GetWritableRawElement(output_index, element.size()); XorDecryptInto(element, write_span); output_index++; - } + } return output_buffer; } default: diff --git a/src/processing/typed_buffer.h b/src/processing/typed_buffer.h index 2750787..f177c73 100644 --- a/src/processing/typed_buffer.h +++ b/src/processing/typed_buffer.h @@ -72,7 +72,7 @@ class ByteBuffer { void SetRawElement(size_t position, tcb::span raw); // Getters for immediately available properties. - size_t GetRawBufferSize() const { return elements_span_.size(); } + size_t GetRawBufferSize() const { return elements_span_size_; } size_t GetElementSize() const { return codec_.element_size(); } // Get the number of elements in the buffer. @@ -81,6 +81,9 @@ class ByteBuffer { // Finalizes the write path and transfers the resulting buffer ownership. std::vector FinalizeAndTakeBuffer(); + // Iterator for read-only elements returning raw bytes. + bool ElementsIteratorNext(tcb::span& out_bytes) const; + // Iterator for read-only elements returning a `value_type` class ConstIterator { public: @@ -146,9 +149,14 @@ class ByteBuffer { // Variables for span elements reading tcb::span elements_span_; + size_t elements_span_size_; mutable size_t num_elements_; Codec codec_; - + + // Variables for element span iterator. + mutable const uint8_t* element_iterator_current_ptr_; + const uint8_t* element_iterator_end_ptr_; + // Variables for determining offset of elements. size_t prefix_size_ = 0; size_t element_size_; // for fixed-size elements @@ -196,6 +204,9 @@ ByteBuffer::ByteBuffer( size_t prefix_size, Codec codec) : elements_span_(elements_span), + elements_span_size_(elements_span.size()), + element_iterator_current_ptr_(elements_span.data()), + element_iterator_end_ptr_(elements_span.data() + elements_span.size()), num_elements_(kUnsetSize), codec_(std::move(codec)), element_size_(0), @@ -210,11 +221,11 @@ ByteBuffer::ByteBuffer( // Called in a lazy manner when the buffer is accessed with GetElement or GetNumElements, avoiding unnecessary initialization. template inline void ByteBuffer::InitializeFromSpan() const { - if (elements_span_.size() < prefix_size_) { + if (elements_span_size_ < prefix_size_) { throw InvalidInputException("Malformed buffer: prefix_size exceeds span size"); } - const size_t readable_size = elements_span_.size() - prefix_size_; + const size_t readable_size = elements_span_size_ - prefix_size_; // No elements to index. Initialize with empty values. if (readable_size == 0) { @@ -249,14 +260,14 @@ inline void ByteBuffer::InitializeFromSpan() const { offsets_.clear(); offsets_.reserve(EstimateOffsetsReserveCountFromSample(elements_span_.subspan(prefix_size_))); size_t cursor = prefix_size_; - while (cursor < elements_span_.size()) { - if (elements_span_.size() - cursor < kSizePrefixBytes) { + while (cursor < elements_span_size_) { + if (elements_span_size_ - cursor < kSizePrefixBytes) { throw InvalidInputException("Malformed variable-size buffer: truncated length prefix"); } offsets_.push_back(cursor); const size_t current_element_size = ReadSizeAt(elements_span_, cursor); cursor += kSizePrefixBytes; - if (elements_span_.size() - cursor < current_element_size) { + if (elements_span_size_ - cursor < current_element_size) { throw InvalidInputException("Malformed variable-size buffer: truncated element payload"); } cursor += current_element_size; @@ -380,6 +391,64 @@ inline typename ByteBuffer::value_type ByteBuffer::GetElement(size return codec_.Decode(GetRawElement(position)); } + + + + + + + + +// ----------------------------------------------------------------------------- +// Element span iterator -- The streamlined version of the iterator. +// ----------------------------------------------------------------------------- + +template +inline bool ByteBuffer::ElementsIteratorNext(tcb::span& out_bytes) const { + if (element_iterator_current_ptr_ >= element_iterator_end_ptr_) { + out_bytes = {}; + return false; + } + + const size_t bytes_remaining = + static_cast(element_iterator_end_ptr_ - element_iterator_current_ptr_); + + if constexpr (is_fixed_sized) { + if (bytes_remaining < element_size_) { + throw InvalidInputException("Malformed fixed-size buffer: truncated element in iterator"); + } + out_bytes = tcb::span(element_iterator_current_ptr_, element_size_); + element_iterator_current_ptr_ += element_size_; + return true; + } + + // Variable-sized elements + if (bytes_remaining < kSizePrefixBytes) { + throw InvalidInputException("Malformed variable-size buffer: truncated length prefix in iterator"); + } + const size_t current_element_size = read_u32_le(element_iterator_current_ptr_); + element_iterator_current_ptr_ += kSizePrefixBytes; + + const size_t payload_remaining = + static_cast(element_iterator_end_ptr_ - element_iterator_current_ptr_); + if (payload_remaining < current_element_size) { + throw InvalidInputException("Malformed variable-size buffer: truncated element payload in iterator"); + } + out_bytes = tcb::span(element_iterator_current_ptr_, current_element_size); + element_iterator_current_ptr_ += current_element_size; + return true; +} + + + + + + + + + + + // ----------------------------------------------------------------------------- // Element span iterator // @@ -392,7 +461,7 @@ template inline ByteBuffer::ConstIterator::ConstIterator(const ByteBuffer* buffer, size_t cursor_offset) : buffer_(buffer), cursor_offset_(cursor_offset), - elements_span_size_(buffer != nullptr ? buffer->elements_span_.size() : 0u), + elements_span_size_(buffer != nullptr ? buffer->elements_span_size_ : 0u), current_element_size_(kUnsetSize) {} template @@ -476,14 +545,14 @@ inline void ByteBuffer::ValidateIteratorReadPreconditions() const { if (is_write_buffer_initialized_) { throw InvalidInputException("Iterator is only available for read buffers"); } - if (elements_span_.size() < prefix_size_) { + if (elements_span_size_ < prefix_size_) { throw InvalidInputException("Malformed buffer: prefix_size exceeds span size"); } if constexpr (is_fixed_sized) { if (element_size_ <= 0) { throw InvalidInputException("Invalid fixed-size buffer: element_size must be greater than zero"); } - const size_t readable_size = elements_span_.size() - prefix_size_; + const size_t readable_size = elements_span_size_ - prefix_size_; if ((readable_size % element_size_) != 0) { throw InvalidInputException("Malformed fixed-size buffer: buffer does not align with element_size"); } @@ -499,7 +568,7 @@ inline typename ByteBuffer::ConstIterator ByteBuffer::begin() cons template inline typename ByteBuffer::ConstIterator ByteBuffer::end() const { ValidateIteratorReadPreconditions(); - return ConstIterator(this, elements_span_.size()); + return ConstIterator(this, elements_span_size_); } template diff --git a/src/processing/typed_buffer_test.cpp b/src/processing/typed_buffer_test.cpp index 061299e..5c19718 100644 --- a/src/processing/typed_buffer_test.cpp +++ b/src/processing/typed_buffer_test.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -948,6 +949,81 @@ TEST(TypedBufferTest, Iterate_OnWriteBuffer_Throws) { EXPECT_THROW((void)buffer.end(), InvalidInputException); } +// ----------------------------------------------------------------------------- +// NextRawElementIterator tests +// ----------------------------------------------------------------------------- + +TEST(TypedBufferTest, NextRawElementIterator_FixedSize_TraversesAllElements) { + constexpr size_t kElementSize = 8u; + const std::vector> expected = { + {0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17}, + {0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27}, + {0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37}, + {0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47}, + {0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57}, + }; + + std::vector bytes; + for (const auto& e : expected) { + bytes.insert(bytes.end(), e.begin(), e.end()); + } + + RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 0, RawBytesFixedSizedCodec{kElementSize}); + + std::vector> collected; + tcb::span element; + while (buffer.NextRawElementIterator(element)) { + collected.push_back(std::vector(element.begin(), element.end())); + } + + ASSERT_EQ(collected.size(), expected.size()); + for (size_t i = 0; i < expected.size(); ++i) { + EXPECT_EQ(collected[i], expected[i]); + } + + EXPECT_TRUE(element.empty()); + + tcb::span extra; + EXPECT_FALSE(buffer.NextRawElementIterator(extra)); + EXPECT_TRUE(extra.empty()); +} + +TEST(TypedBufferTest, NextRawElementIterator_VariableSize_TraversesAllElements) { + const std::vector> expected = { + MakePayload(11, 0x10), + MakePayload(3, 0x20), + MakePayload(27, 0x30), + MakePayload(1, 0x40), + MakePayload(16, 0x50), + MakePayload(8, 0x60), + }; + + std::vector bytes; + for (const auto& e : expected) { + append_u32_le(bytes, static_cast(e.size())); + bytes.insert(bytes.end(), e.begin(), e.end()); + } + + RawBytesVariableSizedBuffer buffer{tcb::span(bytes)}; + + std::vector> collected; + tcb::span element; + while (buffer.NextRawElementIterator(element)) { + collected.push_back(std::vector(element.begin(), element.end())); + } + + ASSERT_EQ(collected.size(), expected.size()); + for (size_t i = 0; i < expected.size(); ++i) { + EXPECT_EQ(collected[i], expected[i]); + } + + EXPECT_TRUE(element.empty()); + + tcb::span extra; + EXPECT_FALSE(buffer.NextRawElementIterator(extra)); + EXPECT_TRUE(extra.empty()); +} + // ----------------------------------------------------------------------------- // GetSpanSize tests // ----------------------------------------------------------------------------- From fb0566fee97646e84dea7905e3697d935eb73807 Mon Sep 17 00:00:00 2001 From: Alejandro Valerio Date: Wed, 11 Mar 2026 15:46:55 -0600 Subject: [PATCH 07/18] - Fixing unittests for streamlined iterator. --- src/processing/typed_buffer_test.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/processing/typed_buffer_test.cpp b/src/processing/typed_buffer_test.cpp index 5c19718..bea88a8 100644 --- a/src/processing/typed_buffer_test.cpp +++ b/src/processing/typed_buffer_test.cpp @@ -953,7 +953,7 @@ TEST(TypedBufferTest, Iterate_OnWriteBuffer_Throws) { // NextRawElementIterator tests // ----------------------------------------------------------------------------- -TEST(TypedBufferTest, NextRawElementIterator_FixedSize_TraversesAllElements) { +TEST(TypedBufferTest, ElementsIteratorNext_FixedSize_TraversesAllElements) { constexpr size_t kElementSize = 8u; const std::vector> expected = { {0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17}, @@ -972,7 +972,7 @@ TEST(TypedBufferTest, NextRawElementIterator_FixedSize_TraversesAllElements) { std::vector> collected; tcb::span element; - while (buffer.NextRawElementIterator(element)) { + while (buffer.ElementsIteratorNext(element)) { collected.push_back(std::vector(element.begin(), element.end())); } @@ -984,11 +984,11 @@ TEST(TypedBufferTest, NextRawElementIterator_FixedSize_TraversesAllElements) { EXPECT_TRUE(element.empty()); tcb::span extra; - EXPECT_FALSE(buffer.NextRawElementIterator(extra)); + EXPECT_FALSE(buffer.ElementsIteratorNext(extra)); EXPECT_TRUE(extra.empty()); } -TEST(TypedBufferTest, NextRawElementIterator_VariableSize_TraversesAllElements) { +TEST(TypedBufferTest, ElementsIteratorNext_VariableSize_TraversesAllElements) { const std::vector> expected = { MakePayload(11, 0x10), MakePayload(3, 0x20), @@ -1008,7 +1008,7 @@ TEST(TypedBufferTest, NextRawElementIterator_VariableSize_TraversesAllElements) std::vector> collected; tcb::span element; - while (buffer.NextRawElementIterator(element)) { + while (buffer.ElementsIteratorNext(element)) { collected.push_back(std::vector(element.begin(), element.end())); } @@ -1020,7 +1020,7 @@ TEST(TypedBufferTest, NextRawElementIterator_VariableSize_TraversesAllElements) EXPECT_TRUE(element.empty()); tcb::span extra; - EXPECT_FALSE(buffer.NextRawElementIterator(extra)); + EXPECT_FALSE(buffer.ElementsIteratorNext(extra)); EXPECT_TRUE(extra.empty()); } From 971903a4e3c3166c78341f5eca12b5313a98ac9e Mon Sep 17 00:00:00 2001 From: Alejandro Valerio Date: Wed, 11 Mar 2026 21:58:26 -0600 Subject: [PATCH 08/18] - Optimizing GetWritableRawElement for variable-size elements. - small performance improvements. --- src/common/bytes_utils.h | 7 ++ .../encryptors/basic_xor_encryptor.cpp | 107 +++++++++++++++--- src/processing/typed_buffer.h | 15 +-- 3 files changed, 107 insertions(+), 22 deletions(-) diff --git a/src/common/bytes_utils.h b/src/common/bytes_utils.h index a9ddf45..3095151 100644 --- a/src/common/bytes_utils.h +++ b/src/common/bytes_utils.h @@ -81,6 +81,13 @@ inline void write_u32_le_at(std::vector& buf, size_t offset, uint32_t v buf[offset + 3] = static_cast((v >> 24) & 0xFF); } +inline void write_u32_le(uint8_t* p, uint32_t v) { + p[0] = static_cast(v); + p[1] = static_cast(v >> 8); + p[2] = static_cast(v >> 16); + p[3] = static_cast(v >> 24); +} + inline uint32_t read_u32_le(const std::vector& in, size_t offset) { return static_cast(in[offset]) | (static_cast(in[offset + 1]) << 8) | diff --git a/src/processing/encryptors/basic_xor_encryptor.cpp b/src/processing/encryptors/basic_xor_encryptor.cpp index eb191eb..245efa9 100644 --- a/src/processing/encryptors/basic_xor_encryptor.cpp +++ b/src/processing/encryptors/basic_xor_encryptor.cpp @@ -21,6 +21,7 @@ #include "../../common/enum_utils.h" #include #include +#include #include #include @@ -48,28 +49,44 @@ namespace { PrintDurationLine(operation, total_ns); } + void PrintDurationLineWithPercent(const char* label, int64_t nanoseconds, int64_t total_ns) { + double pct = total_ns > 0 + ? 100.0 * static_cast(nanoseconds) / static_cast(total_ns) + : 0.0; + std::cout << " " << label << ": " + << ToMicroseconds(nanoseconds) << " us" + << " (" << nanoseconds << " ns)" + << " " << std::fixed << std::setprecision(2) << pct << "%" + << std::endl; + } + void PrintBasicXorEncryptTypedElementsTimings( bool is_fixed, size_t num_elements, size_t element_size, int64_t encrypt_elements_loop_ns, int64_t get_raw_element_ns, + int64_t get_writable_element_ns, int64_t xor_encrypt_ns, - int64_t set_element_ns, int64_t finalize_buffer_ns, int64_t write_header_ns, int64_t total_ns) { - std::cout << "----- BasicXorEncryptor EncryptTypedElements timings (microseconds + nanoseconds) -----" << std::endl; + const int64_t loop_residual_ns = encrypt_elements_loop_ns + - get_raw_element_ns - get_writable_element_ns - xor_encrypt_ns; + const int64_t accounted_sum_ns = get_raw_element_ns + + get_writable_element_ns + xor_encrypt_ns + loop_residual_ns; + + std::cout << "----- BasicXorEncryptor EncryptTypedElements timings -----" << std::endl; std::cout << " mode: " << (is_fixed ? "fixed" : "variable") << std::endl; std::cout << " num_elements: " << num_elements << std::endl; std::cout << " element_size: " << element_size << std::endl; PrintDurationLine("EncryptElementsLoop", encrypt_elements_loop_ns); - PrintDurationLine("GetRawElement(aggregated)", get_raw_element_ns); - PrintDurationLine("XorEncrypt(aggregated)", xor_encrypt_ns); - PrintDurationLine("SetElement(aggregated)", set_element_ns); - PrintDurationLine( - "LoopResidual(iterator+other)", - encrypt_elements_loop_ns - get_raw_element_ns - xor_encrypt_ns - set_element_ns); + std::cout << " Decomposition of EncryptElementsLoop:" << std::endl; + PrintDurationLineWithPercent("GetRawElement(aggregated)", get_raw_element_ns, encrypt_elements_loop_ns); + PrintDurationLineWithPercent("GetWritableRawElement(aggreg'd)", get_writable_element_ns, encrypt_elements_loop_ns); + PrintDurationLineWithPercent("XorEncryptInto(aggregated)", xor_encrypt_ns, encrypt_elements_loop_ns); + PrintDurationLineWithPercent("LoopResidual(iterator+other)", loop_residual_ns, encrypt_elements_loop_ns); + PrintDurationLineWithPercent("AccountedSum", accounted_sum_ns, encrypt_elements_loop_ns); PrintDurationLine("FinalizeBuffer", finalize_buffer_ns); PrintDurationLine("WriteHeader", write_header_ns); PrintDurationLine("EncryptTypedElements(total)", total_ns); @@ -103,12 +120,17 @@ namespace { // --------------------------------------------------------------------------- void BasicXorEncryptor::XorEncryptInto(tcb::span data, tcb::span out) { - if (data.size() != out.size()) { + size_t data_size = data.size(); + size_t out_size = out.size(); + if (data_size != out_size) { throw InvalidInputException("XorEncryptInto: input and output sizes must match"); } + const size_t n = data_size; + const uint8_t* src = data.data(); + uint8_t* dst = out.data(); size_t key_hash = key_id_hash_; - for (size_t i = 0; i < data.size(); ++i) { - out[i] = data[i] ^ (key_hash & 0xFF); + for (size_t i = 0; i < n; ++i) { + dst[i] = src[i] ^ (key_hash & 0xFF); key_hash = (key_hash << 1) | (key_hash >> 31); } } @@ -174,8 +196,8 @@ std::vector BasicXorEncryptor::EncryptTypedElements( constexpr bool is_fixed = TypedBuffer::is_fixed_sized; constexpr size_t prefix_length = is_fixed ? kFixedHeaderLength : kVariableHeaderLength; - const size_t num_elements = 500000; // +++++++++ input_buffer.GetNumElements(); - //const size_t num_elements = input_buffer.GetNumElements(); + // const size_t num_elements = 500000; // +++++++++ input_buffer.GetNumElements(); + const size_t num_elements = input_buffer.GetNumElements(); // If there are no elements, return an empty buffer with the header. if (num_elements == 0) { @@ -189,8 +211,16 @@ std::vector BasicXorEncryptor::EncryptTypedElements( // - For each element: read its raw bytes, encrypt them, write into the output buffer. // - Finalize the output buffer into a contiguous byte vector. + auto total_start = std::chrono::steady_clock::now(); + std::vector final_buffer; size_t element_size = 0; + int64_t get_raw_element_ns = 0; + int64_t get_writable_element_ns = 0; + int64_t xor_encrypt_ns = 0; + int64_t encrypt_elements_loop_ns = 0; + int64_t finalize_buffer_ns = 0; + int64_t write_header_ns = 0; // Encrypt fixed-size elements if constexpr (is_fixed) { @@ -198,14 +228,29 @@ std::vector BasicXorEncryptor::EncryptTypedElements( TypedBufferRawBytesFixedSized output_buffer{ num_elements, prefix_length, RawBytesFixedSizedCodec{element_size}}; + auto loop_start = std::chrono::steady_clock::now(); size_t output_index = 0; tcb::span raw_bytes; - while (input_buffer.ElementsIteratorNext(raw_bytes)) { + while (true) { + // auto t0 = std::chrono::steady_clock::now(); + if (!input_buffer.ElementsIteratorNext(raw_bytes)) break; + // get_raw_element_ns += ElapsedNanosecondsSince(t0); + + // auto t1 = std::chrono::steady_clock::now(); auto write_span = output_buffer.GetWritableRawElement(output_index, raw_bytes.size()); + // get_writable_element_ns += ElapsedNanosecondsSince(t1); + + // auto t2 = std::chrono::steady_clock::now(); XorEncryptInto(raw_bytes, write_span); + // xor_encrypt_ns += ElapsedNanosecondsSince(t2); + output_index++; } + encrypt_elements_loop_ns = ElapsedNanosecondsSince(loop_start); + + auto finalize_start = std::chrono::steady_clock::now(); final_buffer = output_buffer.FinalizeAndTakeBuffer(); + finalize_buffer_ns = ElapsedNanosecondsSince(finalize_start); } // Encrypt variable-size elements @@ -214,20 +259,50 @@ std::vector BasicXorEncryptor::EncryptTypedElements( TypedBufferRawBytesVariableSized output_buffer{ num_elements, reserved_bytes_hint, true, prefix_length}; + auto loop_start = std::chrono::steady_clock::now(); size_t output_index = 0; tcb::span raw_bytes; - while (input_buffer.ElementsIteratorNext(raw_bytes)) { + while (true) { + // auto t0 = std::chrono::steady_clock::now(); + if (!input_buffer.ElementsIteratorNext(raw_bytes)) break; + // get_raw_element_ns += ElapsedNanosecondsSince(t0); + + // auto t1 = std::chrono::steady_clock::now(); auto write_span = output_buffer.GetWritableRawElement(output_index, raw_bytes.size()); + // get_writable_element_ns += ElapsedNanosecondsSince(t1); + + // auto t2 = std::chrono::steady_clock::now(); XorEncryptInto(raw_bytes, write_span); + // xor_encrypt_ns += ElapsedNanosecondsSince(t2); + output_index++; } + encrypt_elements_loop_ns = ElapsedNanosecondsSince(loop_start); + + auto finalize_start = std::chrono::steady_clock::now(); final_buffer = output_buffer.FinalizeAndTakeBuffer(); + finalize_buffer_ns = ElapsedNanosecondsSince(finalize_start); } // Write the header to the final buffer and return it. + auto header_start = std::chrono::steady_clock::now(); WriteHeader(final_buffer, {is_fixed, static_cast(num_elements), static_cast(element_size)}); + write_header_ns = ElapsedNanosecondsSince(header_start); + + int64_t total_ns = ElapsedNanosecondsSince(total_start); + + PrintBasicXorEncryptTypedElementsTimings( + is_fixed, num_elements, element_size, + encrypt_elements_loop_ns, + get_raw_element_ns, + get_writable_element_ns, + xor_encrypt_ns, + finalize_buffer_ns, + write_header_ns, + total_ns); + return final_buffer; } @@ -267,6 +342,7 @@ TypedBuffer BasicXorEncryptor::DecryptFixedSizedElementsIntoTypedBuffer( size_t output_index = 0; tcb::span raw_bytes; for (auto raw_bytes : encrypted_buffer.raw_elements()) { + // +++++++ new iterator not working? // +++++++ while (encrypted_buffer.ElementsIteratorNext(raw_bytes)) { auto write_span = output_buffer.GetWritableRawElement(output_index, raw_bytes.size()); XorDecryptInto(raw_bytes, write_span); @@ -330,6 +406,7 @@ TypedValuesBuffer BasicXorEncryptor::DecryptValueList( size_t output_index = 0; tcb::span element; for (auto element : encrypted_buffer.raw_elements()) { + // +++++++ new iterator not working? // +++++++ while (encrypted_buffer.ElementsIteratorNext(element)) { auto write_span = output_buffer.GetWritableRawElement(output_index, element.size()); XorDecryptInto(element, write_span); diff --git a/src/processing/typed_buffer.h b/src/processing/typed_buffer.h index f177c73..f775567 100644 --- a/src/processing/typed_buffer.h +++ b/src/processing/typed_buffer.h @@ -692,7 +692,7 @@ inline tcb::span ByteBuffer::GetWritableSpanForElement(size_t po // Variable-sized elements - `else` is needed because it's a compile-time check. else { // Defensive check for unlikely extremely large element size that exceeds uint32. - if (payload_size > static_cast(std::numeric_limits::max())) { + if (payload_size > static_cast(std::numeric_limits::max())) [[unlikely]] { throw InvalidInputException("Variable-size element payload exceeds uint32 capacity.. Woohhh!!"); } @@ -704,11 +704,11 @@ inline tcb::span ByteBuffer::GetWritableSpanForElement(size_t po // This is intentional to allow random writes of elements while the buffer is built. // During FinalizeAndTakeBuffer, the buffer is rebuilt to be sequential and orphaned bytes are removed. const size_t offset = write_buffer_.size(); - offsets_[position] = offset; - append_u32_le(write_buffer_, static_cast(payload_size)); - const size_t payload_offset = write_buffer_.size(); - write_buffer_.resize(payload_offset + payload_size); - auto write_span = tcb::span(write_buffer_.data() + payload_offset, payload_size); + write_buffer_.resize(offset + kSizePrefixBytes + payload_size); + auto offset_ptr = write_buffer_.data() + offset; + + // Write the size prefix + write_u32_le(offset_ptr, static_cast(payload_size)); // Update next_expected_write_position_ for sequential write checking. if (next_expected_write_position_ != kUnsetSize) { @@ -720,7 +720,8 @@ inline tcb::span ByteBuffer::GetWritableSpanForElement(size_t po } RebindSpanToWriteBuffer(); - return write_span; + offsets_[position] = offset; + return tcb::span(offset_ptr + kSizePrefixBytes, payload_size);; } } From 097d919b16fbdb66e86aa6ba5720ab27dbdd8669 Mon Sep 17 00:00:00 2001 From: Alejandro Valerio Date: Thu, 12 Mar 2026 10:46:09 -0600 Subject: [PATCH 09/18] - Pushing small cleanups before pushing the Parquet-based num_elements optimization. --- .../encryptors/basic_xor_encryptor.cpp | 16 ++++--- src/processing/typed_buffer.h | 43 +++++++++++-------- 2 files changed, 35 insertions(+), 24 deletions(-) diff --git a/src/processing/encryptors/basic_xor_encryptor.cpp b/src/processing/encryptors/basic_xor_encryptor.cpp index 245efa9..185bc49 100644 --- a/src/processing/encryptors/basic_xor_encryptor.cpp +++ b/src/processing/encryptors/basic_xor_encryptor.cpp @@ -196,8 +196,8 @@ std::vector BasicXorEncryptor::EncryptTypedElements( constexpr bool is_fixed = TypedBuffer::is_fixed_sized; constexpr size_t prefix_length = is_fixed ? kFixedHeaderLength : kVariableHeaderLength; - // const size_t num_elements = 500000; // +++++++++ input_buffer.GetNumElements(); - const size_t num_elements = input_buffer.GetNumElements(); + const size_t num_elements = 500000; // +++++++++ input_buffer.GetNumElements(); + // const size_t num_elements = input_buffer.GetNumElements(); // If there are no elements, return an empty buffer with the header. if (num_elements == 0) { @@ -230,10 +230,12 @@ std::vector BasicXorEncryptor::EncryptTypedElements( auto loop_start = std::chrono::steady_clock::now(); size_t output_index = 0; - tcb::span raw_bytes; + + // ++++++ Restore the while (auto rawbytes = input_buffer.ElementsIteratorNext()) {} form. while (true) { // auto t0 = std::chrono::steady_clock::now(); - if (!input_buffer.ElementsIteratorNext(raw_bytes)) break; + const auto raw_bytes = input_buffer.ElementsIteratorNext(); + if (raw_bytes.empty()) break; // get_raw_element_ns += ElapsedNanosecondsSince(t0); // auto t1 = std::chrono::steady_clock::now(); @@ -261,10 +263,12 @@ std::vector BasicXorEncryptor::EncryptTypedElements( auto loop_start = std::chrono::steady_clock::now(); size_t output_index = 0; - tcb::span raw_bytes; + + // ++++++ Restore the while (auto rawbytes = input_buffer.ElementsIteratorNext()) {} form. while (true) { // auto t0 = std::chrono::steady_clock::now(); - if (!input_buffer.ElementsIteratorNext(raw_bytes)) break; + const auto raw_bytes = input_buffer.ElementsIteratorNext(); + if (raw_bytes.empty()) break; // get_raw_element_ns += ElapsedNanosecondsSince(t0); // auto t1 = std::chrono::steady_clock::now(); diff --git a/src/processing/typed_buffer.h b/src/processing/typed_buffer.h index f775567..49cc4f4 100644 --- a/src/processing/typed_buffer.h +++ b/src/processing/typed_buffer.h @@ -82,7 +82,7 @@ class ByteBuffer { std::vector FinalizeAndTakeBuffer(); // Iterator for read-only elements returning raw bytes. - bool ElementsIteratorNext(tcb::span& out_bytes) const; + tcb::span ElementsIteratorNext() const; // Iterator for read-only elements returning a `value_type` class ConstIterator { @@ -141,6 +141,7 @@ class ByteBuffer { // Helper for calculating the offset of an element by position. size_t CalculateOffsetOfElement(size_t position) const; + // ++++ Needed after deprecating std iterators? // Helper to validate the preconditions for reading the buffer with an iterator. void ValidateIteratorReadPreconditions() const; @@ -179,7 +180,7 @@ class ByteBuffer { // Initialization methods and flags for write buffer void InitializeForWriteBuffer(size_t variable_size_reserved_bytes_hint); void RebindSpanToWriteBuffer(); - bool is_write_buffer_initialized_ = false; + bool is_write_buffer_enabled_ = false; bool is_write_buffer_finalized_ = false; }; @@ -211,6 +212,7 @@ ByteBuffer::ByteBuffer( codec_(std::move(codec)), element_size_(0), prefix_size_(prefix_size), + is_write_buffer_enabled_(false), is_initialized_from_span_(false) { if constexpr (is_fixed_sized) { element_size_ = codec_.element_size(); @@ -283,7 +285,7 @@ inline void ByteBuffer::EnsureInitializedFromSpan() const { return; } // If the write buffer is initialized, we don't need to initialize from the span. - if (is_write_buffer_initialized_) { + if (is_write_buffer_enabled_) { return; } InitializeFromSpan(); @@ -403,11 +405,11 @@ inline typename ByteBuffer::value_type ByteBuffer::GetElement(size // Element span iterator -- The streamlined version of the iterator. // ----------------------------------------------------------------------------- +// ++++++ Additional validation that this is only used for read-only buffers. template -inline bool ByteBuffer::ElementsIteratorNext(tcb::span& out_bytes) const { - if (element_iterator_current_ptr_ >= element_iterator_end_ptr_) { - out_bytes = {}; - return false; +inline tcb::span ByteBuffer::ElementsIteratorNext() const { + if (element_iterator_current_ptr_ == element_iterator_end_ptr_) { + return {}; } const size_t bytes_remaining = @@ -417,9 +419,10 @@ inline bool ByteBuffer::ElementsIteratorNext(tcb::span& ou if (bytes_remaining < element_size_) { throw InvalidInputException("Malformed fixed-size buffer: truncated element in iterator"); } - out_bytes = tcb::span(element_iterator_current_ptr_, element_size_); + const auto out_bytes = + tcb::span(element_iterator_current_ptr_, element_size_); element_iterator_current_ptr_ += element_size_; - return true; + return out_bytes; } // Variable-sized elements @@ -434,9 +437,11 @@ inline bool ByteBuffer::ElementsIteratorNext(tcb::span& ou if (payload_remaining < current_element_size) { throw InvalidInputException("Malformed variable-size buffer: truncated element payload in iterator"); } - out_bytes = tcb::span(element_iterator_current_ptr_, current_element_size); + + const auto out_bytes = + tcb::span(element_iterator_current_ptr_, current_element_size); element_iterator_current_ptr_ += current_element_size; - return true; + return out_bytes; } @@ -542,7 +547,7 @@ inline tcb::span ByteBuffer::ConstRawIterator::operator*() template inline void ByteBuffer::ValidateIteratorReadPreconditions() const { - if (is_write_buffer_initialized_) { + if (is_write_buffer_enabled_) { throw InvalidInputException("Iterator is only available for read buffers"); } if (elements_span_size_ < prefix_size_) { @@ -590,6 +595,7 @@ ByteBuffer::ByteBuffer( : num_elements_(num_elements), codec_(std::move(codec)), element_size_(0), + is_write_buffer_enabled_(true), prefix_size_(prefix_size) { static_assert(is_fixed_sized, "ByteBuffer constructor for fixed-size elements only."); element_size_ = codec_.element_size(); @@ -607,6 +613,7 @@ ByteBuffer::ByteBuffer( : num_elements_(num_elements), codec_(std::move(codec)), element_size_(0), + is_write_buffer_enabled_(true), prefix_size_(prefix_size) { static_assert(!is_fixed_sized, "ByteBuffer constructor for variable-size elements only."); InitializeForWriteBuffer(use_reserve_hint ? reserved_bytes_hint : 0); @@ -626,7 +633,6 @@ inline void ByteBuffer::InitializeForWriteBuffer(size_t variable_size_res const size_t fixed_size_total_bytes = prefix_size_ + (num_elements_ * element_size_); write_buffer_.clear(); write_buffer_.resize(fixed_size_total_bytes, static_cast(0)); - is_write_buffer_initialized_ = true; // offsets_ are not used for fixed-size elements. offsets_.clear(); @@ -647,7 +653,6 @@ inline void ByteBuffer::InitializeForWriteBuffer(size_t variable_size_res write_buffer_.clear(); write_buffer_.resize(prefix_size_, static_cast(0)); write_buffer_.reserve(variable_size_reserved_bytes); - is_write_buffer_initialized_ = true; // offsets_ is initialized so the vector is fully allocated and have random-ish access during writes. offsets_.clear(); @@ -667,7 +672,7 @@ inline void ByteBuffer::InitializeForWriteBuffer(size_t variable_size_res template inline tcb::span ByteBuffer::GetWritableSpanForElement(size_t position, size_t payload_size) { - if (!is_write_buffer_initialized_) { + if (!is_write_buffer_enabled_) { throw InvalidInputException("Cannot GetWriteSpanForElement: write buffer is not initialized."); } @@ -753,7 +758,7 @@ inline std::vector ByteBuffer::FinalizeAndTakeBuffer() { throw InvalidInputException("FinalizeAndTakeBuffer: write buffer has already been finalized"); } - if (!is_write_buffer_initialized_) { + if (!is_write_buffer_enabled_) { throw InvalidInputException("FinalizeAndTakeBuffer: write buffer is not initialized"); } @@ -812,7 +817,7 @@ inline std::vector ByteBuffer::FinalizeAndTakeBuffer() { // Defrag path returns a new buffer; release the original fragmented write buffer. write_buffer_.clear(); write_buffer_.shrink_to_fit(); - is_write_buffer_initialized_ = false; + is_write_buffer_enabled_ = false; is_write_buffer_finalized_ = true; return result; @@ -820,7 +825,9 @@ inline std::vector ByteBuffer::FinalizeAndTakeBuffer() { template inline void ByteBuffer::RebindSpanToWriteBuffer() { - elements_span_ = tcb::span(write_buffer_.data(), write_buffer_.size()); + auto write_buffer_size = write_buffer_.size(); + elements_span_ = tcb::span(write_buffer_.data(), write_buffer_size); + elements_span_size_ = write_buffer_size; } } // namespace dbps::processing From 6d8944d8838050a186d26fa2c9fb619c2174aff4 Mon Sep 17 00:00:00 2001 From: Alejandro Valerio Date: Thu, 12 Mar 2026 11:51:37 -0600 Subject: [PATCH 10/18] - Fixing issue with empty strings on iterator. --- .../encryptors/basic_xor_encryptor.cpp | 14 +++++++------- src/processing/typed_buffer.h | 17 ++++++++--------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/processing/encryptors/basic_xor_encryptor.cpp b/src/processing/encryptors/basic_xor_encryptor.cpp index 185bc49..ca8e932 100644 --- a/src/processing/encryptors/basic_xor_encryptor.cpp +++ b/src/processing/encryptors/basic_xor_encryptor.cpp @@ -196,8 +196,8 @@ std::vector BasicXorEncryptor::EncryptTypedElements( constexpr bool is_fixed = TypedBuffer::is_fixed_sized; constexpr size_t prefix_length = is_fixed ? kFixedHeaderLength : kVariableHeaderLength; - const size_t num_elements = 500000; // +++++++++ input_buffer.GetNumElements(); - // const size_t num_elements = input_buffer.GetNumElements(); + // const size_t num_elements = 500000; // +++++++++ input_buffer.GetNumElements(); + const size_t num_elements = input_buffer.GetNumElements(); // If there are no elements, return an empty buffer with the header. if (num_elements == 0) { @@ -234,8 +234,8 @@ std::vector BasicXorEncryptor::EncryptTypedElements( // ++++++ Restore the while (auto rawbytes = input_buffer.ElementsIteratorNext()) {} form. while (true) { // auto t0 = std::chrono::steady_clock::now(); - const auto raw_bytes = input_buffer.ElementsIteratorNext(); - if (raw_bytes.empty()) break; + tcb::span raw_bytes; + if (!input_buffer.ElementsIteratorNext(raw_bytes)) break; // get_raw_element_ns += ElapsedNanosecondsSince(t0); // auto t1 = std::chrono::steady_clock::now(); @@ -267,8 +267,8 @@ std::vector BasicXorEncryptor::EncryptTypedElements( // ++++++ Restore the while (auto rawbytes = input_buffer.ElementsIteratorNext()) {} form. while (true) { // auto t0 = std::chrono::steady_clock::now(); - const auto raw_bytes = input_buffer.ElementsIteratorNext(); - if (raw_bytes.empty()) break; + tcb::span raw_bytes; + if (!input_buffer.ElementsIteratorNext(raw_bytes)) break; // get_raw_element_ns += ElapsedNanosecondsSince(t0); // auto t1 = std::chrono::steady_clock::now(); @@ -280,7 +280,7 @@ std::vector BasicXorEncryptor::EncryptTypedElements( // xor_encrypt_ns += ElapsedNanosecondsSince(t2); output_index++; - } + } encrypt_elements_loop_ns = ElapsedNanosecondsSince(loop_start); auto finalize_start = std::chrono::steady_clock::now(); diff --git a/src/processing/typed_buffer.h b/src/processing/typed_buffer.h index 49cc4f4..e74f71e 100644 --- a/src/processing/typed_buffer.h +++ b/src/processing/typed_buffer.h @@ -82,7 +82,7 @@ class ByteBuffer { std::vector FinalizeAndTakeBuffer(); // Iterator for read-only elements returning raw bytes. - tcb::span ElementsIteratorNext() const; + bool ElementsIteratorNext(tcb::span& raw_bytes) const; // Iterator for read-only elements returning a `value_type` class ConstIterator { @@ -407,9 +407,10 @@ inline typename ByteBuffer::value_type ByteBuffer::GetElement(size // ++++++ Additional validation that this is only used for read-only buffers. template -inline tcb::span ByteBuffer::ElementsIteratorNext() const { +inline bool ByteBuffer::ElementsIteratorNext(tcb::span& raw_bytes) const { if (element_iterator_current_ptr_ == element_iterator_end_ptr_) { - return {}; + raw_bytes = {}; + return false; } const size_t bytes_remaining = @@ -419,10 +420,9 @@ inline tcb::span ByteBuffer::ElementsIteratorNext() const if (bytes_remaining < element_size_) { throw InvalidInputException("Malformed fixed-size buffer: truncated element in iterator"); } - const auto out_bytes = - tcb::span(element_iterator_current_ptr_, element_size_); + raw_bytes = tcb::span(element_iterator_current_ptr_, element_size_); element_iterator_current_ptr_ += element_size_; - return out_bytes; + return true; } // Variable-sized elements @@ -438,10 +438,9 @@ inline tcb::span ByteBuffer::ElementsIteratorNext() const throw InvalidInputException("Malformed variable-size buffer: truncated element payload in iterator"); } - const auto out_bytes = - tcb::span(element_iterator_current_ptr_, current_element_size); + raw_bytes = tcb::span(element_iterator_current_ptr_, current_element_size); element_iterator_current_ptr_ += current_element_size; - return out_bytes; + return true; } From 44357ac2bbcca54d4b35eb4e4b2c86de91f3bd63 Mon Sep 17 00:00:00 2001 From: Alejandro Valerio Date: Thu, 12 Mar 2026 20:21:39 -0600 Subject: [PATCH 11/18] - Added support for Parquet utils to read/decode num_elements from headers+level bytes without actually reading the value bytes. - num_elements for DICTIONARY_PAGE and DATA_PAGE_V2 pages read from the headers values directly (easy) - num_elements for DATA_PAGE_V1 pages read from the level bytes and RLE bit decoding (hard!) - Added unittests for RLE bit decoding and count present values from definition levels for DATA_PAGE_V1 pages. - Updated all scripts and unittests for new Parquet parsing requirements. --- src/common/dbpa_local_test.cpp | 14 +- src/processing/encryption_sequencer.cpp | 6 +- src/processing/encryption_sequencer_test.cpp | 54 ++-- src/processing/parquet_utils.cpp | 253 +++++++++++++++-- src/processing/parquet_utils.h | 25 +- src/processing/parquet_utils_test.cpp | 274 ++++++++++++++++++- src/processing/typed_buffer.h | 15 +- src/scripts/dbpa_remote_testapp.cpp | 66 +++-- src/scripts/performance_test.cpp | 29 +- 9 files changed, 628 insertions(+), 108 deletions(-) diff --git a/src/common/dbpa_local_test.cpp b/src/common/dbpa_local_test.cpp index a094650..0904f93 100644 --- a/src/common/dbpa_local_test.cpp +++ b/src/common/dbpa_local_test.cpp @@ -50,7 +50,7 @@ TEST_F(LocalDataBatchProtectionAgentTest, SuccessfulEncryption) { Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, std::nullopt)); std::vector test_data = BuildByteArrayValueBytesForTesting("test_ABC"); - std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}}; + std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}; auto result = agent.Encrypt(test_data, encoding_attributes); ASSERT_NE(result, nullptr); @@ -74,7 +74,7 @@ TEST_F(LocalDataBatchProtectionAgentTest, SuccessfulEncryptionCompressedDictiona 0x03, 0x32, 0x92, 0x12, 0xF3, 0x80, 0x10, 0x00, 0xC7, 0xB8, 0x50, 0xFC, 0x13, 0x00, 0x00, 0x00 }; - std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}}; + std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}; auto result = agent.Encrypt(test_data_gzip, encoding_attributes); ASSERT_NE(result, nullptr); @@ -93,7 +93,7 @@ TEST_F(LocalDataBatchProtectionAgentTest, SuccessfulDecryption) { Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, DBPS_ENCRYPTION_METADATA)); std::vector test_data = BuildByteArrayValueBytesForTesting("test_EFG"); - std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}}; + std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}; auto result = agent.Decrypt(test_data, encoding_attributes); ASSERT_NE(result, nullptr); @@ -113,7 +113,7 @@ TEST_F(LocalDataBatchProtectionAgentTest, RoundTripEncryptDecrypt) { // Original data to encrypt std::vector original_data = BuildByteArrayValueBytesForTesting("roundtrip_XYZ"); - std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}}; + std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}; // Encrypt the data auto encrypt_result = encrypt_agent.Encrypt(original_data, encoding_attributes); @@ -156,7 +156,7 @@ TEST_F(LocalDataBatchProtectionAgentTest, EncryptWithoutInit) { LocalDataBatchProtectionAgent agent; std::vector test_data = {1, 2, 3, 4}; - std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}}; + std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}; auto result = agent.Encrypt(test_data, encoding_attributes); ASSERT_NE(result, nullptr); @@ -171,7 +171,7 @@ TEST_F(LocalDataBatchProtectionAgentTest, DecryptWithoutInit) { LocalDataBatchProtectionAgent agent; std::vector test_data = {1, 2, 3, 4}; - std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}}; + std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}; auto result = agent.Decrypt(test_data, encoding_attributes); ASSERT_NE(result, nullptr); @@ -203,7 +203,7 @@ TEST_F(LocalDataBatchProtectionAgentTest, MissingPageEncoding) { Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, std::nullopt)); std::vector test_data = {1, 2, 3, 4}; - std::map encoding_attributes = {{"page_type", "DICTIONARY_PAGE"}}; + std::map encoding_attributes = {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}; auto result = agent.Encrypt(test_data, encoding_attributes); ASSERT_NE(result, nullptr); diff --git a/src/processing/encryption_sequencer.cpp b/src/processing/encryption_sequencer.cpp index 91c32b6..6636fcb 100644 --- a/src/processing/encryption_sequencer.cpp +++ b/src/processing/encryption_sequencer.cpp @@ -62,7 +62,7 @@ namespace { int64_t join_with_length_prefix_ns, int64_t compress_ns) { - std::cout << "+++++ DecodeAndEncrypt timings (microseconds + nanoseconds) +++++" << std::endl; + std::cout << "----- DecodeAndEncrypt timings (microseconds + nanoseconds) -----" << std::endl; PrintDurationLine("DecompressAndSplit", decompress_and_split_ns); PrintDurationLine("ParseValueBytesIntoTypedList", parse_value_bytes_into_typed_list_ns); PrintDurationLine("EncryptValueList", encrypt_value_list_ns); @@ -174,7 +174,7 @@ bool DataBatchEncryptionSequencer::DecodeAndEncrypt(tcb::span pla // Decompress and split plaintext into level and value bytes auto stage_start = std::chrono::steady_clock::now(); - auto [level_bytes, value_bytes] = DecompressAndSplit( + auto [level_bytes, value_bytes, num_elements] = DecompressAndSplit( plaintext, compression_, encoding_attributes_converted_); decompress_and_split_ns = std::chrono::duration_cast( std::chrono::steady_clock::now() - stage_start).count(); @@ -355,7 +355,7 @@ bool DataBatchEncryptionSequencer::ConvertEncodingAttributesToValues() { add_int("page_v2_num_nulls"); add_bool("page_v2_is_compressed"); } else if (page_type == "DICTIONARY_PAGE") { - // DICTIONARY_PAGE has no specific encoding attributes + add_int("dict_page_num_values"); } else { throw InvalidInputException("Unexpected page type: " + page_type); } diff --git a/src/processing/encryption_sequencer_test.cpp b/src/processing/encryption_sequencer_test.cpp index 4073639..0342ab0 100644 --- a/src/processing/encryption_sequencer_test.cpp +++ b/src/processing/encryption_sequencer_test.cpp @@ -81,7 +81,7 @@ TEST(EncryptionSequencer, EncryptionDecryption) { std::nullopt, // datatype_length CompressionCodec::UNCOMPRESSED, // compression Encoding::PLAIN, // encoding - {{"page_type", "DICTIONARY_PAGE"}}, // encoding_attributes (mostly empty for basic test) + {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, // encoding_attributes (mostly empty for basic test) CompressionCodec::UNCOMPRESSED, // encrypted_compression "test_key_123", // key_id "test_user", // user_id @@ -97,11 +97,11 @@ TEST(EncryptionSequencer, EncryptionDecryption) { // Test 2: Different key_id produces different encryption { DataBatchEncryptionSequencer sequencer1( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "key1", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "key1", "test_user", "{}", {} ); DataBatchEncryptionSequencer sequencer2( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "key2", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "key2", "test_user", "{}", {} ); @@ -115,11 +115,11 @@ TEST(EncryptionSequencer, EncryptionDecryption) { // Test 3: Same key_id produces consistent encryption { DataBatchEncryptionSequencer sequencer1( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "same_key", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "same_key", "test_user", "{}", {} ); DataBatchEncryptionSequencer sequencer2( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "same_key", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "same_key", "test_user", "{}", {} ); @@ -133,7 +133,7 @@ TEST(EncryptionSequencer, EncryptionDecryption) { // Test 4: Empty data encryption { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} ); // This should fail because empty input is rejected @@ -144,7 +144,7 @@ TEST(EncryptionSequencer, EncryptionDecryption) { // Test 5: Binary data encryption { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} ); // Binary data: 0x00, 0x01, 0x02, 0x03, 0x04, 0x05 @@ -160,7 +160,7 @@ TEST(EncryptionSequencer, ParameterValidation) { // Test 1: Valid parameters, should succeed { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} ); bool result = sequencer.DecodeAndEncrypt(HELLO_WORLD_DATA); ASSERT_TRUE(result) << "Valid parameters test failed: " << sequencer.error_stage_ << " - " << sequencer.error_message_; @@ -169,7 +169,7 @@ TEST(EncryptionSequencer, ParameterValidation) { // Test 2: Invalid compression (should succeed with warning) { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::GZIP, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::GZIP, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} ); bool result = sequencer.DecodeAndEncrypt(HELLO_WORLD_DATA); ASSERT_TRUE(result); @@ -179,7 +179,7 @@ TEST(EncryptionSequencer, ParameterValidation) { // Test 3: Undefined encoding is supported { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::UNDEFINED, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::UNDEFINED, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} ); bool result = sequencer.DecodeAndEncrypt(HELLO_WORLD_DATA); ASSERT_TRUE(result) << "Encoding UNDEFINED should be supported: " << sequencer.error_message_; @@ -189,7 +189,7 @@ TEST(EncryptionSequencer, ParameterValidation) { // Test 4: All encodings now supported (including RLE) { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::RLE, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::RLE, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} ); bool result = sequencer.DecodeAndEncrypt(HELLO_WORLD_DATA); ASSERT_TRUE(result) << "Encoding RLE should now be supported: " << sequencer.error_message_; @@ -203,7 +203,7 @@ TEST(EncryptionSequencer, InputValidation) { // Test 1: Empty plaintext { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} ); bool result = sequencer.DecodeAndEncrypt(EMPTY_DATA); EXPECT_FALSE(result) << "Empty plaintext test should have failed"; @@ -213,7 +213,7 @@ TEST(EncryptionSequencer, InputValidation) { // Test 2: Empty ciphertext { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} ); bool result = sequencer.DecryptAndEncode(EMPTY_DATA); EXPECT_FALSE(result) << "Empty ciphertext test should have failed"; @@ -223,7 +223,7 @@ TEST(EncryptionSequencer, InputValidation) { // Test 3: Empty key_id { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "", "test_user", "{}", {} ); bool result = sequencer.DecodeAndEncrypt(HELLO_WORLD_DATA); EXPECT_FALSE(result) << "Empty key_id test should have failed"; @@ -233,7 +233,7 @@ TEST(EncryptionSequencer, InputValidation) { // Test 4: Missing encryption_metadata { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} // encryption_metadata, setting it to empty map. ); bool result = sequencer.DecryptAndEncode(HELLO_WORLD_DATA); @@ -245,7 +245,7 @@ TEST(EncryptionSequencer, InputValidation) { // Test 5: Incorrect encryption_metadata version { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {{"dbps_agent_version", "v0.09"}} // encryption_metadata, setting it to incorrect version. ); bool result = sequencer.DecryptAndEncode(HELLO_WORLD_DATA); @@ -261,7 +261,7 @@ TEST(EncryptionSequencer, RoundTripEncryption) { // Test 1: Basic round trip - "Hello, World!" { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "test_key_123", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key_123", "test_user", "{}", {} ); // Encrypt @@ -280,7 +280,7 @@ TEST(EncryptionSequencer, RoundTripEncryption) { // Test 2: Binary data round trip { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "binary_test_key", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "binary_test_key", "test_user", "{}", {} ); // Encrypt @@ -299,7 +299,7 @@ TEST(EncryptionSequencer, RoundTripEncryption) { // Test 3: Single character round trip { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "single_char_key", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "single_char_key", "test_user", "{}", {} ); // "A" @@ -320,11 +320,11 @@ TEST(EncryptionSequencer, RoundTripEncryption) { // Test 4: Different keys produce different encrypted results { DataBatchEncryptionSequencer sequencer1( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "key1", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "key1", "test_user", "{}", {} ); DataBatchEncryptionSequencer sequencer2( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "key2", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "key2", "test_user", "{}", {} ); bool result1 = sequencer1.DecodeAndEncrypt(HELLO_WORLD_DATA); @@ -355,7 +355,7 @@ TEST(EncryptionSequencer, ResultStorage) { // Test 1: Verify encrypted result is stored { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} ); bool result = sequencer.DecodeAndEncrypt(HELLO_WORLD_DATA); @@ -370,7 +370,7 @@ TEST(EncryptionSequencer, ResultStorage) { // Test 2: Verify decrypted result is stored { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} ); // First encrypt something @@ -401,7 +401,7 @@ TEST(EncryptionSequencer, BooleanTypeUsesPerBlockEncryption) { std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, - {{"page_type", "DICTIONARY_PAGE"}}, + {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", @@ -433,7 +433,7 @@ TEST(EncryptionSequencer, RleDictionaryEncodingUsesPerBlockEncryption) { std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::RLE_DICTIONARY, - {{"page_type", "DICTIONARY_PAGE"}}, + {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", @@ -460,7 +460,7 @@ TEST(EncryptionSequencer, FixedLenByteArrayValidation) { // Helper function to test validation failure auto testValidationFailure = [&](const std::optional& datatype_length, const std::string& expected_msg) -> bool { DataBatchEncryptionSequencer sequencer( - "test_column", Type::FIXED_LEN_BYTE_ARRAY, datatype_length, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "test_key_123", "test_user", "{}", {} + "test_column", Type::FIXED_LEN_BYTE_ARRAY, datatype_length, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key_123", "test_user", "{}", {} ); bool result = sequencer.DecodeAndEncrypt(HELLO_WORLD_DATA); @@ -484,7 +484,7 @@ TEST(EncryptionSequencer, FixedLenByteArrayValidation) { EXPECT_TRUE(testValidationFailure(0, "FIXED_LEN_BYTE_ARRAY datatype_length must be positive")); // Test valid case (should pass parameter validation) - DataBatchEncryptionSequencer sequencer("test_column", Type::FIXED_LEN_BYTE_ARRAY, 16, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "test_key_123", "test_user", "{}", {}); + DataBatchEncryptionSequencer sequencer("test_column", Type::FIXED_LEN_BYTE_ARRAY, 16, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key_123", "test_user", "{}", {}); bool result = sequencer.DecodeAndEncrypt(FIXED_LEN_BYTE_ARRAY_DATA); if (!result && sequencer.error_stage_ == "parameter_validation") { diff --git a/src/processing/parquet_utils.cpp b/src/processing/parquet_utils.cpp index 9b5930b..a963e20 100644 --- a/src/processing/parquet_utils.cpp +++ b/src/processing/parquet_utils.cpp @@ -28,31 +28,186 @@ using namespace dbps::compression; using namespace dbps::processing; // ----------------------------------------------------------------------------- -// Process Parquet formatted Dictionary and Data pages +// Helper functions for Parquet DATA_PAGE_V1 definition level bytes parsing to count present values. // ----------------------------------------------------------------------------- -int CalculateLevelBytesLength(tcb::span raw, - const AttributesMap& encoding_attribs) { - - // Helper function to skip V1 RLE level data in raw bytes - // Returns number of bytes consumed: [4-byte len] + [level bytes indicated by `len`] - auto SkipV1RLELevel = [&raw](size_t& offset) -> int { - if (offset + 4 > raw.size()) { - throw InvalidInputException( - "Invalid RLE level data: offset + 4 exceeds data size (offset=" + - std::to_string(offset) + ", size=" + std::to_string(raw.size()) + ")"); +// Decodes one unsigned LEB128 (base-128 varint) run header from `bytes`, +// starting at `offset`, and advances `offset` past the decoded header bytes. +// +// A "run header" is a variable-length encoded integer that indicates the length of a run. +// +// In Parquet V1 hybrid RLE/bit-packed streams, this run header indicates: +// - RLE run when (header & 1) == 0, with run_length = header >> 1 +// - Bit-packed run when (header & 1) == 1, with num_groups = header >> 1 +// and run_length = num_groups * 8 values +// +// This is used by V1 definition-level decoding to iterate runs and compute +// the count of present (non-null) values in nullable data pages. +// +uint32_t ReadV1RunHeaderUleb128(tcb::span bytes, size_t& offset) { + uint32_t value = 0; + int shift = 0; + while (true) { + if (offset >= bytes.size()) { + throw InvalidInputException("Invalid DATA_PAGE_V1 level stream: truncated varint header"); } - uint32_t len = read_u32_le(raw, offset); - if (offset + 4 + len > raw.size()) { - throw InvalidInputException( - "Invalid RLE level data: length field overflows (offset=" + - std::to_string(offset) + ", len=" + std::to_string(len) + ", size=" + - std::to_string(raw.size()) + ")"); + uint8_t b = bytes[offset++]; + value |= static_cast(b & 0x7F) << shift; + if ((b & 0x80) == 0) { + return value; } - offset += 4 + len; - return 4 + len; - }; - + shift += 7; + if (shift > 28) { + throw InvalidInputException("Invalid DATA_PAGE_V1 level stream: varint header too large"); + } + } +} + +// Decodes a DATA_PAGE_V1 definition-level payload (hybrid RLE/bit-packed) and +// returns the number of present (non-null) values in the page. +// +// Inputs: +// - def_payload: bytes of the V1 definition-level stream payload only +// (without the outer [u32 length] prefix). +// - num_values: total number of logical values in the page (includes nulls). +// - max_def_level: maximum definition level for the column in this page. +// +// Output: +// - present_count = number of decoded definition levels equal to max_def_level. +// +size_t CountPresentFromDefinitionLevelsV1(tcb::span def_payload, int32_t num_values, int32_t max_def_level) { + // Definition level bit width is ceil(log2(max_def_level + 1)). + int bit_width = 0; + while ((1u << bit_width) <= static_cast(max_def_level)) { + ++bit_width; + } + if (bit_width <= 0) { + throw InvalidInputException("Invalid V1 definition levels: computed bit_width must be positive"); + } + + size_t present_count = 0; + size_t decoded_values = 0; + size_t def_offset = 0; + + // Hybrid RLE/bit-packed decode loop. + while (decoded_values < static_cast(num_values)) { + uint32_t header = ReadV1RunHeaderUleb128(def_payload, def_offset); + + if ((header & 1u) == 0u) { + // RLE run: header = (run_len << 1), then repeated value in ceil(bit_width/8) bytes. + const size_t run_len = static_cast(header >> 1); + const size_t remaining = static_cast(num_values) - decoded_values; + if (run_len == 0 || run_len > remaining) { + throw InvalidInputException("Invalid DATA_PAGE_V1 definition levels: invalid RLE run length"); + } + + const size_t byte_width = static_cast((bit_width + 7) / 8); + if (def_offset + byte_width > def_payload.size()) { + throw InvalidInputException("Invalid V1 definition levels: truncated RLE run value"); + } + + uint32_t level = 0; + for (size_t i = 0; i < byte_width; ++i) { + level |= static_cast(def_payload[def_offset + i]) << (8 * i); + } + def_offset += byte_width; + if (level > static_cast(max_def_level)) { + throw InvalidInputException("Invalid DATA_PAGE_V1 definition levels: decoded level exceeds max_def_level"); + } + + if (level == static_cast(max_def_level)) { + present_count += run_len; + } + decoded_values += run_len; + } else { + // Bit-packed run: header = (num_groups << 1) | 1, each group has 8 values. + const size_t num_groups = static_cast(header >> 1); + const size_t run_len = num_groups * 8; + const size_t remaining = static_cast(num_values) - decoded_values; + if (num_groups == 0 || run_len > remaining) { + throw InvalidInputException("Invalid DATA_PAGE_V1 definition levels: invalid bit-packed run length"); + } + + const size_t total_bits = run_len * static_cast(bit_width); + const size_t byte_len = (total_bits + 7) / 8; + if (def_offset + byte_len > def_payload.size()) { + throw InvalidInputException("Invalid DATA_PAGE_V1 definition levels: truncated bit-packed run payload"); + } + auto packed = tcb::span(def_payload.data() + def_offset, byte_len); + def_offset += byte_len; + + auto ReadPacked = [&](size_t bit_offset) -> uint32_t { + uint32_t v = 0; + for (int b = 0; b < bit_width; ++b) { + size_t abs_bit = bit_offset + static_cast(b); + size_t byte_index = abs_bit / 8; + size_t bit_index = abs_bit % 8; + uint8_t bit = static_cast((packed[byte_index] >> bit_index) & 0x01); + v |= static_cast(bit) << b; + } + return v; + }; + + for (size_t i = 0; i < run_len; ++i) { + uint32_t level = ReadPacked(i * static_cast(bit_width)); + if (level > static_cast(max_def_level)) { + throw InvalidInputException("Invalid DATA_PAGE_V1 definition levels: decoded level exceeds max_def_level"); + } + if (level == static_cast(max_def_level)) { + ++present_count; + } + } + decoded_values += run_len; + } + } + return present_count; +} + +// ----------------------------------------------------------------------------- +// Helper functions to read/split DATA_PAGE_V1 level bytes -- Read length-prefixed level bytes +// ----------------------------------------------------------------------------- + +// Function to read a length-prefixed payload from the level bytes. +tcb::span ReadV1LengthPrefixedPayload(tcb::span bytes, size_t& offset) { + if (offset + 4 > bytes.size()) { + throw InvalidInputException( + "Invalid Parquet DATA_PAGE_V1 level bytes: missing 4-byte length prefix"); + } + uint32_t len = read_u32_le(bytes, offset); + const size_t payload_offset = offset + 4; + if (len > bytes.size() - payload_offset) { + throw InvalidInputException( + "Invalid Parquet DATA_PAGE_V1 level bytes: length-prefixed block exceeds bounds"); + } + offset = payload_offset + static_cast(len); + return tcb::span(bytes.data() + payload_offset, static_cast(len)); +} + +// Function to skip the repetition levels bytes and return the definition levels bytes payload. +tcb::span ReadDefinitionLevelBytesV1(tcb::span level_bytes, int32_t max_rep_level) { + // V1 level bytes are [rep_levels?][def_levels], each as [u32 len][payload]. + size_t level_offset = 0; + + // Skip the repetition levels bytes if any. + if (max_rep_level > 0) { + (void) ReadV1LengthPrefixedPayload(level_bytes, level_offset); + } + + // Read the definition levels bytes. + auto def_payload = ReadV1LengthPrefixedPayload(level_bytes, level_offset); + if (level_offset != level_bytes.size()) { + throw InvalidInputException("Invalid Parquet DATA_PAGE_V1 level bytes: trailing bytes after definition levels block"); + } + return def_payload; +} + +// ----------------------------------------------------------------------------- +// Helper function to calculate level bytes length +// ----------------------------------------------------------------------------- + +int CalculateLevelBytesLength(tcb::span raw, + const AttributesMap& encoding_attribs) { + // Get page_type from the converted attributes const std::string& page_type = std::get(encoding_attribs.at("page_type")); int total_level_bytes = 0; @@ -72,17 +227,21 @@ int CalculateLevelBytesLength(tcb::span raw, ", definition_level_encoding=" + def_encoding + " (only RLE is expected)"); } - // if max_rep_level > 0, there are repetition levels bytes. Same for definition levels. + // Read and skip the repetition/definition level bytes to calculate the final offset where + // the level bytes end and the value bytes start. + // - If max_rep_level > 0, there are repetition levels bytes. Same for definition levels. int32_t max_rep_level = std::get(encoding_attribs.at("data_page_max_repetition_level")); int32_t max_def_level = std::get(encoding_attribs.at("data_page_max_definition_level")); size_t offset = 0; if (max_rep_level > 0) { - int bytes_skipped = SkipV1RLELevel(offset); - total_level_bytes += bytes_skipped; + size_t start_offset = offset; + (void) ReadV1LengthPrefixedPayload(raw, offset); + total_level_bytes += static_cast(offset - start_offset); } if (max_def_level > 0) { - int bytes_skipped = SkipV1RLELevel(offset); - total_level_bytes += bytes_skipped; + size_t start_offset = offset; + (void) ReadV1LengthPrefixedPayload(raw, offset); + total_level_bytes += static_cast(offset - start_offset); } } else if (page_type == "DICTIONARY_PAGE") { @@ -107,6 +266,10 @@ int CalculateLevelBytesLength(tcb::span raw, return total_level_bytes; } +// ----------------------------------------------------------------------------- +// Public functions to process Parquet formatted Dictionary and Data pages +// ----------------------------------------------------------------------------- + LevelAndValueBytes DecompressAndSplit( tcb::span plaintext, CompressionCodec::type compression, @@ -123,7 +286,23 @@ LevelAndValueBytes DecompressAndSplit( int leading_bytes_to_strip = CalculateLevelBytesLength( decompressed_bytes, encoding_attributes); auto [level_bytes, value_bytes] = Split(decompressed_bytes, leading_bytes_to_strip); - return LevelAndValueBytes{std::move(level_bytes), std::move(value_bytes)}; + + // For DATA_PAGE_V1, calculate num_elements by parsing the level bytes. + size_t num_elements = 0; + const int32_t num_values = std::get(encoding_attributes.at("data_page_num_values")); + int32_t max_def_level = std::get(encoding_attributes.at("data_page_max_definition_level")); + int32_t max_rep_level = std::get(encoding_attributes.at("data_page_max_repetition_level")); + if (max_def_level == 0) { + // All values are present in the value bytes section. + num_elements = static_cast(num_values); + } + // If max_def_level > 0, there are definition levels bytes. So parse it and count the present values. + else { + auto def_bytes_payload = ReadDefinitionLevelBytesV1(level_bytes, max_rep_level); + num_elements = CountPresentFromDefinitionLevelsV1(def_bytes_payload, num_values, max_def_level); + } + + return LevelAndValueBytes{std::move(level_bytes), std::move(value_bytes), num_elements}; } // On DATA_PAGE_V2, only the value bytes are compressed. @@ -143,14 +322,26 @@ LevelAndValueBytes DecompressAndSplit( } else { value_bytes = std::vector(compressed_value_bytes_span.begin(), compressed_value_bytes_span.end()); } - return LevelAndValueBytes{std::move(level_bytes), std::move(value_bytes)}; + + // For DATA_PAGE_V2, get num_elements from num_values and num_nulls in encoding_attributes. + int32_t num_values = std::get(encoding_attributes.at("data_page_num_values")); + int32_t num_nulls = std::get(encoding_attributes.at("page_v2_num_nulls")); + if (num_nulls > num_values) { + throw InvalidInputException( + "Invalid num_nulls: " + std::to_string(num_nulls) + " > num_values: " + + std::to_string(num_values) + " in DATA_PAGE_V2 encoding attributes"); + } + size_t num_elements = static_cast(num_values - num_nulls); + + return LevelAndValueBytes{std::move(level_bytes), std::move(value_bytes), num_elements}; } // DICTIONARY_PAGE has no level bytes. if (page_type == "DICTIONARY_PAGE") { auto level_bytes = std::vector(); auto value_bytes = Decompress(plaintext, compression); - return LevelAndValueBytes{std::move(level_bytes), std::move(value_bytes)}; + size_t num_elements = static_cast( std::get(encoding_attributes.at("dict_page_num_values"))); + return LevelAndValueBytes{std::move(level_bytes), std::move(value_bytes), num_elements}; } throw InvalidInputException("Unexpected page type: " + page_type); @@ -196,7 +387,7 @@ std::vector CompressAndJoin( } // ----------------------------------------------------------------------------- -// Build Parquet formatted value bytes into TypedValuesBuffer +// Public functions to build Parquet formatted value bytes into TypedValuesBuffer // ----------------------------------------------------------------------------- TypedValuesBuffer ReinterpretValueBytesAsTypedValuesBuffer(tcb::span value_bytes, @@ -254,3 +445,5 @@ std::vector GetTypedValuesBufferAsValueBytes(TypedValuesBuffer&& buffer return buf.FinalizeAndTakeBuffer(); }, buffer); } + +// ----------------------------------------------------------------------------- diff --git a/src/processing/parquet_utils.h b/src/processing/parquet_utils.h index 48f53e9..8aba317 100644 --- a/src/processing/parquet_utils.h +++ b/src/processing/parquet_utils.h @@ -33,12 +33,13 @@ struct LevelAndValueBytes { std::vector level_bytes; std::vector value_bytes; + size_t num_elements; }; using namespace dbps::external; // ----------------------------------------------------------------------------- -// Helper functions for processing Parquet formatted data pages and dictionary pages. +// Helper functions for lower-level Parquet metadata and level bytes parsing. // ----------------------------------------------------------------------------- /** @@ -52,6 +53,24 @@ using namespace dbps::external; int CalculateLevelBytesLength(tcb::span raw, const AttributesMap& encoding_attribs); +/** + * Decode DATA_PAGE_V1 definition-level payload bytes (hybrid RLE/bit-packed) + * and return the number of present (non-null) values. + * + * @param def_payload Definition-level payload bytes only (without [u32 length] prefix) + * @param num_values Total logical values in page (includes nulls) + * @param max_def_level Maximum definition level for the column + * @return Count of decoded definition levels equal to max_def_level + */ +size_t CountPresentFromDefinitionLevelsV1( + tcb::span def_payload, + int32_t num_values, + int32_t max_def_level); + +// ----------------------------------------------------------------------------- +// Functions to decompress and split a Parquet page into level and value bytes. +// ----------------------------------------------------------------------------- + /** * Decompresses and splits a Parquet page into level and value bytes. * Handles DATA_PAGE_V1, DATA_PAGE_V2 (including optional compression on value bytes), @@ -74,7 +93,7 @@ std::vector CompressAndJoin( // ----------------------------------------------------------------------------- -// Helper functions for zero-copy reinterpretation of raw value bytes into a typed buffer. +// Functions for zero-copy reinterpretation of raw value bytes into a typed buffer. // ----------------------------------------------------------------------------- /** @@ -106,3 +125,5 @@ dbps::processing::TypedValuesBuffer ReinterpretValueBytesAsTypedValuesBuffer( */ std::vector GetTypedValuesBufferAsValueBytes( dbps::processing::TypedValuesBuffer&& buffer); + +// ----------------------------------------------------------------------------- diff --git a/src/processing/parquet_utils_test.cpp b/src/processing/parquet_utils_test.cpp index 1c4f8b5..bc2be69 100644 --- a/src/processing/parquet_utils_test.cpp +++ b/src/processing/parquet_utils_test.cpp @@ -32,6 +32,75 @@ using namespace dbps::external; using namespace dbps::compression; using namespace dbps::processing; +// ----------------------------------------------------------------------------- +// Helper functions to generate Parquet DATA_PAGE_V1 level bytes payloads for testing. +// ----------------------------------------------------------------------------- + +namespace { + +std::vector EncodeUleb128(uint32_t value) { + std::vector out; + do { + uint8_t byte = static_cast(value & 0x7F); + value >>= 7; + if (value != 0) { + byte |= 0x80; + } + out.push_back(byte); + } while (value != 0); + return out; +} + +std::vector MakeRleDefPayload(uint32_t run_len, uint32_t level, int bit_width) { + std::vector out = EncodeUleb128(run_len << 1); // RLE run + const size_t byte_width = static_cast((bit_width + 7) / 8); + for (size_t i = 0; i < byte_width; ++i) { + out.push_back(static_cast((level >> (8 * i)) & 0xFF)); + } + return out; +} + +std::vector PackBitPackedLevels( + const std::vector& levels, + int bit_width) { + const size_t total_bits = levels.size() * static_cast(bit_width); + std::vector out((total_bits + 7) / 8, 0); + size_t bit_offset = 0; + for (uint32_t level : levels) { + for (int b = 0; b < bit_width; ++b) { + uint8_t bit = static_cast((level >> b) & 0x01); + size_t abs_bit = bit_offset + static_cast(b); + out[abs_bit / 8] |= static_cast(bit << (abs_bit % 8)); + } + bit_offset += static_cast(bit_width); + } + return out; +} + +std::vector MakeBitPackedDefPayload( + const std::vector& levels, + int bit_width) { + EXPECT_EQ(levels.size() % 8, 0u); + const uint32_t num_groups = static_cast(levels.size() / 8); + std::vector out = EncodeUleb128((num_groups << 1) | 1u); // bit-packed run + auto packed = PackBitPackedLevels(levels, bit_width); + out.insert(out.end(), packed.begin(), packed.end()); + return out; +} + +std::vector WrapLengthPrefixed(const std::vector& payload) { + std::vector out; + append_u32_le(out, static_cast(payload.size())); + out.insert(out.end(), payload.begin(), payload.end()); + return out; +} + +} // namespace + +// ----------------------------------------------------------------------------- +// Tests for CalculateLevelBytesLength function. +// ----------------------------------------------------------------------------- + TEST(ParquetUtils, CalculateLevelBytesLength_DATA_PAGE_V2) { std::vector raw = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}; AttributesMap attribs = { @@ -158,6 +227,120 @@ TEST(ParquetUtils, CalculateLevelBytesLength_NegativeTotalSize) { EXPECT_THROW(CalculateLevelBytesLength(raw, attribs), InvalidInputException); } +// ----------------------------------------------------------------------------- +// Tests for CountPresentFromDefinitionLevelsV1 function. +// ----------------------------------------------------------------------------- + +TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_RleAllPresent) { + auto def_payload = MakeRleDefPayload(10, 1, 1); + EXPECT_EQ(10u, CountPresentFromDefinitionLevelsV1(def_payload, 10, 1)); +} + +TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_RleAllNull) { + auto def_payload = MakeRleDefPayload(10, 0, 1); + EXPECT_EQ(0u, CountPresentFromDefinitionLevelsV1(def_payload, 10, 1)); +} + +TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_BitPackedMixed) { + std::vector levels = {1, 0, 1, 0, 1, 0, 1, 0}; + auto def_payload = MakeBitPackedDefPayload(levels, 1); + EXPECT_EQ(4u, CountPresentFromDefinitionLevelsV1(def_payload, 8, 1)); +} + +TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_MixedRuns) { + auto rle_part = MakeRleDefPayload(4, 1, 1); // 4 present + std::vector levels = {0, 1, 0, 1, 0, 0, 0, 0}; // +2 present + auto bp_part = MakeBitPackedDefPayload(levels, 1); + rle_part.insert(rle_part.end(), bp_part.begin(), bp_part.end()); + EXPECT_EQ(6u, CountPresentFromDefinitionLevelsV1(rle_part, 12, 1)); +} + +TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_InvalidRunLength) { + auto def_payload = MakeRleDefPayload(9, 1, 1); // run_len > num_values + EXPECT_THROW(CountPresentFromDefinitionLevelsV1(def_payload, 8, 1), InvalidInputException); +} + +TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_InvalidLevelExceedsMax) { + auto def_payload = MakeRleDefPayload(1, 2, 1); // level 2 > max_def_level 1 + EXPECT_THROW(CountPresentFromDefinitionLevelsV1(def_payload, 1, 1), InvalidInputException); +} + +TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_TruncatedVarint) { + std::vector def_payload = {0x80}; // continuation bit set, no next byte + EXPECT_THROW(CountPresentFromDefinitionLevelsV1(def_payload, 1, 1), InvalidInputException); +} + +TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_TruncatedRleValue) { + auto def_payload = EncodeUleb128(2); // run_len = 1, but missing repeated value byte + EXPECT_THROW(CountPresentFromDefinitionLevelsV1(def_payload, 1, 1), InvalidInputException); +} + +TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_BitPackedCanonical_0to7_88C6FA) { + // Canonical example from Parquet Encodings.md: + // values 0..7 with bit_width=3 are packed as bytes 0x88, 0xC6, 0xFA. + // Bit-packed header for one group of 8 values is varint((1 << 1) | 1) = 0x03. + std::vector def_payload = {0x03, 0x88, 0xC6, 0xFA}; + + EXPECT_EQ(1u, CountPresentFromDefinitionLevelsV1(def_payload, 8, 7)); // only value 7 + EXPECT_EQ(1u, CountPresentFromDefinitionLevelsV1(def_payload, 8, 3)); // only value 3 +} + +TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_ManualBytes_RleRunLen4_Level1) { + // Manual payload: + // - header 0x08 => RLE run, run_len = 0x08 >> 1 = 4 + // - repeated value byte = 0x01 (bit_width=1, level=1) + std::vector def_payload = {0x08, 0x01}; + EXPECT_EQ(4u, CountPresentFromDefinitionLevelsV1(def_payload, 4, 1)); +} + +TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_ManualBytes_BitPackedAlternating_0xAA) { + // Manual payload: + // - header 0x03 => bit-packed, num_groups = 1, run_len = 8 + // - packed byte 0xAA => bits (LSB->MSB): 0,1,0,1,0,1,0,1 + std::vector def_payload = {0x03, 0xAA}; + EXPECT_EQ(4u, CountPresentFromDefinitionLevelsV1(def_payload, 8, 1)); +} + +TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_ManualBytes_MixedRleAndBitPacked) { + // Manual payload: + // - 0x06,0x01 => RLE run_len=3, level=1 + // - 0x03,0x0F => bit-packed 8 values, bits: 1,1,1,1,0,0,0,0 + // Total values = 3 + 8 = 11; present count at max_def_level=1 is 3 + 4 = 7. + std::vector def_payload = {0x06, 0x01, 0x03, 0x0F}; + EXPECT_EQ(7u, CountPresentFromDefinitionLevelsV1(def_payload, 11, 1)); +} + +TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_BitWidth1_ExhaustiveOneGroup) { + // Exhaustive check for all 8-value bit-packed patterns at bit_width=1. + // Payload form: [0x03][packed-byte], where 0x03 means one bit-packed group (8 values). + for (int packed = 0; packed <= 0xFF; ++packed) { + std::vector def_payload = {0x03, static_cast(packed)}; + + size_t ones = 0; + for (int bit = 0; bit < 8; ++bit) { + ones += static_cast((packed >> bit) & 0x01); + } + + EXPECT_EQ(ones, CountPresentFromDefinitionLevelsV1(def_payload, 8, 1)) + << "packed=0x" << std::hex << packed; + } +} + +TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_RejectsZeroRleRunLength) { + // Header 0 means RLE with run_len = 0, which is invalid. + std::vector def_payload = {0x00, 0x00}; + EXPECT_THROW(CountPresentFromDefinitionLevelsV1(def_payload, 1, 1), InvalidInputException); +} + +TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_RejectsZeroBitPackedGroups) { + // Header 1 means bit-packed with num_groups = 0, which is invalid. + std::vector def_payload = {0x01}; + EXPECT_THROW(CountPresentFromDefinitionLevelsV1(def_payload, 8, 1), InvalidInputException); +} + +// ----------------------------------------------------------------------------- +// Tests for DecompressAndSplit function. +// ----------------------------------------------------------------------------- TEST(ParquetUtils, DecompressAndSplit_DataPageV2_Uncompressed) { AttributesMap attribs_conv = { @@ -229,6 +412,95 @@ TEST(ParquetUtils, DecompressAndSplit_DataPageV2_UnsupportedCompression) { DBPSUnsupportedException); } +TEST(ParquetUtils, DecompressAndSplit_DataPageV1_Required_NoLevels) { + AttributesMap attribs = { + {"page_type", std::string("DATA_PAGE_V1")}, + {"data_page_num_values", int32_t(4)}, + {"data_page_max_repetition_level", int32_t(0)}, + {"data_page_max_definition_level", int32_t(0)}, + {"page_v1_repetition_level_encoding", std::string("RLE")}, + {"page_v1_definition_level_encoding", std::string("RLE")} + }; + + std::vector value_bytes = {0x11, 0x22, 0x33, 0x44}; + auto result = DecompressAndSplit(value_bytes, CompressionCodec::UNCOMPRESSED, attribs); + + EXPECT_TRUE(result.level_bytes.empty()); + EXPECT_EQ(value_bytes, result.value_bytes); + EXPECT_EQ(4u, result.num_elements); +} + +TEST(ParquetUtils, DecompressAndSplit_DataPageV1_Nullable_RleAllPresent) { + AttributesMap attribs = { + {"page_type", std::string("DATA_PAGE_V1")}, + {"data_page_num_values", int32_t(5)}, + {"data_page_max_repetition_level", int32_t(0)}, + {"data_page_max_definition_level", int32_t(1)}, + {"page_v1_repetition_level_encoding", std::string("RLE")}, + {"page_v1_definition_level_encoding", std::string("RLE")} + }; + + std::vector def_payload = MakeRleDefPayload(5, 1, 1); + std::vector level_bytes = WrapLengthPrefixed(def_payload); + std::vector value_bytes = {0x10, 0x20, 0x30, 0x40, 0x50}; + std::vector plaintext = Join(level_bytes, value_bytes); + + auto result = DecompressAndSplit(plaintext, CompressionCodec::UNCOMPRESSED, attribs); + EXPECT_EQ(level_bytes, result.level_bytes); + EXPECT_EQ(value_bytes, result.value_bytes); + EXPECT_EQ(5u, result.num_elements); +} + +TEST(ParquetUtils, DecompressAndSplit_DataPageV1_Nullable_BitPackedWithRepLevels) { + AttributesMap attribs = { + {"page_type", std::string("DATA_PAGE_V1")}, + {"data_page_num_values", int32_t(8)}, + {"data_page_max_repetition_level", int32_t(1)}, + {"data_page_max_definition_level", int32_t(1)}, + {"page_v1_repetition_level_encoding", std::string("RLE")}, + {"page_v1_definition_level_encoding", std::string("RLE")} + }; + + // Rep levels are present but ignored for present-count logic. + std::vector rep_payload = MakeRleDefPayload(8, 0, 1); + std::vector def_payload = MakeBitPackedDefPayload({1, 0, 1, 0, 1, 0, 0, 0}, 1); + std::vector level_bytes = WrapLengthPrefixed(rep_payload); + auto def_wrapped = WrapLengthPrefixed(def_payload); + level_bytes.insert(level_bytes.end(), def_wrapped.begin(), def_wrapped.end()); + + std::vector value_bytes = {0x01, 0x02, 0x03}; + std::vector plaintext = Join(level_bytes, value_bytes); + + auto result = DecompressAndSplit(plaintext, CompressionCodec::UNCOMPRESSED, attribs); + EXPECT_EQ(level_bytes, result.level_bytes); + EXPECT_EQ(value_bytes, result.value_bytes); + EXPECT_EQ(3u, result.num_elements); +} + +TEST(ParquetUtils, DecompressAndSplit_DataPageV1_InvalidDefinitionPayload) { + AttributesMap attribs = { + {"page_type", std::string("DATA_PAGE_V1")}, + {"data_page_num_values", int32_t(4)}, + {"data_page_max_repetition_level", int32_t(0)}, + {"data_page_max_definition_level", int32_t(1)}, + {"page_v1_repetition_level_encoding", std::string("RLE")}, + {"page_v1_definition_level_encoding", std::string("RLE")} + }; + + std::vector def_payload = {0x80}; // truncated varint run header + std::vector level_bytes = WrapLengthPrefixed(def_payload); + std::vector value_bytes = {0xAA, 0xBB, 0xCC, 0xDD}; + std::vector plaintext = Join(level_bytes, value_bytes); + + EXPECT_THROW( + DecompressAndSplit(plaintext, CompressionCodec::UNCOMPRESSED, attribs), + InvalidInputException); +} + +// ----------------------------------------------------------------------------- +// Tests for CompressAndJoin function. +// ----------------------------------------------------------------------------- + TEST(ParquetUtils, CompressAndJoin_DataPageV1_Compressed) { AttributesMap attribs = { {"page_type", std::string("DATA_PAGE_V1")}, @@ -241,7 +513,7 @@ TEST(ParquetUtils, CompressAndJoin_DataPageV1_Compressed) { std::vector level_bytes; append_u32_le(level_bytes, 2); // RLE block length - level_bytes.insert(level_bytes.end(), {0x0A, 0x0B}); + level_bytes.insert(level_bytes.end(), {0x04, 0x01}); // RLE run: len=2, level=1 std::vector value_bytes = {0x21, 0x22, 0x23, 0x24}; auto joined = CompressAndJoin(level_bytes, value_bytes, CompressionCodec::SNAPPY, attribs); diff --git a/src/processing/typed_buffer.h b/src/processing/typed_buffer.h index e74f71e..aa73e87 100644 --- a/src/processing/typed_buffer.h +++ b/src/processing/typed_buffer.h @@ -157,6 +157,7 @@ class ByteBuffer { // Variables for element span iterator. mutable const uint8_t* element_iterator_current_ptr_; const uint8_t* element_iterator_end_ptr_; + mutable size_t element_iterator_count_ = 0; // Variables for determining offset of elements. size_t prefix_size_ = 0; @@ -208,6 +209,7 @@ ByteBuffer::ByteBuffer( elements_span_size_(elements_span.size()), element_iterator_current_ptr_(elements_span.data()), element_iterator_end_ptr_(elements_span.data() + elements_span.size()), + element_iterator_count_(0), num_elements_(kUnsetSize), codec_(std::move(codec)), element_size_(0), @@ -405,10 +407,16 @@ inline typename ByteBuffer::value_type ByteBuffer::GetElement(size // Element span iterator -- The streamlined version of the iterator. // ----------------------------------------------------------------------------- -// ++++++ Additional validation that this is only used for read-only buffers. +// ++++++ Add validation that this is only used for read-only buffers. template inline bool ByteBuffer::ElementsIteratorNext(tcb::span& raw_bytes) const { if (element_iterator_current_ptr_ == element_iterator_end_ptr_) { + // ++++++++ RESTORE THIS WHEN WE POPULATE ALWAYS THE num_elements_ FIELD FROM PARQUET DIRECTLY. + // if (element_iterator_count_ != num_elements_) { + // throw InvalidInputException(std::string("Malformed buffer: element iterator count does not match num_elements. ") + // + "element_iterator_count_=" + std::to_string(element_iterator_count_) + ", " + // + "num_elements_=" + std::to_string(num_elements_)); + // } raw_bytes = {}; return false; } @@ -422,6 +430,7 @@ inline bool ByteBuffer::ElementsIteratorNext(tcb::span& ra } raw_bytes = tcb::span(element_iterator_current_ptr_, element_size_); element_iterator_current_ptr_ += element_size_; + element_iterator_count_++; return true; } @@ -440,6 +449,7 @@ inline bool ByteBuffer::ElementsIteratorNext(tcb::span& ra raw_bytes = tcb::span(element_iterator_current_ptr_, current_element_size); element_iterator_current_ptr_ += current_element_size; + element_iterator_count_++; return true; } @@ -680,7 +690,8 @@ inline tcb::span ByteBuffer::GetWritableSpanForElement(size_t po } if (position >= num_elements_) { - throw InvalidInputException("Element position out of range during GetWriteSpanForElement"); + throw InvalidInputException(std::string("Element position out of range during GetWriteSpanForElement: ") + + "position=" + std::to_string(position) + " num_elements_=" + std::to_string(num_elements_)); } // For fixed-size elements, we write directly at the fixed offset. No need to re-bind the span. diff --git a/src/scripts/dbpa_remote_testapp.cpp b/src/scripts/dbpa_remote_testapp.cpp index f0c06da..cfae787 100644 --- a/src/scripts/dbpa_remote_testapp.cpp +++ b/src/scripts/dbpa_remote_testapp.cpp @@ -247,7 +247,7 @@ class DBPARemoteTestApp { auto compressed_plaintext = Compress(plaintext, CompressionCodec::SNAPPY); // Encrypt once for the combined payload - std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}}; + std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}; auto encrypt_result = agent_->Encrypt(span(compressed_plaintext), encoding_attributes); if (!encrypt_result || !encrypt_result->success()) { @@ -361,7 +361,7 @@ class DBPARemoteTestApp { auto compressed_plaintext = Compress(plaintext, CompressionCodec::SNAPPY); // Encrypt - std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}}; + std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}; auto encrypt_result = agent_->Encrypt(span(compressed_plaintext), encoding_attributes); if (!encrypt_result || !encrypt_result->success()) { @@ -619,12 +619,17 @@ class DBPARemoteTestApp { std::cout << "Test data: 3 fixed-length strings (8 bytes each)" << std::endl; std::cout << "Original size: " << fixed_length_data.size() << " bytes" << std::endl; - // Build DATA_PAGE_V1 payload: level bytes use RLE blocks with length prefixes + // Build a minimal valid DATA_PAGE_V1 nullable payload (max_def_level=1, max_rep_level=0): + // level bytes layout is [u32 def_payload_len][def_payload]. + // def_payload is hybrid RLE/bit-packed: + // - 0x06 = varint header for an RLE run (LSB=0), run_len = 0x06 >> 1 = 3 + // - 0x01 = repeated definition level value (bit_width=1, value=1 => "present") + // This yields 3 definition levels, all present, matching the 3 fixed-length values below. std::vector level_bytes; - append_u32_le(level_bytes, 3); // repetition level block length - level_bytes.insert(level_bytes.end(), 3, 0xAA); - append_u32_le(level_bytes, 5); // definition level block length - level_bytes.insert(level_bytes.end(), 5, 0xBB); + std::vector def_payload = {0x06, 0x01}; + append_u32_le(level_bytes, static_cast(def_payload.size())); + level_bytes.insert(level_bytes.end(), def_payload.begin(), def_payload.end()); + auto combined_uncompressed = Join(level_bytes, fixed_length_data); auto joined_plaintext = Compress(combined_uncompressed, CompressionCodec::SNAPPY); std::cout << "Compressed size: " << joined_plaintext.size() << " bytes" << std::endl; @@ -632,7 +637,7 @@ class DBPARemoteTestApp { {"page_type", "DATA_PAGE_V1"}, {"data_page_num_values", std::to_string(std::size(test_strings))}, {"data_page_max_definition_level", "1"}, - {"data_page_max_repetition_level", "2"}, + {"data_page_max_repetition_level", "0"}, {"page_v1_repetition_level_encoding", "RLE"}, {"page_v1_definition_level_encoding", "RLE"}, {"page_encoding", "PLAIN"} @@ -690,26 +695,18 @@ class DBPARemoteTestApp { std::vector(decrypted_plaintext.begin(), decrypted_plaintext.end()), CompressionCodec::SNAPPY); size_t offset = 0; - if (offset + 4 > decompressed_combined.size()) { - std::cout << "ERROR: Decompressed payload too small for rep level length" << std::endl; - return false; - } - uint32_t rep_len = read_u32_le(decompressed_combined, static_cast(offset)); - offset += 4; - if (offset + rep_len > decompressed_combined.size()) { - std::cout << "ERROR: Decompressed payload too small for rep level bytes" << std::endl; - return false; - } - auto rep_bytes = std::vector(decompressed_combined.begin() + static_cast(offset), - decompressed_combined.begin() + static_cast(offset + rep_len)); - offset += rep_len; - if (offset + 4 > decompressed_combined.size()) { std::cout << "ERROR: Decompressed payload too small for def level length" << std::endl; return false; } uint32_t def_len = read_u32_le(decompressed_combined, static_cast(offset)); offset += 4; + if (def_len != 2) { + std::cout << "ERROR: Unexpected definition-level payload length" << std::endl; + std::cout << " Expected def_len: 2" << std::endl; + std::cout << " Got def_len: " << def_len << std::endl; + return false; + } if (offset + def_len > decompressed_combined.size()) { std::cout << "ERROR: Decompressed payload too small for def level bytes" << std::endl; return false; @@ -718,8 +715,19 @@ class DBPARemoteTestApp { decompressed_combined.begin() + static_cast(offset + def_len)); offset += def_len; - if (rep_bytes != std::vector{0xAA, 0xAA, 0xAA} || def_bytes != std::vector{0xBB, 0xBB, 0xBB, 0xBB, 0xBB}) { - std::cout << "ERROR: Level bytes content mismatch" << std::endl; + // Expected def-level payload for this demo: + // - 0x06: RLE run header => run_len = 3 + // - 0x01: repeated def level value 1 (present) + if (def_bytes != std::vector{0x06, 0x01}) { + std::cout << "ERROR: Definition-level bytes content mismatch" << std::endl; + return false; + } + + const size_t expected_total_decompressed_size = 4u + 2u + fixed_length_data.size(); + if (decompressed_combined.size() != expected_total_decompressed_size) { + std::cout << "ERROR: Unexpected decompressed payload size" << std::endl; + std::cout << " Expected size: " << expected_total_decompressed_size << std::endl; + std::cout << " Got size: " << decompressed_combined.size() << std::endl; return false; } @@ -729,6 +737,12 @@ class DBPARemoteTestApp { } auto value_bytes = std::vector(decompressed_combined.begin() + static_cast(offset), decompressed_combined.end()); + if (offset != 6u) { + std::cout << "ERROR: Unexpected value-byte offset after level bytes" << std::endl; + std::cout << " Expected offset: 6" << std::endl; + std::cout << " Got offset: " << offset << std::endl; + return false; + } std::cout << "Decompressed value size: " << value_bytes.size() << " bytes" << std::endl; // Verify data integrity @@ -760,7 +774,7 @@ class DBPARemoteTestApp { try { auto empty_data = BuildByteArrayValueBytesForTesting(""); auto compressed_empty = Compress(empty_data, CompressionCodec::SNAPPY); - std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}}; + std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}; auto result = agent_->Encrypt(span(compressed_empty), encoding_attributes); if (result && result->success()) { @@ -780,7 +794,7 @@ class DBPARemoteTestApp { std::string large_data(1000, 'X'); // 1KB of data auto plaintext = BuildByteArrayValueBytesForTesting(large_data); auto compressed_plaintext = Compress(plaintext, CompressionCodec::SNAPPY); - std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}}; + std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}; auto result = agent_->Encrypt(span(compressed_plaintext), encoding_attributes); if (result && result->success()) { diff --git a/src/scripts/performance_test.cpp b/src/scripts/performance_test.cpp index ef71311..772b4be 100644 --- a/src/scripts/performance_test.cpp +++ b/src/scripts/performance_test.cpp @@ -173,20 +173,26 @@ namespace { size_t num_values, CompressionCodec::type compression_type, const std::string& page_encoding, - int32_t max_definition_level = 1, + int32_t max_definition_level = 0, int32_t max_repetition_level = 2, uint32_t definition_level_block_length = 3, uint32_t repetition_level_block_length = 5) { DataPageBuildResult result; result.level_bytes.clear(); - append_u32_le(result.level_bytes, repetition_level_block_length); // repetition level block length - result.level_bytes.insert(result.level_bytes.end(), - repetition_level_block_length, - 0xAA); - append_u32_le(result.level_bytes, definition_level_block_length); // definition level block length - result.level_bytes.insert(result.level_bytes.end(), - definition_level_block_length, - 0xBB); + + // V1 level bytes are emitted only when the corresponding max level is > 0. + if (max_repetition_level > 0) { + append_u32_le(result.level_bytes, repetition_level_block_length); // repetition level block length + result.level_bytes.insert(result.level_bytes.end(), + repetition_level_block_length, + 0xAA); + } + if (max_definition_level > 0) { + append_u32_le(result.level_bytes, definition_level_block_length); // definition level block length + result.level_bytes.insert(result.level_bytes.end(), + definition_level_block_length, + 0xBB); + } auto combined_uncompressed = Join(result.level_bytes, value_bytes); result.payload = Compress(combined_uncompressed, compression_type); result.attrs = { @@ -204,6 +210,7 @@ namespace { DataPageBuildResult BuildDictionaryPagePayload( const std::vector& value_bytes, + size_t num_values, CompressionCodec::type compression_type, const std::string& page_encoding) { DataPageBuildResult result; @@ -211,7 +218,8 @@ namespace { result.payload = Compress(value_bytes, compression_type); result.attrs = { {"page_type", "DICTIONARY_PAGE"}, - {"page_encoding", page_encoding} + {"page_encoding", page_encoding}, + {"dict_page_num_values", std::to_string(num_values)} }; return result; } @@ -297,6 +305,7 @@ class DBPALocalTestApp { } else if (scenario.page_type == "DICTIONARY_PAGE") { page = BuildDictionaryPagePayload( value_bytes, + num_values, scenario.compression, scenario.page_encoding); } else { From 2d7689a724fc9849cf1e10350ba023ca3e7406fb Mon Sep 17 00:00:00 2001 From: Alejandro Valerio Date: Fri, 13 Mar 2026 09:06:49 -0600 Subject: [PATCH 12/18] - Propagate num_elements Parquet-based to TypedValuesBuffer and BasicXorEncryptor. - Now num_elements is an invariant / constant all along TypedValuesBuffer and BasicXorEncryptor (Yay!) - Fixed issue on iterator to account for prefix size on input buffers. - Many unittest fixes for interfaces updates for num_elements. --- src/processing/encryption_sequencer.cpp | 3 +- src/processing/encryption_sequencer_test.cpp | 2 +- .../encryptors/basic_xor_encryptor.cpp | 51 +++--- .../encryptors/basic_xor_encryptor_test.cpp | 6 +- src/processing/parquet_utils.cpp | 18 +- src/processing/parquet_utils.h | 1 + src/processing/parquet_utils_test.cpp | 35 ++-- src/processing/typed_buffer.h | 158 ++++++++++-------- src/processing/typed_buffer_test.cpp | 86 +++++----- src/processing/typed_buffer_values_test.cpp | 54 +++--- src/scripts/dbpa_remote_testapp.cpp | 5 +- 11 files changed, 221 insertions(+), 198 deletions(-) diff --git a/src/processing/encryption_sequencer.cpp b/src/processing/encryption_sequencer.cpp index 6636fcb..268743b 100644 --- a/src/processing/encryption_sequencer.cpp +++ b/src/processing/encryption_sequencer.cpp @@ -181,7 +181,8 @@ bool DataBatchEncryptionSequencer::DecodeAndEncrypt(tcb::span pla // Parse value bytes into typed values buffer stage_start = std::chrono::steady_clock::now(); - auto typed_buffer = ReinterpretValueBytesAsTypedValuesBuffer(value_bytes, datatype_, datatype_length_, encoding_); + auto typed_buffer = ReinterpretValueBytesAsTypedValuesBuffer( + value_bytes, num_elements, datatype_, datatype_length_, encoding_); parse_value_bytes_into_typed_list_ns = std::chrono::duration_cast( std::chrono::steady_clock::now() - stage_start).count(); diff --git a/src/processing/encryption_sequencer_test.cpp b/src/processing/encryption_sequencer_test.cpp index 0342ab0..9af024e 100644 --- a/src/processing/encryption_sequencer_test.cpp +++ b/src/processing/encryption_sequencer_test.cpp @@ -484,7 +484,7 @@ TEST(EncryptionSequencer, FixedLenByteArrayValidation) { EXPECT_TRUE(testValidationFailure(0, "FIXED_LEN_BYTE_ARRAY datatype_length must be positive")); // Test valid case (should pass parameter validation) - DataBatchEncryptionSequencer sequencer("test_column", Type::FIXED_LEN_BYTE_ARRAY, 16, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key_123", "test_user", "{}", {}); + DataBatchEncryptionSequencer sequencer("test_column", Type::FIXED_LEN_BYTE_ARRAY, 16, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "2"}}, CompressionCodec::UNCOMPRESSED, "test_key_123", "test_user", "{}", {}); bool result = sequencer.DecodeAndEncrypt(FIXED_LEN_BYTE_ARRAY_DATA); if (!result && sequencer.error_stage_ == "parameter_validation") { diff --git a/src/processing/encryptors/basic_xor_encryptor.cpp b/src/processing/encryptors/basic_xor_encryptor.cpp index ca8e932..e003b62 100644 --- a/src/processing/encryptors/basic_xor_encryptor.cpp +++ b/src/processing/encryptors/basic_xor_encryptor.cpp @@ -196,7 +196,7 @@ std::vector BasicXorEncryptor::EncryptTypedElements( constexpr bool is_fixed = TypedBuffer::is_fixed_sized; constexpr size_t prefix_length = is_fixed ? kFixedHeaderLength : kVariableHeaderLength; - // const size_t num_elements = 500000; // +++++++++ input_buffer.GetNumElements(); + // GetNumElements is read from Parquet metadata and level bytes, not calculated from payload. const size_t num_elements = input_buffer.GetNumElements(); // If there are no elements, return an empty buffer with the header. @@ -229,15 +229,11 @@ std::vector BasicXorEncryptor::EncryptTypedElements( num_elements, prefix_length, RawBytesFixedSizedCodec{element_size}}; auto loop_start = std::chrono::steady_clock::now(); + size_t output_index = 0; + tcb::span raw_bytes; - // ++++++ Restore the while (auto rawbytes = input_buffer.ElementsIteratorNext()) {} form. - while (true) { - // auto t0 = std::chrono::steady_clock::now(); - tcb::span raw_bytes; - if (!input_buffer.ElementsIteratorNext(raw_bytes)) break; - // get_raw_element_ns += ElapsedNanosecondsSince(t0); - + while (input_buffer.ElementsIteratorNext(raw_bytes)) { // auto t1 = std::chrono::steady_clock::now(); auto write_span = output_buffer.GetWritableRawElement(output_index, raw_bytes.size()); // get_writable_element_ns += ElapsedNanosecondsSince(t1); @@ -262,15 +258,14 @@ std::vector BasicXorEncryptor::EncryptTypedElements( num_elements, reserved_bytes_hint, true, prefix_length}; auto loop_start = std::chrono::steady_clock::now(); - size_t output_index = 0; - // ++++++ Restore the while (auto rawbytes = input_buffer.ElementsIteratorNext()) {} form. - while (true) { - // auto t0 = std::chrono::steady_clock::now(); - tcb::span raw_bytes; - if (!input_buffer.ElementsIteratorNext(raw_bytes)) break; - // get_raw_element_ns += ElapsedNanosecondsSince(t0); + size_t output_index = 0; + tcb::span raw_bytes; + // ++++++ REMOVE ALL EXTRA INSTRUMENTATION AND TIMING CODE. + // - CAREFUL WITH NOT REMOVING CORRECT INLINED COMMENTS !!!! + + while (input_buffer.ElementsIteratorNext(raw_bytes)) { // auto t1 = std::chrono::steady_clock::now(); auto write_span = output_buffer.GetWritableRawElement(output_index, raw_bytes.size()); // get_writable_element_ns += ElapsedNanosecondsSince(t1); @@ -344,12 +339,11 @@ template TypedBuffer BasicXorEncryptor::DecryptFixedSizedElementsIntoTypedBuffer( const TypedBufferRawBytesFixedSized& encrypted_buffer, TypedBuffer output_buffer) { size_t output_index = 0; - tcb::span raw_bytes; - for (auto raw_bytes : encrypted_buffer.raw_elements()) { - // +++++++ new iterator not working? - // +++++++ while (encrypted_buffer.ElementsIteratorNext(raw_bytes)) { - auto write_span = output_buffer.GetWritableRawElement(output_index, raw_bytes.size()); - XorDecryptInto(raw_bytes, write_span); + tcb::span element_bytes; + size_t element_size = encrypted_buffer.GetElementSize(); + while (encrypted_buffer.ElementsIteratorNext(element_bytes)) { + auto write_span = output_buffer.GetWritableRawElement(output_index, element_size); + XorDecryptInto(element_bytes, write_span); output_index++; } return output_buffer; @@ -366,7 +360,7 @@ TypedValuesBuffer BasicXorEncryptor::DecryptValueList( // Create a fixed-sized byte buffer for reading the encrypted elements. TypedBufferRawBytesFixedSized encrypted_buffer{ - encrypted_bytes, kFixedHeaderLength, RawBytesFixedSizedCodec{header.element_size}}; + encrypted_bytes, num_elements, kFixedHeaderLength, RawBytesFixedSizedCodec{header.element_size}}; // Populate a typed buffer with the decrypted elements in the corresponding type. switch (datatype_) { @@ -399,21 +393,18 @@ TypedValuesBuffer BasicXorEncryptor::DecryptValueList( // Decrypt variable-size elements else { // Create a variable-sized byte buffer for reading the encrypted elements. - TypedBufferRawBytesVariableSized encrypted_buffer{ encrypted_bytes, kVariableHeaderLength}; + TypedBufferRawBytesVariableSized encrypted_buffer{ encrypted_bytes, num_elements, kVariableHeaderLength}; switch (datatype_) { // Create a BYTE-ARRAY typed buffer for storing the decrypted elements. case Type::BYTE_ARRAY: { auto reserved_bytes_hint = encrypted_buffer.GetRawBufferSize(); TypedBufferRawBytesVariableSized output_buffer{num_elements, reserved_bytes_hint, true}; - size_t output_index = 0; - tcb::span element; - for (auto element : encrypted_buffer.raw_elements()) { - // +++++++ new iterator not working? - // +++++++ while (encrypted_buffer.ElementsIteratorNext(element)) { - auto write_span = output_buffer.GetWritableRawElement(output_index, element.size()); - XorDecryptInto(element, write_span); + tcb::span element_bytes; + while (encrypted_buffer.ElementsIteratorNext(element_bytes)) { + auto write_span = output_buffer.GetWritableRawElement(output_index, element_bytes.size()); + XorDecryptInto(element_bytes, write_span); output_index++; } return output_buffer; diff --git a/src/processing/encryptors/basic_xor_encryptor_test.cpp b/src/processing/encryptors/basic_xor_encryptor_test.cpp index e5e7322..43c285b 100644 --- a/src/processing/encryptors/basic_xor_encryptor_test.cpp +++ b/src/processing/encryptors/basic_xor_encryptor_test.cpp @@ -71,7 +71,7 @@ TEST(BasicXorEncryptor, EncryptDecryptValueList_RoundTrip_INT32) { // re-wrap bytes as a read buffer to match production read path behavior. std::vector input_buffer_bytes = input_buffer_write.FinalizeAndTakeBuffer(); const auto input_span = tcb::span(input_buffer_bytes.data(), input_buffer_bytes.size()); - TypedBufferI32 input_buffer_read{input_span}; + TypedBufferI32 input_buffer_read{input_span, values.size()}; TypedValuesBuffer typed_buffer = std::move(input_buffer_read); std::vector encrypted_blob = encryptor.EncryptValueList(typed_buffer); @@ -99,7 +99,7 @@ TEST(BasicXorEncryptor, EncryptDecryptValueList_RoundTrip_DOUBLE) { // re-wrap bytes as a read buffer to match production read path behavior. std::vector input_buffer_bytes = input_buffer_write.FinalizeAndTakeBuffer(); const auto input_span = tcb::span(input_buffer_bytes.data(), input_buffer_bytes.size()); - TypedBufferDouble input_buffer_read{input_span}; + TypedBufferDouble input_buffer_read{input_span, values.size()}; TypedValuesBuffer typed_buffer = std::move(input_buffer_read); std::vector encrypted_blob = encryptor.EncryptValueList(typed_buffer); @@ -149,7 +149,7 @@ TEST(BasicXorEncryptor, EncryptDecryptValueList_RoundTrip_BYTE_ARRAY) { // re-wrap bytes as a read buffer to match production read path behavior. std::vector input_buffer_bytes = input_buffer_write.FinalizeAndTakeBuffer(); const auto input_span = tcb::span(input_buffer_bytes.data(), input_buffer_bytes.size()); - TypedBufferRawBytesVariableSized input_buffer_read{input_span}; + TypedBufferRawBytesVariableSized input_buffer_read{input_span, values.size()}; TypedValuesBuffer typed_buffer = std::move(input_buffer_read); std::vector encrypted_blob = encryptor.EncryptValueList(typed_buffer); diff --git a/src/processing/parquet_utils.cpp b/src/processing/parquet_utils.cpp index a963e20..a88fa60 100644 --- a/src/processing/parquet_utils.cpp +++ b/src/processing/parquet_utils.cpp @@ -390,7 +390,9 @@ std::vector CompressAndJoin( // Public functions to build Parquet formatted value bytes into TypedValuesBuffer // ----------------------------------------------------------------------------- -TypedValuesBuffer ReinterpretValueBytesAsTypedValuesBuffer(tcb::span value_bytes, +TypedValuesBuffer ReinterpretValueBytesAsTypedValuesBuffer( + tcb::span value_bytes, + size_t num_elements, Type::type datatype, const std::optional& datatype_length, Encoding::type encoding) { @@ -414,24 +416,24 @@ TypedValuesBuffer ReinterpretValueBytesAsTypedValuesBuffer(tcb::span(datatype_length.value())}}; + value_bytes, num_elements, 0, RawBytesFixedSizedCodec{static_cast(datatype_length.value())}}; } case Type::BYTE_ARRAY: - return TypedBufferRawBytesVariableSized{value_bytes}; + return TypedBufferRawBytesVariableSized{value_bytes, num_elements}; default: throw InvalidInputException( "Invalid datatype: " + std::string(to_string(datatype))); diff --git a/src/processing/parquet_utils.h b/src/processing/parquet_utils.h index 8aba317..f6a6bd6 100644 --- a/src/processing/parquet_utils.h +++ b/src/processing/parquet_utils.h @@ -113,6 +113,7 @@ std::vector CompressAndJoin( */ dbps::processing::TypedValuesBuffer ReinterpretValueBytesAsTypedValuesBuffer( tcb::span value_bytes, + size_t num_elements, Type::type datatype, const std::optional& datatype_length, Encoding::type encoding); diff --git a/src/processing/parquet_utils_test.cpp b/src/processing/parquet_utils_test.cpp index bc2be69..d92638b 100644 --- a/src/processing/parquet_utils_test.cpp +++ b/src/processing/parquet_utils_test.cpp @@ -657,7 +657,7 @@ TEST(ParquetUtils, Reinterpret_INT32) { reinterpret_cast(values.data()) + values.size() * sizeof(int32_t)); TypedValuesBuffer result = ReinterpretValueBytesAsTypedValuesBuffer( - bytes, Type::INT32, std::nullopt, Encoding::PLAIN); + bytes, values.size(), Type::INT32, std::nullopt, Encoding::PLAIN); auto* buf = std::get_if(&result); ASSERT_NE(nullptr, buf); @@ -676,7 +676,7 @@ TEST(ParquetUtils, Reinterpret_DOUBLE) { reinterpret_cast(values.data()) + values.size() * sizeof(double)); TypedValuesBuffer result = ReinterpretValueBytesAsTypedValuesBuffer( - bytes, Type::DOUBLE, std::nullopt, Encoding::PLAIN); + bytes, values.size(), Type::DOUBLE, std::nullopt, Encoding::PLAIN); auto* buf = std::get_if(&result); ASSERT_NE(nullptr, buf); @@ -704,7 +704,7 @@ TEST(ParquetUtils, Reinterpret_INT96) { } TypedValuesBuffer result = ReinterpretValueBytesAsTypedValuesBuffer( - bytes, Type::INT96, std::nullopt, Encoding::PLAIN); + bytes, expected.size(), Type::INT96, std::nullopt, Encoding::PLAIN); auto* buf = std::get_if(&result); ASSERT_NE(nullptr, buf); @@ -736,7 +736,7 @@ TEST(ParquetUtils, Reinterpret_BYTE_ARRAY) { } TypedValuesBuffer result = ReinterpretValueBytesAsTypedValuesBuffer( - bytes, Type::BYTE_ARRAY, std::nullopt, Encoding::PLAIN); + bytes, expected.size(), Type::BYTE_ARRAY, std::nullopt, Encoding::PLAIN); auto* buf = std::get_if(&result); ASSERT_NE(nullptr, buf); @@ -764,7 +764,7 @@ TEST(ParquetUtils, Reinterpret_FIXED_LEN_BYTE_ARRAY) { } TypedValuesBuffer result = ReinterpretValueBytesAsTypedValuesBuffer( - bytes, Type::FIXED_LEN_BYTE_ARRAY, element_len, Encoding::PLAIN); + bytes, expected.size(), Type::FIXED_LEN_BYTE_ARRAY, element_len, Encoding::PLAIN); auto* buf = std::get_if(&result); ASSERT_NE(nullptr, buf); @@ -780,28 +780,28 @@ TEST(ParquetUtils, Reinterpret_FIXED_LEN_BYTE_ARRAY) { TEST(ParquetUtils, Reinterpret_UnsupportedEncoding) { std::vector bytes = {0x01, 0x02, 0x03, 0x04}; EXPECT_THROW( - ReinterpretValueBytesAsTypedValuesBuffer(bytes, Type::INT32, std::nullopt, Encoding::RLE), + ReinterpretValueBytesAsTypedValuesBuffer(bytes, 1u, Type::INT32, std::nullopt, Encoding::RLE), DBPSUnsupportedException); } TEST(ParquetUtils, Reinterpret_RLE_DICTIONARY_Throws) { std::vector bytes = {0x01, 0x02, 0x03, 0x04}; EXPECT_THROW( - ReinterpretValueBytesAsTypedValuesBuffer(bytes, Type::INT32, std::nullopt, Encoding::RLE_DICTIONARY), + ReinterpretValueBytesAsTypedValuesBuffer(bytes, 1u, Type::INT32, std::nullopt, Encoding::RLE_DICTIONARY), DBPSUnsupportedException); } TEST(ParquetUtils, Reinterpret_BOOLEAN_Throws) { std::vector bytes = {0xB4}; EXPECT_THROW( - ReinterpretValueBytesAsTypedValuesBuffer(bytes, Type::BOOLEAN, std::nullopt, Encoding::PLAIN), + ReinterpretValueBytesAsTypedValuesBuffer(bytes, 1u, Type::BOOLEAN, std::nullopt, Encoding::PLAIN), DBPSUnsupportedException); } TEST(ParquetUtils, Reinterpret_InvalidDataSize) { std::vector bytes = {0xAA, 0xBB, 0xCC}; auto result = ReinterpretValueBytesAsTypedValuesBuffer( - bytes, Type::INT32, std::nullopt, Encoding::PLAIN); + bytes, 1u, Type::INT32, std::nullopt, Encoding::PLAIN); auto* buf = std::get_if(&result); ASSERT_NE(nullptr, buf); @@ -813,28 +813,29 @@ TEST(ParquetUtils, Reinterpret_InvalidDataSize) { TEST(ParquetUtils, Reinterpret_FIXED_LEN_BYTE_ARRAY_MissingLength_Throws) { std::vector bytes = {0x01, 0x02, 0x03}; EXPECT_THROW( - ReinterpretValueBytesAsTypedValuesBuffer(bytes, Type::FIXED_LEN_BYTE_ARRAY, std::nullopt, Encoding::PLAIN), + ReinterpretValueBytesAsTypedValuesBuffer( + bytes, 1u, Type::FIXED_LEN_BYTE_ARRAY, std::nullopt, Encoding::PLAIN), InvalidInputException); } TEST(ParquetUtils, Reinterpret_FIXED_LEN_BYTE_ARRAY_ZeroLength_Throws) { std::vector bytes = {0x01, 0x02, 0x03}; EXPECT_THROW( - ReinterpretValueBytesAsTypedValuesBuffer(bytes, Type::FIXED_LEN_BYTE_ARRAY, 0, Encoding::PLAIN), + ReinterpretValueBytesAsTypedValuesBuffer(bytes, 1u, Type::FIXED_LEN_BYTE_ARRAY, 0, Encoding::PLAIN), InvalidInputException); } TEST(ParquetUtils, Reinterpret_FIXED_LEN_BYTE_ARRAY_NegativeLength_Throws) { std::vector bytes = {0x01, 0x02, 0x03}; EXPECT_THROW( - ReinterpretValueBytesAsTypedValuesBuffer(bytes, Type::FIXED_LEN_BYTE_ARRAY, -1, Encoding::PLAIN), + ReinterpretValueBytesAsTypedValuesBuffer(bytes, 1u, Type::FIXED_LEN_BYTE_ARRAY, -1, Encoding::PLAIN), InvalidInputException); } TEST(ParquetUtils, Reinterpret_EmptyBytes_FixedSize) { std::vector bytes; TypedValuesBuffer result = ReinterpretValueBytesAsTypedValuesBuffer( - bytes, Type::DOUBLE, std::nullopt, Encoding::PLAIN); + bytes, 0u, Type::DOUBLE, std::nullopt, Encoding::PLAIN); auto* buf = std::get_if(&result); ASSERT_NE(nullptr, buf); @@ -846,7 +847,7 @@ TEST(ParquetUtils, Reinterpret_EmptyBytes_FixedSize) { TEST(ParquetUtils, Reinterpret_EmptyBytes_VariableSize) { std::vector bytes; TypedValuesBuffer result = ReinterpretValueBytesAsTypedValuesBuffer( - bytes, Type::BYTE_ARRAY, std::nullopt, Encoding::PLAIN); + bytes, 0u, Type::BYTE_ARRAY, std::nullopt, Encoding::PLAIN); auto* buf = std::get_if(&result); ASSERT_NE(nullptr, buf); @@ -866,7 +867,7 @@ TEST(ParquetUtils, RoundTrip_INT32) { reinterpret_cast(values.data()) + values.size() * sizeof(int32_t)); auto read_buf = ReinterpretValueBytesAsTypedValuesBuffer( - input_bytes, Type::INT32, std::nullopt, Encoding::PLAIN); + input_bytes, values.size(), Type::INT32, std::nullopt, Encoding::PLAIN); auto* src = std::get_if(&read_buf); ASSERT_NE(nullptr, src); @@ -900,7 +901,7 @@ TEST(ParquetUtils, RoundTrip_BYTE_ARRAY) { } auto read_buf = ReinterpretValueBytesAsTypedValuesBuffer( - input_bytes, Type::BYTE_ARRAY, std::nullopt, Encoding::PLAIN); + input_bytes, payloads.size(), Type::BYTE_ARRAY, std::nullopt, Encoding::PLAIN); auto* src = std::get_if(&read_buf); ASSERT_NE(nullptr, src); @@ -931,7 +932,7 @@ TEST(ParquetUtils, RoundTrip_FIXED_LEN_BYTE_ARRAY) { } auto read_buf = ReinterpretValueBytesAsTypedValuesBuffer( - input_bytes, Type::FIXED_LEN_BYTE_ARRAY, element_len, Encoding::PLAIN); + input_bytes, payloads.size(), Type::FIXED_LEN_BYTE_ARRAY, element_len, Encoding::PLAIN); auto* src = std::get_if(&read_buf); ASSERT_NE(nullptr, src); diff --git a/src/processing/typed_buffer.h b/src/processing/typed_buffer.h index aa73e87..7cf61c5 100644 --- a/src/processing/typed_buffer.h +++ b/src/processing/typed_buffer.h @@ -47,6 +47,7 @@ class ByteBuffer { // Elements are stored contiguously in the span. ByteBuffer( tcb::span elements_span, + size_t num_elements, size_t prefix_size = 0, Codec codec = Codec{}); @@ -72,11 +73,9 @@ class ByteBuffer { void SetRawElement(size_t position, tcb::span raw); // Getters for immediately available properties. + size_t GetNumElements() const { return num_elements_; } + size_t GetElementSize() const { return element_size_; } size_t GetRawBufferSize() const { return elements_span_size_; } - size_t GetElementSize() const { return codec_.element_size(); } - - // Get the number of elements in the buffer. - size_t GetNumElements() const; // Finalizes the write path and transfers the resulting buffer ownership. std::vector FinalizeAndTakeBuffer(); @@ -138,9 +137,6 @@ class ByteBuffer { // Helper for reserve heuristics in variable-size parsing. static size_t EstimateOffsetsReserveCountFromSample(tcb::span bytes); - // Helper for calculating the offset of an element by position. - size_t CalculateOffsetOfElement(size_t position) const; - // ++++ Needed after deprecating std iterators? // Helper to validate the preconditions for reading the buffer with an iterator. void ValidateIteratorReadPreconditions() const; @@ -151,7 +147,7 @@ class ByteBuffer { // Variables for span elements reading tcb::span elements_span_; size_t elements_span_size_; - mutable size_t num_elements_; + const size_t num_elements_; Codec codec_; // Variables for element span iterator. @@ -160,8 +156,8 @@ class ByteBuffer { mutable size_t element_iterator_count_ = 0; // Variables for determining offset of elements. - size_t prefix_size_ = 0; - size_t element_size_; // for fixed-size elements + const size_t prefix_size_ = 0; + const size_t element_size_; // for fixed-size elements mutable std::vector offsets_; // for variable-size elements // Variables for write buffer. @@ -176,6 +172,7 @@ class ByteBuffer { // Initialization methods and flags for read-only buffer void InitializeFromSpan() const; void EnsureInitializedFromSpan() const; + static size_t InitElementSize(const Codec& codec); mutable bool is_initialized_from_span_ = false; // Initialization methods and flags for write buffer @@ -203,26 +200,34 @@ inline size_t ReadSizeAt(tcb::span bytes, size_t offset) { template ByteBuffer::ByteBuffer( tcb::span elements_span, + size_t num_elements, size_t prefix_size, Codec codec) : elements_span_(elements_span), elements_span_size_(elements_span.size()), - element_iterator_current_ptr_(elements_span.data()), - element_iterator_end_ptr_(elements_span.data() + elements_span.size()), - element_iterator_count_(0), - num_elements_(kUnsetSize), + num_elements_(num_elements), codec_(std::move(codec)), - element_size_(0), + element_iterator_current_ptr_( + elements_span.data() + std::min(prefix_size, elements_span_size_)), + element_iterator_end_ptr_(elements_span.data() + elements_span_size_), + element_iterator_count_(0), prefix_size_(prefix_size), + element_size_(InitElementSize(codec_)), is_write_buffer_enabled_(false), - is_initialized_from_span_(false) { + is_initialized_from_span_(false) {} + +// Initialize element size based on the codec. Wrapper just needed for readability, all resolves in compile-time. +template +inline size_t ByteBuffer::InitElementSize(const Codec& codec) { if constexpr (is_fixed_sized) { - element_size_ = codec_.element_size(); + return codec.element_size(); } + // For variable-size elements, element size is undefined. + return kUnsetSize; } -// Initializes `num_elements_` and `offsets_` from the span. -// Called in a lazy manner when the buffer is accessed with GetElement or GetNumElements, avoiding unnecessary initialization. +// Initializes `offsets_` from the span. +// Called in a lazy manner when the buffer is accessed with GetElement[i] avoiding unnecessary initialization. template inline void ByteBuffer::InitializeFromSpan() const { if (elements_span_size_ < prefix_size_) { @@ -233,7 +238,6 @@ inline void ByteBuffer::InitializeFromSpan() const { // No elements to index. Initialize with empty values. if (readable_size == 0) { - num_elements_ = 0; offsets_.clear(); is_initialized_from_span_ = true; return; @@ -248,12 +252,13 @@ inline void ByteBuffer::InitializeFromSpan() const { if ((readable_size % element_size_) != 0) { throw InvalidInputException("Malformed fixed-size buffer: buffer does not align with element_size"); } - num_elements_ = readable_size / element_size_; - const size_t expected_payload_bytes = num_elements_ * element_size_; - if (expected_payload_bytes != readable_size) { - throw InvalidInputException( - "Malformed fixed-size buffer: computed payload size does not match readable size"); + + // Check if the num_elements passed at contruction time coincides with the calculated from the payload size. + const size_t num_elements_on_payload = readable_size / element_size_; + if (num_elements_on_payload != num_elements_) { + throw InvalidInputException("Malformed fixed-size buffer: num_elements on payload != num_elements_ expected."); } + offsets_.clear(); is_initialized_from_span_ = true; return; @@ -262,7 +267,7 @@ inline void ByteBuffer::InitializeFromSpan() const { // Variable-size layout stores [u32 size][element value] back-to-back. // Single pass validates shape and captures per-element prefix offsets. offsets_.clear(); - offsets_.reserve(EstimateOffsetsReserveCountFromSample(elements_span_.subspan(prefix_size_))); + offsets_.reserve(num_elements_); size_t cursor = prefix_size_; while (cursor < elements_span_size_) { if (elements_span_size_ - cursor < kSizePrefixBytes) { @@ -276,7 +281,13 @@ inline void ByteBuffer::InitializeFromSpan() const { } cursor += current_element_size; } - num_elements_ = offsets_.size(); + + // Check if the num_elements passed at contruction time coincides with the calculated from the payload size. + const size_t num_elements_on_payload = offsets_.size(); + if (num_elements_on_payload != num_elements_) { + throw InvalidInputException("Malformed variable-size buffer: num_elements on payload != num_elements_ expected."); + } + is_initialized_from_span_ = true; } @@ -293,19 +304,8 @@ inline void ByteBuffer::EnsureInitializedFromSpan() const { InitializeFromSpan(); } -// For read-only buffers, gets the number of elements in the buffer and sets num_elements_ if not already set. -// A lighter version to only get num_elements_ and avoid calling InitializeFromSpan that also builds offsets_. -template -inline size_t ByteBuffer::GetNumElements() const { - if (num_elements_ != kUnsetSize) { - return num_elements_; - } - - // If the buffer is not initialized, initialize it from the span. - EnsureInitializedFromSpan(); - return num_elements_; -} - +// ++++++ PROBABLY DEPRECATED NOW. MAY LEAVE IT FOR FUTURE USAGE. MARKE IT AS SO +++++++++ +// - Since num_elements_ is now known/an invariant, we no longer need this pretty method (sadly). template inline size_t ByteBuffer::EstimateOffsetsReserveCountFromSample(tcb::span bytes) { if (bytes.empty()) @@ -353,18 +353,6 @@ inline size_t ByteBuffer::EstimateOffsetsReserveCountFromSample(tcb::span // Element span reader methods // ----------------------------------------------------------------------------- -template -inline size_t ByteBuffer::CalculateOffsetOfElement(size_t position) const { - EnsureInitializedFromSpan(); - if (position >= num_elements_) { - throw InvalidInputException("Element position out of range during CalculateOffsetOfElement"); - } - if constexpr (is_fixed_sized) { - return prefix_size_ + (position * element_size_); - } - return offsets_[position]; -} - template inline tcb::span ByteBuffer::GetRawElement(size_t position) const { EnsureInitializedFromSpan(); @@ -372,7 +360,9 @@ inline tcb::span ByteBuffer::GetRawElement(size_t position // For fixed-size elements are stored contiguously. if constexpr (is_fixed_sized) { if (position >= num_elements_) { - throw InvalidInputException("Element position out of range during GetRawElement"); + throw InvalidInputException( + "Element index out of bounds during GetRawElement: index=" + std::to_string(position) + + " size=" + std::to_string(num_elements_)); } const size_t offset = prefix_size_ + (position * element_size_); return elements_span_.subspan(offset, element_size_); @@ -380,7 +370,9 @@ inline tcb::span ByteBuffer::GetRawElement(size_t position // For variable-size elements, we need to read the size first [u32 size][element]. if (position >= num_elements_) { - throw InvalidInputException("Element position out of range during GetRawElement"); + throw InvalidInputException( + "Element index out of bounds during GetRawElement: index=" + std::to_string(position) + + " size=" + std::to_string(num_elements_)); } const size_t offset = offsets_[position]; if (offset == kUnsetSize) { @@ -407,16 +399,25 @@ inline typename ByteBuffer::value_type ByteBuffer::GetElement(size // Element span iterator -- The streamlined version of the iterator. // ----------------------------------------------------------------------------- -// ++++++ Add validation that this is only used for read-only buffers. + template inline bool ByteBuffer::ElementsIteratorNext(tcb::span& raw_bytes) const { + + // Check that this is only used for read-only buffers. + if (is_write_buffer_enabled_) { + throw InvalidInputException("ElementsIteratorNext is only defined for read-only buffers"); + } + if (elements_span_size_ < prefix_size_) { + throw InvalidInputException("Malformed buffer: prefix_size exceeds span size"); + } + + // If the last element was consumed, check that the number of elements iterated matches the number of elements expected. if (element_iterator_current_ptr_ == element_iterator_end_ptr_) { - // ++++++++ RESTORE THIS WHEN WE POPULATE ALWAYS THE num_elements_ FIELD FROM PARQUET DIRECTLY. - // if (element_iterator_count_ != num_elements_) { - // throw InvalidInputException(std::string("Malformed buffer: element iterator count does not match num_elements. ") - // + "element_iterator_count_=" + std::to_string(element_iterator_count_) + ", " - // + "num_elements_=" + std::to_string(num_elements_)); - // } + if (element_iterator_count_ != num_elements_) { + throw InvalidInputException( + "Malformed buffer: iterator count mismatch: actual=" + std::to_string(element_iterator_count_) + + " expected=" + std::to_string(num_elements_)); + } raw_bytes = {}; return false; } @@ -462,6 +463,7 @@ inline bool ByteBuffer::ElementsIteratorNext(tcb::span& ra +// ++++++++++++++ STL ITERATORS ARE DEPRECATED. REMOVE THIS CODE. ++++++++++++++ // ----------------------------------------------------------------------------- // Element span iterator @@ -591,6 +593,22 @@ inline typename ByteBuffer::RawElementsView ByteBuffer::raw_elemen return RawElementsView{this}; } + + + + + + + + + + + + + + + + // ----------------------------------------------------------------------------- // Constructors and initializers for write buffer // ----------------------------------------------------------------------------- @@ -603,11 +621,10 @@ ByteBuffer::ByteBuffer( Codec codec) : num_elements_(num_elements), codec_(std::move(codec)), - element_size_(0), - is_write_buffer_enabled_(true), - prefix_size_(prefix_size) { + prefix_size_(prefix_size), + element_size_(InitElementSize(codec_)), + is_write_buffer_enabled_(true) { static_assert(is_fixed_sized, "ByteBuffer constructor for fixed-size elements only."); - element_size_ = codec_.element_size(); InitializeForWriteBuffer(0); } @@ -621,9 +638,9 @@ ByteBuffer::ByteBuffer( Codec codec) : num_elements_(num_elements), codec_(std::move(codec)), - element_size_(0), - is_write_buffer_enabled_(true), - prefix_size_(prefix_size) { + prefix_size_(prefix_size), + element_size_(InitElementSize(codec_)), + is_write_buffer_enabled_(true) { static_assert(!is_fixed_sized, "ByteBuffer constructor for variable-size elements only."); InitializeForWriteBuffer(use_reserve_hint ? reserved_bytes_hint : 0); } @@ -690,8 +707,9 @@ inline tcb::span ByteBuffer::GetWritableSpanForElement(size_t po } if (position >= num_elements_) { - throw InvalidInputException(std::string("Element position out of range during GetWriteSpanForElement: ") + - "position=" + std::to_string(position) + " num_elements_=" + std::to_string(num_elements_)); + throw InvalidInputException( + "Element index out of bounds during GetWriteSpanForElement: index=" + std::to_string(position) + + " size=" + std::to_string(num_elements_)); } // For fixed-size elements, we write directly at the fixed offset. No need to re-bind the span. diff --git a/src/processing/typed_buffer_test.cpp b/src/processing/typed_buffer_test.cpp index bea88a8..9819d7d 100644 --- a/src/processing/typed_buffer_test.cpp +++ b/src/processing/typed_buffer_test.cpp @@ -38,9 +38,10 @@ class TypedBufferTestProxy : public ByteBuffer { public: explicit TypedBufferTestProxy( tcb::span elements_span, + size_t num_elements, size_t prefix_size = 0, Codec codec = Codec{}) - : Base(elements_span, prefix_size, std::move(codec)) {} + : Base(elements_span, num_elements, prefix_size, std::move(codec)) {} TypedBufferTestProxy( size_t num_elements, @@ -96,7 +97,7 @@ std::vector MakePayload(size_t size, uint8_t seed) { TEST(TypedBufferTest, ConstructFixedSize_ValidBuffer_InitializesExpectedState) { std::vector bytes = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06}; - RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 0, RawBytesFixedSizedCodec{2}); + RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 3u, 0u, RawBytesFixedSizedCodec{2}); // Trigger lazy span initialization. EXPECT_NO_THROW((void)buffer.GetElement(0)); ExpectCommonState(buffer, 3u, 2u); @@ -105,7 +106,7 @@ TEST(TypedBufferTest, ConstructFixedSize_ValidBuffer_InitializesExpectedState) { TEST(TypedBufferTest, GetElement_FixedSize_ReturnsExpectedSlices) { std::vector bytes = {0x10, 0x11, 0x20, 0x21, 0x30, 0x31}; - RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 0, RawBytesFixedSizedCodec{2}); + RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 3u, 0u, RawBytesFixedSizedCodec{2}); const auto first = buffer.GetElement(0); const auto second = buffer.GetElement(1); @@ -127,7 +128,7 @@ TEST(TypedBufferTest, GetElement_FixedSize_WithPrefixSize_SkipsPrefix) { 0xFE, 0xFD, 0xFC, // prefix 0x10, 0x11, 0x20, 0x21, 0x30, 0x31 }; - RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 3u, RawBytesFixedSizedCodec{2u}); + RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 3u, 3u, RawBytesFixedSizedCodec{2u}); const auto first = buffer.GetElement(0); const auto second = buffer.GetElement(1); @@ -144,7 +145,7 @@ TEST(TypedBufferTest, GetElement_VariableSize_WithPrefixSize_SkipsPrefix) { 0x02, 0x00, 0x00, 0x00, 0x41, 0x42, 0x03, 0x00, 0x00, 0x00, 0x78, 0x79, 0x7A }; - ByteBuffer buffer(tcb::span(bytes), 2u); + ByteBuffer buffer(tcb::span(bytes), 2u, 2u); const auto first = buffer.GetElement(0); const auto second = buffer.GetElement(1); @@ -159,7 +160,7 @@ TEST(TypedBufferTest, ConstructFixedSize_ZeroElementSize_Throws) { TEST(TypedBufferTest, ConstructFixedSize_NonDivisibleSize_Throws) { std::vector bytes = {0x01, 0x02, 0x03}; - RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 0, RawBytesFixedSizedCodec{2}); + RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 2u, 0u, RawBytesFixedSizedCodec{2}); EXPECT_THROW((void)buffer.GetElement(0), InvalidInputException); } @@ -169,10 +170,10 @@ TEST(TypedBufferTest, ConstructVariableSize_ValidEncodedBuffer_InitializesExpect 0x05, 0x00, 0x00, 0x00, 0x41, 0x42, 0x43, 0x44, 0x45, 0x07, 0x00, 0x00, 0x00, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37 }; - RawBytesVariableSizedBuffer buffer{tcb::span(bytes)}; + RawBytesVariableSizedBuffer buffer{tcb::span(bytes), 2u}; // Trigger lazy variable-size index parsing. EXPECT_NO_THROW((void)buffer.GetElement(0)); - ExpectCommonState(buffer, 2u, 0u); + ExpectCommonState(buffer, 2u, kUnsetSize); ASSERT_EQ(buffer.GetOffsets().size(), 2u); EXPECT_EQ(buffer.GetOffsets()[0], 0u); EXPECT_EQ(buffer.GetOffsets()[1], 9u); // 4 bytes length prefix + 5 bytes first payload. @@ -238,7 +239,7 @@ TEST(TypedBufferTest, GetElement_VariableSize_ReturnsExpectedPayload) { 0x05, 0x00, 0x00, 0x00, 0x41, 0x42, 0x43, 0x44, 0x45, 0x07, 0x00, 0x00, 0x00, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37 }; - RawBytesVariableSizedBuffer buffer{tcb::span(bytes)}; + RawBytesVariableSizedBuffer buffer{tcb::span(bytes), 2u}; const auto first = buffer.GetElement(0); const auto second = buffer.GetElement(1); @@ -257,7 +258,7 @@ TEST(TypedBufferTest, Iterate_ReadOnlyVariableSize_WithPrefixSize_SkipsPrefix) { 0x02, 0x00, 0x00, 0x00, 0x41, 0x42, 0x03, 0x00, 0x00, 0x00, 0x78, 0x79, 0x7A }; - ByteBuffer buffer(tcb::span(bytes), 2u); + ByteBuffer buffer(tcb::span(bytes), 2u, 2u); std::vector> collected; for (const auto element : buffer) { @@ -274,7 +275,7 @@ TEST(TypedBufferTest, Iterate_ReadOnlyFixedSize_WithPrefixSize_SkipsPrefix) { 0xFE, 0xFD, 0xFC, // prefix 0x10, 0x11, 0x20, 0x21, 0x30, 0x31 }; - RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 3u, RawBytesFixedSizedCodec{2u}); + RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 3u, 3u, RawBytesFixedSizedCodec{2u}); std::vector> collected; for (const auto element : buffer) { @@ -289,7 +290,7 @@ TEST(TypedBufferTest, Iterate_ReadOnlyFixedSize_WithPrefixSize_SkipsPrefix) { TEST(TypedBufferTest, Iterate_ReadOnlyFixedSize_TraversesInOrder) { std::vector bytes = {0x10, 0x11, 0x20, 0x21, 0x30, 0x31}; - RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 0, RawBytesFixedSizedCodec{2}); + RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 3u, 0u, RawBytesFixedSizedCodec{2}); std::vector> collected; for (const auto element : buffer) { @@ -324,7 +325,7 @@ TEST(TypedBufferTest, Iterate_ReadOnlyVariableSize_TraversesInOrder) { 0x05, 0x00, 0x00, 0x00, 0x41, 0x42, 0x43, 0x44, 0x45, 0x07, 0x00, 0x00, 0x00, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37 }; - RawBytesVariableSizedBuffer buffer{tcb::span(bytes)}; + RawBytesVariableSizedBuffer buffer{tcb::span(bytes), 2u}; std::vector> collected; for (const auto element : buffer) { @@ -365,7 +366,8 @@ TEST(TypedBufferTest, TransformFixedSize_ReadIterateWriteFinalize_RoundTrip) { }; // Create a source buffer to read the elements from. - RawBytesFixedSizedBuffer source_buffer(tcb::span(source_bytes), 0, RawBytesFixedSizedCodec{kElementSize}); + RawBytesFixedSizedBuffer source_buffer( + tcb::span(source_bytes), 3u, 0u, RawBytesFixedSizedCodec{kElementSize}); // Create a new buffer to write the transformed elements into. RawBytesFixedSizedBuffer transformed_buffer(3u, 0, RawBytesFixedSizedCodec{kElementSize}); @@ -396,7 +398,8 @@ TEST(TypedBufferTest, TransformFixedSize_ReadIterateWriteFinalize_RoundTrip) { // Now finalize the transformed buffer and populate a third buffer to read the elements from. std::vector finalized_bytes = transformed_buffer.FinalizeAndTakeBuffer(); - RawBytesFixedSizedBuffer finalized_read_buffer(tcb::span(finalized_bytes), 0, RawBytesFixedSizedCodec{kElementSize}); + RawBytesFixedSizedBuffer finalized_read_buffer( + tcb::span(finalized_bytes), 3u, 0u, RawBytesFixedSizedCodec{kElementSize}); // Compare source and finalized read buffer using the same transformation rule. for (size_t i = 0; i < position; ++i) { @@ -421,7 +424,7 @@ TEST(TypedBufferTest, TransformVariableSize_ReadIterateWriteFinalize_RoundTrip) append_u32_le(source_bytes, 3u); source_bytes.insert(source_bytes.end(), {0x61, 0x62, 0x63}); // "abc" - RawBytesVariableSizedBuffer source_buffer{tcb::span(source_bytes)}; + RawBytesVariableSizedBuffer source_buffer{tcb::span(source_bytes), 3u}; RawBytesVariableSizedBuffer transformed_buffer(3u, source_bytes.size(), true); size_t position = 0; @@ -449,7 +452,7 @@ TEST(TypedBufferTest, TransformVariableSize_ReadIterateWriteFinalize_RoundTrip) } std::vector finalized_bytes = transformed_buffer.FinalizeAndTakeBuffer(); - RawBytesVariableSizedBuffer finalized_read_buffer{tcb::span(finalized_bytes)}; + RawBytesVariableSizedBuffer finalized_read_buffer{tcb::span(finalized_bytes), 3u}; // Compare source and finalized read buffer using the same transformation rule. for (size_t i = 0; i < position; ++i) { @@ -469,7 +472,7 @@ TEST(TypedBufferTest, Iterate_ReadOnlyEmptySpan_VisitsNoElements) { std::vector empty_bytes; // Fixed-size empty span. - RawBytesFixedSizedBuffer fixed_buffer(tcb::span(empty_bytes), 0, RawBytesFixedSizedCodec{2}); + RawBytesFixedSizedBuffer fixed_buffer(tcb::span(empty_bytes), 0u, 0u, RawBytesFixedSizedCodec{2}); size_t fixed_count = 0; for (const auto element : fixed_buffer) { (void)element; @@ -478,7 +481,7 @@ TEST(TypedBufferTest, Iterate_ReadOnlyEmptySpan_VisitsNoElements) { EXPECT_EQ(fixed_count, 0u); // Variable-size empty span. - RawBytesVariableSizedBuffer variable_buffer{tcb::span(empty_bytes)}; + RawBytesVariableSizedBuffer variable_buffer{tcb::span(empty_bytes), 0u}; size_t variable_count = 0; for (const auto element : variable_buffer) { (void)element; @@ -489,7 +492,7 @@ TEST(TypedBufferTest, Iterate_ReadOnlyEmptySpan_VisitsNoElements) { TEST(TypedBufferTest, GetElement_OutOfRange_Throws) { std::vector bytes = {0x01, 0x02, 0x03, 0x04}; - RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 0, RawBytesFixedSizedCodec{2}); + RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 2u, 0u, RawBytesFixedSizedCodec{2}); EXPECT_THROW((void)buffer.GetElement(2), InvalidInputException); } @@ -546,7 +549,7 @@ TEST(TypedBufferTest, ConstructWithNumElements_FixedSize_WithPrefixSize_Preserve TEST(TypedBufferTest, ConstructWithNumElements_VariableSize_AllocatesAndSets) { RawBytesVariableSizedBuffer buffer(2u, 8u, true); EXPECT_EQ(buffer.GetNumElements(), 2u); - EXPECT_EQ(buffer.GetElementSize(), 0u); + EXPECT_EQ(buffer.GetElementSize(), kUnsetSize); ASSERT_EQ(buffer.GetOffsets().size(), 2u); EXPECT_EQ(buffer.GetOffsets()[0], kUnsetSize); EXPECT_EQ(buffer.GetOffsets()[1], kUnsetSize); @@ -757,7 +760,7 @@ TEST(TypedBufferTest, FinalizeAndTakeBuffer_VariableSize_PartialWrite_ThrowsAndA std::vector final_buffer = buffer.FinalizeAndTakeBuffer(); EXPECT_NE(final_buffer.data(), data_ptr_before); // Different allocation (defragmented after retry). - RawBytesVariableSizedBuffer read_back{tcb::span(final_buffer)}; + RawBytesVariableSizedBuffer read_back{tcb::span(final_buffer), 2u}; const auto r0 = read_back.GetElement(0); const auto r1 = read_back.GetElement(1); ASSERT_EQ(r0.size(), first.size()); @@ -823,7 +826,7 @@ TEST(TypedBufferTest, FinalizeAndTakeBuffer_VariableSize_OutOfOrder_Defragments) EXPECT_NE(final_buffer, raw_before_finalize); // Different byte content. EXPECT_NE(final_buffer.data(), data_ptr_before); // Different allocation (defragmented copy). - RawBytesVariableSizedBuffer read_back{tcb::span(final_buffer)}; + RawBytesVariableSizedBuffer read_back{tcb::span(final_buffer), 3u}; const auto r0 = read_back.GetElement(0); const auto r1 = read_back.GetElement(1); const auto r2 = read_back.GetElement(2); @@ -860,7 +863,7 @@ TEST(TypedBufferTest, FinalizeAndTakeBuffer_VariableSize_Fragmented_Defragments) EXPECT_NE(final_buffer, raw_before_finalize); EXPECT_NE(final_buffer.data(), data_ptr_before); // Different allocation (defragmented copy). - RawBytesVariableSizedBuffer read_back{tcb::span(final_buffer)}; + RawBytesVariableSizedBuffer read_back{tcb::span(final_buffer), 2u}; const auto r0 = read_back.GetElement(0); const auto r1 = read_back.GetElement(1); ASSERT_EQ(r0.size(), first_overwrite.size()); @@ -938,7 +941,7 @@ TEST(TypedBufferTest, SetElement_FixedSize_WrongPayloadSize_Throws) { TEST(TypedBufferTest, SetElement_OnReadOnlyBuffer_Throws) { std::vector bytes = {0x10, 0x11, 0x20, 0x21}; - RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 0, RawBytesFixedSizedCodec{2}); + RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 2u, 0u, RawBytesFixedSizedCodec{2}); std::vector replacement = {0xAA, 0xBB}; EXPECT_THROW((void)buffer.SetElement(0, tcb::span(replacement)), InvalidInputException); } @@ -968,7 +971,8 @@ TEST(TypedBufferTest, ElementsIteratorNext_FixedSize_TraversesAllElements) { bytes.insert(bytes.end(), e.begin(), e.end()); } - RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 0, RawBytesFixedSizedCodec{kElementSize}); + RawBytesFixedSizedBuffer buffer( + tcb::span(bytes), expected.size(), 0u, RawBytesFixedSizedCodec{kElementSize}); std::vector> collected; tcb::span element; @@ -1004,7 +1008,7 @@ TEST(TypedBufferTest, ElementsIteratorNext_VariableSize_TraversesAllElements) { bytes.insert(bytes.end(), e.begin(), e.end()); } - RawBytesVariableSizedBuffer buffer{tcb::span(bytes)}; + RawBytesVariableSizedBuffer buffer{tcb::span(bytes), expected.size()}; std::vector> collected; tcb::span element; @@ -1030,18 +1034,18 @@ TEST(TypedBufferTest, ElementsIteratorNext_VariableSize_TraversesAllElements) { TEST(TypedBufferTest, GetSpanSize_ReturnsUnderlyingSpanByteCount) { std::vector fixed_bytes = {0x10, 0x11, 0x20, 0x21, 0x30, 0x31}; - RawBytesFixedSizedBuffer fixed_buffer(tcb::span(fixed_bytes), 0, RawBytesFixedSizedCodec{2}); + RawBytesFixedSizedBuffer fixed_buffer(tcb::span(fixed_bytes), 3u, 0u, RawBytesFixedSizedCodec{2}); EXPECT_EQ(fixed_buffer.GetRawBufferSize(), 6u); std::vector var_bytes = { 0x02, 0x00, 0x00, 0x00, 0x41, 0x42, 0x03, 0x00, 0x00, 0x00, 0x78, 0x79, 0x7A }; - RawBytesVariableSizedBuffer var_buffer{tcb::span(var_bytes)}; + RawBytesVariableSizedBuffer var_buffer{tcb::span(var_bytes), 2u}; EXPECT_EQ(var_buffer.GetRawBufferSize(), 13u); std::vector empty_bytes; - RawBytesFixedSizedBuffer empty_buffer(tcb::span(empty_bytes), 0, RawBytesFixedSizedCodec{2}); + RawBytesFixedSizedBuffer empty_buffer(tcb::span(empty_bytes), 0u, 0u, RawBytesFixedSizedCodec{2}); EXPECT_EQ(empty_buffer.GetRawBufferSize(), 0u); } @@ -1053,7 +1057,7 @@ TEST(TypedBufferTest, RawIterate_FixedSize_ReturnsRawBytesNotDecodedValues) { using dbps::processing::StringFixedSizedCodec; std::vector bytes = {'H', 'i', '!', 'B', 'y', 'e'}; ByteBuffer buffer( - tcb::span(bytes), 0, StringFixedSizedCodec{3u}); + tcb::span(bytes), 2u, 0u, StringFixedSizedCodec{3u}); std::vector decoded; for (const auto value : buffer) { @@ -1080,7 +1084,7 @@ TEST(TypedBufferTest, RawIterate_VariableSize_ReturnsRawBytesNotDecodedValues) { append_u32_le(bytes, 2u); bytes.insert(bytes.end(), {'O', 'K'}); - ByteBuffer buffer{tcb::span(bytes)}; + ByteBuffer buffer{tcb::span(bytes), 2u}; std::vector decoded; for (const auto value : buffer) { @@ -1101,8 +1105,8 @@ TEST(TypedBufferTest, RawIterate_VariableSize_ReturnsRawBytesNotDecodedValues) { TEST(TypedBufferTest, ConstructVariableSize_EmptyBuffer_InitializesEmptyState) { std::vector bytes; - RawBytesVariableSizedBuffer buffer{tcb::span(bytes)}; - ExpectCommonState(buffer, 0u, 0u); + RawBytesVariableSizedBuffer buffer{tcb::span(bytes), 0u}; + ExpectCommonState(buffer, 0u, kUnsetSize); EXPECT_TRUE(buffer.GetOffsets().empty()); } @@ -1112,9 +1116,11 @@ TEST(TypedBufferTest, GetNumElements_ReadOnlyVariable_WithPrefix) { 0x05, 0x00, 0x00, 0x00, 0x41, 0x42, 0x43, 0x44, 0x45, // len=5, payload="ABCDE" 0x07, 0x00, 0x00, 0x00, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37 // len=7, payload="1234567" }; - RawBytesVariableSizedBuffer buffer{tcb::span(bytes), 3u}; + RawBytesVariableSizedBuffer buffer{tcb::span(bytes), 2u, 3u}; EXPECT_EQ(buffer.GetNumElements(), 2u); + // Offsets are populated lazily on first element access. + EXPECT_NO_THROW((void)buffer.GetElement(0)); ASSERT_EQ(buffer.GetOffsets().size(), 2u); EXPECT_EQ(buffer.GetOffsets()[0], 3u); EXPECT_EQ(buffer.GetOffsets()[1], 12u); @@ -1126,7 +1132,7 @@ TEST(TypedBufferTest, GetNumElements_ReadOnlyFixed_WithPrefix) { 0xFE, 0xFD, 0xFC, 0x10, 0x11, 0x20, 0x21, 0x30, 0x31 }; - RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 3u, RawBytesFixedSizedCodec{2u}); + RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 3u, 3u, RawBytesFixedSizedCodec{2u}); EXPECT_EQ(buffer.GetNumElements(), 3u); // Fixed-size buffers never build offsets. @@ -1143,13 +1149,13 @@ TEST(TypedBufferTest, GetNumElements_WriteBuffer_ReturnsConstructorCount) { TEST(TypedBufferTest, GetNumElements_ReadOnly_PrefixExceedsSpan_Throws) { std::vector bytes = {0xAA, 0xBB}; - RawBytesVariableSizedBuffer buffer{tcb::span(bytes), 3u}; - EXPECT_THROW((void)buffer.GetNumElements(), InvalidInputException); + RawBytesVariableSizedBuffer buffer{tcb::span(bytes), 1u, 3u}; + EXPECT_THROW((void)buffer.GetElement(0), InvalidInputException); } TEST(TypedBufferTest, ConstructVariableSize_TruncatedLengthPrefix_Throws) { std::vector bytes = {0x01, 0x00, 0x00}; // only 3 bytes - RawBytesVariableSizedBuffer buffer{tcb::span(bytes)}; + RawBytesVariableSizedBuffer buffer{tcb::span(bytes), 1u}; EXPECT_THROW((void)buffer.GetElement(0), InvalidInputException); } @@ -1158,7 +1164,7 @@ TEST(TypedBufferTest, ConstructVariableSize_TruncatedPayload_Throws) { std::vector bytes = { 0x05, 0x00, 0x00, 0x00, 0xAA, 0xBB }; - RawBytesVariableSizedBuffer buffer{tcb::span(bytes)}; + RawBytesVariableSizedBuffer buffer{tcb::span(bytes), 1u}; EXPECT_THROW((void)buffer.GetElement(0), InvalidInputException); } diff --git a/src/processing/typed_buffer_values_test.cpp b/src/processing/typed_buffer_values_test.cpp index 38d6843..0329d0a 100644 --- a/src/processing/typed_buffer_values_test.cpp +++ b/src/processing/typed_buffer_values_test.cpp @@ -40,7 +40,7 @@ TEST(TypedBufferValuesTest, Int32_ReadBack) { append_i32_le(bytes, 0); append_i32_le(bytes, 2147483647); - TypedBufferI32 buffer{tcb::span(bytes)}; + TypedBufferI32 buffer{tcb::span(bytes), 4u}; EXPECT_EQ(buffer.GetElement(0), 42); EXPECT_EQ(buffer.GetElement(1), -1); @@ -67,7 +67,7 @@ TEST(TypedBufferValuesTest, Int32_WriteRoundTrip) { writer.SetElement(2, 30); std::vector finalized = writer.FinalizeAndTakeBuffer(); - TypedBufferI32 reader{tcb::span(finalized)}; + TypedBufferI32 reader{tcb::span(finalized), 3u}; EXPECT_EQ(reader.GetElement(0), 10); EXPECT_EQ(reader.GetElement(1), 20); @@ -80,7 +80,7 @@ TEST(TypedBufferValuesTest, Int32_Iterate) { append_i32_le(bytes, 10); append_i32_le(bytes, 15); - TypedBufferI32 buffer{tcb::span(bytes)}; + TypedBufferI32 buffer{tcb::span(bytes), 3u}; std::vector collected; for (const auto value : buffer) { @@ -105,7 +105,7 @@ TEST(TypedBufferValuesTest, Int32_OutOfOrderWrite_RoundTrip) { writer.SetElement(2, 30); std::vector finalized = writer.FinalizeAndTakeBuffer(); - TypedBufferI32 reader{tcb::span(finalized)}; + TypedBufferI32 reader{tcb::span(finalized), 4u}; EXPECT_EQ(reader.GetElement(0), 10); EXPECT_EQ(reader.GetElement(1), 20); @@ -125,7 +125,7 @@ TEST(TypedBufferValuesTest, Int32_OverwriteThenRoundTrip) { EXPECT_EQ(writer.GetElement(1), 222); std::vector finalized = writer.FinalizeAndTakeBuffer(); - TypedBufferI32 reader{tcb::span(finalized)}; + TypedBufferI32 reader{tcb::span(finalized), 2u}; EXPECT_EQ(reader.GetElement(0), 999); EXPECT_EQ(reader.GetElement(1), 222); @@ -159,7 +159,7 @@ TEST(TypedBufferValuesTest, Int96_ReadBack) { AppendInt96LE(bytes, b); AppendInt96LE(bytes, c); - TypedBufferInt96 buffer{tcb::span(bytes)}; + TypedBufferInt96 buffer{tcb::span(bytes), 3u}; ExpectInt96Eq(buffer.GetElement(0), a); ExpectInt96Eq(buffer.GetElement(1), b); @@ -175,7 +175,7 @@ TEST(TypedBufferValuesTest, Int96_WriteRoundTrip) { writer.SetElement(1, b); std::vector finalized = writer.FinalizeAndTakeBuffer(); - TypedBufferInt96 reader{tcb::span(finalized)}; + TypedBufferInt96 reader{tcb::span(finalized), 2u}; ExpectInt96Eq(reader.GetElement(0), a); ExpectInt96Eq(reader.GetElement(1), b); @@ -189,7 +189,7 @@ TEST(TypedBufferValuesTest, Int96_Iterate) { AppendInt96LE(bytes, a); AppendInt96LE(bytes, b); - TypedBufferInt96 buffer{tcb::span(bytes)}; + TypedBufferInt96 buffer{tcb::span(bytes), 2u}; std::vector collected; for (const auto value : buffer) { @@ -212,7 +212,7 @@ TEST(TypedBufferValuesTest, Int96_OutOfOrderWrite_RoundTrip) { writer.SetElement(1, b); std::vector finalized = writer.FinalizeAndTakeBuffer(); - TypedBufferInt96 reader{tcb::span(finalized)}; + TypedBufferInt96 reader{tcb::span(finalized), 3u}; ExpectInt96Eq(reader.GetElement(0), a); ExpectInt96Eq(reader.GetElement(1), b); @@ -229,7 +229,7 @@ TEST(TypedBufferValuesTest, Float_ReadBack) { append_f32_le(bytes, -0.0f); append_f32_le(bytes, 1.0e10f); - TypedBufferFloat buffer{tcb::span(bytes)}; + TypedBufferFloat buffer{tcb::span(bytes), 3u}; EXPECT_FLOAT_EQ(buffer.GetElement(0), 3.14f); EXPECT_FLOAT_EQ(buffer.GetElement(1), -0.0f); @@ -243,7 +243,7 @@ TEST(TypedBufferValuesTest, Float_WriteRoundTrip) { writer.SetElement(2, 0.0f); std::vector finalized = writer.FinalizeAndTakeBuffer(); - TypedBufferFloat reader{tcb::span(finalized)}; + TypedBufferFloat reader{tcb::span(finalized), 3u}; EXPECT_FLOAT_EQ(reader.GetElement(0), 1.5f); EXPECT_FLOAT_EQ(reader.GetElement(1), -2.25f); @@ -256,7 +256,7 @@ TEST(TypedBufferValuesTest, Float_Iterate) { append_f32_le(bytes, 2.0f); append_f32_le(bytes, 3.0f); - TypedBufferFloat buffer{tcb::span(bytes)}; + TypedBufferFloat buffer{tcb::span(bytes), 3u}; std::vector collected; for (const auto value : buffer) { @@ -284,7 +284,7 @@ TEST(TypedBufferValuesTest, Float_OutOfOrderWrite_InterleavedReads) { EXPECT_FLOAT_EQ(writer.GetElement(2), 99.0f); std::vector finalized = writer.FinalizeAndTakeBuffer(); - TypedBufferFloat reader{tcb::span(finalized)}; + TypedBufferFloat reader{tcb::span(finalized), 3u}; EXPECT_FLOAT_EQ(reader.GetElement(0), 10.0f); EXPECT_FLOAT_EQ(reader.GetElement(1), 20.0f); @@ -304,7 +304,7 @@ TEST(TypedBufferValuesTest, StringFixedSized_ReadBack) { }; TypedBufferStringFixedSized buffer( - tcb::span(bytes), 0, StringFixedSizedCodec{4}); + tcb::span(bytes), 3u, 0u, StringFixedSizedCodec{4}); EXPECT_EQ(buffer.GetElement(0), "ABCD"); EXPECT_EQ(buffer.GetElement(1), "EFGH"); @@ -320,7 +320,7 @@ TEST(TypedBufferValuesTest, StringFixedSized_WriteRoundTrip) { std::vector finalized = writer.FinalizeAndTakeBuffer(); TypedBufferStringFixedSized reader( - tcb::span(finalized), 0, StringFixedSizedCodec{4}); + tcb::span(finalized), 3u, 0u, StringFixedSizedCodec{4}); EXPECT_EQ(reader.GetElement(0), "AAAA"); EXPECT_EQ(reader.GetElement(1), "BBBB"); @@ -334,7 +334,7 @@ TEST(TypedBufferValuesTest, StringFixedSized_Iterate) { }; TypedBufferStringFixedSized buffer( - tcb::span(bytes), 0, StringFixedSizedCodec{4}); + tcb::span(bytes), 2u, 0u, StringFixedSizedCodec{4}); std::vector collected; for (const auto value : buffer) { @@ -361,7 +361,7 @@ TEST(TypedBufferValuesTest, StringFixedSized_OutOfOrderWrite_RoundTrip) { std::vector finalized = writer.FinalizeAndTakeBuffer(); TypedBufferStringFixedSized reader( - tcb::span(finalized), 0, StringFixedSizedCodec{3}); + tcb::span(finalized), 3u, 0u, StringFixedSizedCodec{3}); EXPECT_EQ(reader.GetElement(0), "ZZZ"); EXPECT_EQ(reader.GetElement(1), "BBB"); @@ -384,7 +384,7 @@ TEST(TypedBufferValuesTest, StringVariableSized_ReadBack) { append_u32_le(bytes, 6u); bytes.insert(bytes.end(), {'W', 'o', 'r', 'l', 'd', '!'}); - TypedBufferStringVariableSized buffer{tcb::span(bytes)}; + TypedBufferStringVariableSized buffer{tcb::span(bytes), 2u}; EXPECT_EQ(buffer.GetElement(0), "Hello"); EXPECT_EQ(buffer.GetElement(1), "World!"); @@ -398,7 +398,7 @@ TEST(TypedBufferValuesTest, StringVariableSized_WriteRoundTrip) { writer.SetElement(2, std::string_view("x")); std::vector finalized = writer.FinalizeAndTakeBuffer(); - TypedBufferStringVariableSized reader{tcb::span(finalized)}; + TypedBufferStringVariableSized reader{tcb::span(finalized), 3u}; EXPECT_EQ(reader.GetElement(0), "short"); EXPECT_EQ(reader.GetElement(1), "a longer string value"); @@ -412,7 +412,7 @@ TEST(TypedBufferValuesTest, StringVariableSized_Iterate) { append_u32_le(bytes, 6u); bytes.insert(bytes.end(), {'b', 'a', 'r', 'b', 'a', 'z'}); - TypedBufferStringVariableSized buffer{tcb::span(bytes)}; + TypedBufferStringVariableSized buffer{tcb::span(bytes), 2u}; std::vector collected; for (const auto value : buffer) { @@ -439,7 +439,7 @@ TEST(TypedBufferValuesTest, StringVariableSized_OutOfOrderWrite_RoundTrip) { EXPECT_EQ(writer.GetElement(0), "replaced"); std::vector finalized = writer.FinalizeAndTakeBuffer(); - TypedBufferStringVariableSized reader{tcb::span(finalized)}; + TypedBufferStringVariableSized reader{tcb::span(finalized), 3u}; EXPECT_EQ(reader.GetElement(0), "replaced"); EXPECT_EQ(reader.GetElement(1), "second"); @@ -459,7 +459,7 @@ TEST(TypedBufferValuesTest, StringVariableSized_EmptyStringsMixedWithNonEmpty) { writer.SetElement(4, std::string_view("")); std::vector finalized = writer.FinalizeAndTakeBuffer(); - TypedBufferStringVariableSized reader{tcb::span(finalized)}; + TypedBufferStringVariableSized reader{tcb::span(finalized), 8u}; EXPECT_EQ(reader.GetElement(0), ""); EXPECT_EQ(reader.GetElement(0).size(), 0u); @@ -512,7 +512,7 @@ TEST(TypedBufferValuesTest, StringVariableSized_UTF8_ReadBack) { append_u32_le(bytes, static_cast(emojis.size())); bytes.insert(bytes.end(), emojis.begin(), emojis.end()); - TypedBufferStringVariableSized buffer{tcb::span(bytes)}; + TypedBufferStringVariableSized buffer{tcb::span(bytes), 3u}; EXPECT_EQ(buffer.GetElement(0), "Привет"); EXPECT_EQ(buffer.GetElement(1), "مرحبا"); @@ -528,7 +528,7 @@ TEST(TypedBufferValuesTest, StringVariableSized_UTF8_WriteRoundTrip) { writer.SetElement(3, std::string_view("Ελληνικά")); std::vector finalized = writer.FinalizeAndTakeBuffer(); - TypedBufferStringVariableSized reader{tcb::span(finalized)}; + TypedBufferStringVariableSized reader{tcb::span(finalized), 4u}; EXPECT_EQ(reader.GetElement(0), "Héllo Wörld"); EXPECT_EQ(reader.GetElement(1), "中文测试"); @@ -547,7 +547,7 @@ TEST(TypedBufferValuesTest, TypedValuesBufferToString_BasicUsage) { append_i32_le(i32_bytes, -1); append_i32_le(i32_bytes, 100); - TypedValuesBuffer i32_variant = TypedBufferI32{tcb::span(i32_bytes)}; + TypedValuesBuffer i32_variant = TypedBufferI32{tcb::span(i32_bytes), 3u}; std::string i32_str = PrintableTypedValuesBuffer(i32_variant); EXPECT_NE(i32_str.find("INT32"), std::string::npos); @@ -561,7 +561,7 @@ TEST(TypedBufferValuesTest, TypedValuesBufferToString_BasicUsage) { append_f32_le(float_bytes, 3.14f); append_f32_le(float_bytes, -0.5f); - TypedValuesBuffer float_variant = TypedBufferFloat{tcb::span(float_bytes)}; + TypedValuesBuffer float_variant = TypedBufferFloat{tcb::span(float_bytes), 2u}; std::string float_str = PrintableTypedValuesBuffer(float_variant); EXPECT_NE(float_str.find("FLOAT"), std::string::npos); @@ -577,7 +577,7 @@ TEST(TypedBufferValuesTest, TypedValuesBufferToString_BasicUsage) { std::vector raw_finalized = writer.FinalizeAndTakeBuffer(); TypedValuesBuffer raw_variant = - TypedBufferRawBytesVariableSized{tcb::span(raw_finalized)}; + TypedBufferRawBytesVariableSized{tcb::span(raw_finalized), 2u}; std::string raw_str = PrintableTypedValuesBuffer(raw_variant); EXPECT_NE(raw_str.find("raw bytes (variable-length)"), std::string::npos); diff --git a/src/scripts/dbpa_remote_testapp.cpp b/src/scripts/dbpa_remote_testapp.cpp index cfae787..f5d8d4e 100644 --- a/src/scripts/dbpa_remote_testapp.cpp +++ b/src/scripts/dbpa_remote_testapp.cpp @@ -247,7 +247,10 @@ class DBPARemoteTestApp { auto compressed_plaintext = Compress(plaintext, CompressionCodec::SNAPPY); // Encrypt once for the combined payload - std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}; + std::map encoding_attributes = { + {"page_encoding", "PLAIN"}, + {"page_type", "DICTIONARY_PAGE"}, + {"dict_page_num_values", std::to_string(sample_data.size())}}; auto encrypt_result = agent_->Encrypt(span(compressed_plaintext), encoding_attributes); if (!encrypt_result || !encrypt_result->success()) { From ccfb0793238cc1161b06a7e1ce4033c784140ebf Mon Sep 17 00:00:00 2001 From: Alejandro Valerio Date: Fri, 13 Mar 2026 12:28:30 -0600 Subject: [PATCH 13/18] - Fixed corner case in Parquet page v1 that didn't account for padded trailing values in bit-packed runs. --- src/processing/parquet_utils.cpp | 26 +++++++++++++++--- src/processing/parquet_utils_test.cpp | 38 ++++++++++++++++++++++++++- 2 files changed, 59 insertions(+), 5 deletions(-) diff --git a/src/processing/parquet_utils.cpp b/src/processing/parquet_utils.cpp index a88fa60..4f6a7ee 100644 --- a/src/processing/parquet_utils.cpp +++ b/src/processing/parquet_utils.cpp @@ -76,10 +76,22 @@ uint32_t ReadV1RunHeaderUleb128(tcb::span bytes, size_t& offset) // - present_count = number of decoded definition levels equal to max_def_level. // size_t CountPresentFromDefinitionLevelsV1(tcb::span def_payload, int32_t num_values, int32_t max_def_level) { + if (num_values < 0) { + throw InvalidInputException("Invalid V1 definition levels: num_values must be non-negative, got " + + std::to_string(num_values)); + } + if (max_def_level <= 0) { + throw InvalidInputException("Invalid V1 definition levels: max_def_level must be positive, got " + + std::to_string(max_def_level)); + } + // Definition level bit width is ceil(log2(max_def_level + 1)). + // Computes the minimum number of bits needed to represent definition levels from 0..max_def_level. int bit_width = 0; - while ((1u << bit_width) <= static_cast(max_def_level)) { + uint32_t def_level_domain = static_cast(max_def_level); + while (def_level_domain > 0) { ++bit_width; + def_level_domain >>= 1; } if (bit_width <= 0) { throw InvalidInputException("Invalid V1 definition levels: computed bit_width must be positive"); @@ -124,7 +136,7 @@ size_t CountPresentFromDefinitionLevelsV1(tcb::span def_payload, const size_t num_groups = static_cast(header >> 1); const size_t run_len = num_groups * 8; const size_t remaining = static_cast(num_values) - decoded_values; - if (num_groups == 0 || run_len > remaining) { + if (num_groups == 0) { throw InvalidInputException("Invalid DATA_PAGE_V1 definition levels: invalid bit-packed run length"); } @@ -148,7 +160,10 @@ size_t CountPresentFromDefinitionLevelsV1(tcb::span def_payload, return v; }; - for (size_t i = 0; i < run_len; ++i) { + // A final bit-packed run may include padded trailing values to complete + // 8-value groups. Decode only the logical values still remaining. + const size_t values_to_decode = std::min(run_len, remaining); + for (size_t i = 0; i < values_to_decode; ++i) { uint32_t level = ReadPacked(i * static_cast(bit_width)); if (level > static_cast(max_def_level)) { throw InvalidInputException("Invalid DATA_PAGE_V1 definition levels: decoded level exceeds max_def_level"); @@ -157,9 +172,12 @@ size_t CountPresentFromDefinitionLevelsV1(tcb::span def_payload, ++present_count; } } - decoded_values += run_len; + decoded_values += values_to_decode; } } + if (def_offset != def_payload.size()) { + throw InvalidInputException("Invalid DATA_PAGE_V1 definition levels: trailing bytes after decoding"); + } return present_count; } diff --git a/src/processing/parquet_utils_test.cpp b/src/processing/parquet_utils_test.cpp index d92638b..29c92bb 100644 --- a/src/processing/parquet_utils_test.cpp +++ b/src/processing/parquet_utils_test.cpp @@ -279,10 +279,21 @@ TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_BitPackedCanonical_0to7_88 // Canonical example from Parquet Encodings.md: // values 0..7 with bit_width=3 are packed as bytes 0x88, 0xC6, 0xFA. // Bit-packed header for one group of 8 values is varint((1 << 1) | 1) = 0x03. + // + // Important detail: decoder bit_width is derived from max_def_level at runtime. + // So this payload is valid only when max_def_level implies bit_width=3 (e.g. 7). + // Reusing these same bytes with max_def_level=3 (bit_width=2) is a different + // encoding contract and can leave unread trailing bytes by design. std::vector def_payload = {0x03, 0x88, 0xC6, 0xFA}; EXPECT_EQ(1u, CountPresentFromDefinitionLevelsV1(def_payload, 8, 7)); // only value 7 - EXPECT_EQ(1u, CountPresentFromDefinitionLevelsV1(def_payload, 8, 3)); // only value 3 + + // Separate bit_width=2 payload for max_def_level=3. + // This keeps payload bit-width consistent with decoder configuration and + // avoids mixing a 3-bit packed stream with a 2-bit decode expectation. + std::vector levels_bw2 = {0, 1, 2, 3, 0, 1, 2, 0}; // one value at level 3 + auto def_payload_bw2 = MakeBitPackedDefPayload(levels_bw2, 2); + EXPECT_EQ(1u, CountPresentFromDefinitionLevelsV1(def_payload_bw2, 8, 3)); } TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_ManualBytes_RleRunLen4_Level1) { @@ -338,6 +349,31 @@ TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_RejectsZeroBitPackedGroups EXPECT_THROW(CountPresentFromDefinitionLevelsV1(def_payload, 8, 1), InvalidInputException); } +TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_BitPackedFinalRunAllowsPadding) { + // Corner case: a final bit-packed run is encoded as a full 8-value group, + // while logical num_values ends mid-group. Decode only the logical values + // and ignore padded trailing values in the last group. + // Payload: header=0x03 (1 bit-packed group => 8 values), packed=0x07 (bits 1,1,1,0,0,0,0,0). + std::vector def_payload = {0x03, 0x07}; + EXPECT_EQ(3u, CountPresentFromDefinitionLevelsV1(def_payload, 3, 1)); +} + +TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_RejectsTrailingBytesAfterDecoding) { + // One full bit-packed group (8 values) plus extra trailing byte that must be rejected. + std::vector def_payload = {0x03, 0xAA, 0xFF}; + EXPECT_THROW(CountPresentFromDefinitionLevelsV1(def_payload, 8, 1), InvalidInputException); +} + +TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_RejectsNonPositiveMaxDefLevel) { + auto def_payload = MakeRleDefPayload(1, 0, 1); + EXPECT_THROW(CountPresentFromDefinitionLevelsV1(def_payload, 1, 0), InvalidInputException); +} + +TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_RejectsNegativeNumValues) { + auto def_payload = MakeRleDefPayload(1, 1, 1); + EXPECT_THROW(CountPresentFromDefinitionLevelsV1(def_payload, -1, 1), InvalidInputException); +} + // ----------------------------------------------------------------------------- // Tests for DecompressAndSplit function. // ----------------------------------------------------------------------------- From af827a0078ca531b79fc84c14bc647f1eb8387a6 Mon Sep 17 00:00:00 2001 From: Alejandro Valerio Date: Fri, 13 Mar 2026 15:37:22 -0600 Subject: [PATCH 14/18] - Added StringFixedSizedCodec and StringVariableSizedCodec to typed_buffer_testing_codecs.h for testing border cases. - Added StringToBytes helper function to bytes_utils.h for testing. --- src/.!1578!.DS_Store | 0 src/.!1595!.DS_Store | 0 src/client/dbps_api_client_test.cpp | 7 +- src/common/bytes_utils.h | 5 + src/processing/encryption_sequencer.cpp | 62 ----- .../encryptors/basic_xor_encryptor.cpp | 147 +--------- src/processing/parquet_utils_test.cpp | 59 ++-- src/processing/typed_buffer.h | 260 ++---------------- src/processing/typed_buffer_codecs.h | 69 ----- src/processing/typed_buffer_test.cpp | 56 ++-- src/processing/typed_buffer_testing_codecs.h | 107 +++++++ src/processing/typed_buffer_values.h | 9 +- src/processing/typed_buffer_values_test.cpp | 22 +- 13 files changed, 225 insertions(+), 578 deletions(-) create mode 100644 src/.!1578!.DS_Store create mode 100644 src/.!1595!.DS_Store create mode 100644 src/processing/typed_buffer_testing_codecs.h diff --git a/src/.!1578!.DS_Store b/src/.!1578!.DS_Store new file mode 100644 index 0000000..e69de29 diff --git a/src/.!1595!.DS_Store b/src/.!1595!.DS_Store new file mode 100644 index 0000000..e69de29 diff --git a/src/client/dbps_api_client_test.cpp b/src/client/dbps_api_client_test.cpp index dd43a10..a51a690 100644 --- a/src/client/dbps_api_client_test.cpp +++ b/src/client/dbps_api_client_test.cpp @@ -23,6 +23,7 @@ #include "tcb/span.hpp" #include "dbps_api_client.h" #include "http_client_base.h" +#include "../common/bytes_utils.h" #include "../common/enums.h" #include #include @@ -30,12 +31,6 @@ using namespace dbps::external; using namespace dbps::enum_utils; -// TODO: Move this to a common test utility file. -// Helper function to convert string to binary data -std::vector StringToBytes(const std::string& str) { - return std::vector(str.begin(), str.end()); -} - // Utility function to compare JSON strings, ignoring specified fields bool CompareJsonStrings(const std::string& json1, const std::string& json2, const std::vector& ignore_fields = {}) { try { diff --git a/src/common/bytes_utils.h b/src/common/bytes_utils.h index 3095151..8aa3013 100644 --- a/src/common/bytes_utils.h +++ b/src/common/bytes_utils.h @@ -302,3 +302,8 @@ inline std::string AddStringAttribute( out[key] = value; return value; } + +// Helper function to convert string to binary data +inline std::vector StringToBytes(const std::string& str) { + return std::vector(str.begin(), str.end()); +} diff --git a/src/processing/encryption_sequencer.cpp b/src/processing/encryption_sequencer.cpp index 268743b..15464c2 100644 --- a/src/processing/encryption_sequencer.cpp +++ b/src/processing/encryption_sequencer.cpp @@ -22,9 +22,7 @@ #include "compression_utils.h" #include "../common/exceptions.h" #include "encryptors/basic_xor_encryptor.h" -#include #include -#include #include #include #include @@ -43,33 +41,6 @@ namespace { constexpr const char* ENCRYPTION_MODE_KEY_DATA_PAGE = "encrypt_mode_data_page"; constexpr const char* ENCRYPTION_MODE_PER_BLOCK = "per_block"; constexpr const char* ENCRYPTION_MODE_PER_VALUE = "per_value"; - - int64_t ToMicroseconds(int64_t nanoseconds) { - return nanoseconds / 1000; - } - - void PrintDurationLine(const char* label, int64_t nanoseconds) { - std::cout << " " << std::left << std::setw(34) << label - << " : " << std::right << std::setw(12) << ToMicroseconds(nanoseconds) << " us" - << " (" << std::setw(12) << nanoseconds << " ns)" << std::endl; - } - - void PrintDecodeAndEncryptTimings( - int64_t decompress_and_split_ns, - int64_t parse_value_bytes_into_typed_list_ns, - int64_t encrypt_value_list_ns, - int64_t encrypt_block_level_bytes_ns, - int64_t join_with_length_prefix_ns, - int64_t compress_ns) { - - std::cout << "----- DecodeAndEncrypt timings (microseconds + nanoseconds) -----" << std::endl; - PrintDurationLine("DecompressAndSplit", decompress_and_split_ns); - PrintDurationLine("ParseValueBytesIntoTypedList", parse_value_bytes_into_typed_list_ns); - PrintDurationLine("EncryptValueList", encrypt_value_list_ns); - PrintDurationLine("EncryptBlock(level_bytes)", encrypt_block_level_bytes_ns); - PrintDurationLine("JoinWithLengthPrefix", join_with_length_prefix_ns); - PrintDurationLine("Compress", compress_ns); - } } // Helper function to create encryptor instance @@ -165,54 +136,21 @@ bool DataBatchEncryptionSequencer::DecodeAndEncrypt(tcb::span pla * - Once per-value encryption for all cases is complete, the try-catch block and the call to EncryptBlock must be removed. */ try { - int64_t decompress_and_split_ns = 0; - int64_t parse_value_bytes_into_typed_list_ns = 0; - int64_t encrypt_value_list_ns = 0; - int64_t encrypt_block_level_bytes_ns = 0; - int64_t join_with_length_prefix_ns = 0; - int64_t compress_ns = 0; - // Decompress and split plaintext into level and value bytes - auto stage_start = std::chrono::steady_clock::now(); auto [level_bytes, value_bytes, num_elements] = DecompressAndSplit( plaintext, compression_, encoding_attributes_converted_); - decompress_and_split_ns = std::chrono::duration_cast( - std::chrono::steady_clock::now() - stage_start).count(); // Parse value bytes into typed values buffer - stage_start = std::chrono::steady_clock::now(); auto typed_buffer = ReinterpretValueBytesAsTypedValuesBuffer( value_bytes, num_elements, datatype_, datatype_length_, encoding_); - parse_value_bytes_into_typed_list_ns = std::chrono::duration_cast( - std::chrono::steady_clock::now() - stage_start).count(); // Encrypt the typed values buffer and level bytes, then join them into a single encrypted byte vector. - stage_start = std::chrono::steady_clock::now(); auto encrypted_value_bytes = encryptor_->EncryptValueList(typed_buffer); - encrypt_value_list_ns = std::chrono::duration_cast( - std::chrono::steady_clock::now() - stage_start).count(); - stage_start = std::chrono::steady_clock::now(); auto encrypted_level_bytes = encryptor_->EncryptBlock(level_bytes); - encrypt_block_level_bytes_ns = std::chrono::duration_cast( - std::chrono::steady_clock::now() - stage_start).count(); - stage_start = std::chrono::steady_clock::now(); auto joined_encrypted_bytes = JoinWithLengthPrefix(encrypted_level_bytes, encrypted_value_bytes); - join_with_length_prefix_ns = std::chrono::duration_cast( - std::chrono::steady_clock::now() - stage_start).count(); // Compress the joined encrypted bytes - stage_start = std::chrono::steady_clock::now(); encrypted_result_ = Compress(joined_encrypted_bytes, encrypted_compression_); - compress_ns = std::chrono::duration_cast( - std::chrono::steady_clock::now() - stage_start).count(); - - PrintDecodeAndEncryptTimings( - decompress_and_split_ns, - parse_value_bytes_into_typed_list_ns, - encrypt_value_list_ns, - encrypt_block_level_bytes_ns, - join_with_length_prefix_ns, - compress_ns); // Set the encryption type to per-value encryption_metadata_[encryption_mode_key] = ENCRYPTION_MODE_PER_VALUE; diff --git a/src/processing/encryptors/basic_xor_encryptor.cpp b/src/processing/encryptors/basic_xor_encryptor.cpp index e003b62..e4b0b9d 100644 --- a/src/processing/encryptors/basic_xor_encryptor.cpp +++ b/src/processing/encryptors/basic_xor_encryptor.cpp @@ -19,102 +19,10 @@ #include "encryptor_utils.h" #include "../../common/exceptions.h" #include "../../common/enum_utils.h" -#include -#include -#include -#include -#include using namespace dbps::processing; using namespace dbps::external; -namespace { - int64_t ElapsedNanosecondsSince(const std::chrono::steady_clock::time_point& start) { - return std::chrono::duration_cast( - std::chrono::steady_clock::now() - start).count(); - } - - int64_t ToMicroseconds(int64_t nanoseconds) { - return nanoseconds / 1000; - } - - void PrintDurationLine(const char* label, int64_t nanoseconds) { - std::cout << " " << label << ": " - << ToMicroseconds(nanoseconds) << " us" - << " (" << nanoseconds << " ns)" << std::endl; - } - - void PrintBasicXorBlockTimings(const char* operation, int64_t total_ns) { - std::cout << "----- BasicXorEncryptor timings (microseconds + nanoseconds) -----" << std::endl; - PrintDurationLine(operation, total_ns); - } - - void PrintDurationLineWithPercent(const char* label, int64_t nanoseconds, int64_t total_ns) { - double pct = total_ns > 0 - ? 100.0 * static_cast(nanoseconds) / static_cast(total_ns) - : 0.0; - std::cout << " " << label << ": " - << ToMicroseconds(nanoseconds) << " us" - << " (" << nanoseconds << " ns)" - << " " << std::fixed << std::setprecision(2) << pct << "%" - << std::endl; - } - - void PrintBasicXorEncryptTypedElementsTimings( - bool is_fixed, - size_t num_elements, - size_t element_size, - int64_t encrypt_elements_loop_ns, - int64_t get_raw_element_ns, - int64_t get_writable_element_ns, - int64_t xor_encrypt_ns, - int64_t finalize_buffer_ns, - int64_t write_header_ns, - int64_t total_ns) { - const int64_t loop_residual_ns = encrypt_elements_loop_ns - - get_raw_element_ns - get_writable_element_ns - xor_encrypt_ns; - const int64_t accounted_sum_ns = get_raw_element_ns - + get_writable_element_ns + xor_encrypt_ns + loop_residual_ns; - - std::cout << "----- BasicXorEncryptor EncryptTypedElements timings -----" << std::endl; - std::cout << " mode: " << (is_fixed ? "fixed" : "variable") << std::endl; - std::cout << " num_elements: " << num_elements << std::endl; - std::cout << " element_size: " << element_size << std::endl; - PrintDurationLine("EncryptElementsLoop", encrypt_elements_loop_ns); - std::cout << " Decomposition of EncryptElementsLoop:" << std::endl; - PrintDurationLineWithPercent("GetRawElement(aggregated)", get_raw_element_ns, encrypt_elements_loop_ns); - PrintDurationLineWithPercent("GetWritableRawElement(aggreg'd)", get_writable_element_ns, encrypt_elements_loop_ns); - PrintDurationLineWithPercent("XorEncryptInto(aggregated)", xor_encrypt_ns, encrypt_elements_loop_ns); - PrintDurationLineWithPercent("LoopResidual(iterator+other)", loop_residual_ns, encrypt_elements_loop_ns); - PrintDurationLineWithPercent("AccountedSum", accounted_sum_ns, encrypt_elements_loop_ns); - PrintDurationLine("FinalizeBuffer", finalize_buffer_ns); - PrintDurationLine("WriteHeader", write_header_ns); - PrintDurationLine("EncryptTypedElements(total)", total_ns); - } - - void PrintBasicXorEncryptValueListTimings(int64_t visit_dispatch_ns, int64_t total_ns) { - std::cout << "----- BasicXorEncryptor EncryptValueList timings (microseconds + nanoseconds) -----" << std::endl; - PrintDurationLine("VariantVisitAndEncrypt", visit_dispatch_ns); - PrintDurationLine("EncryptValueList(total)", total_ns); - } - - void PrintBasicXorDecryptValueListTimings( - bool is_fixed, - size_t num_elements, - int64_t read_header_ns, - int64_t setup_buffer_ns, - int64_t decrypt_elements_ns, - int64_t total_ns) { - std::cout << "----- BasicXorEncryptor DecryptValueList timings (microseconds + nanoseconds) -----" << std::endl; - std::cout << " mode: " << (is_fixed ? "fixed" : "variable") << std::endl; - std::cout << " num_elements: " << num_elements << std::endl; - PrintDurationLine("ReadHeader", read_header_ns); - PrintDurationLine("SetupEncryptedBuffer", setup_buffer_ns); - PrintDurationLine("DecryptElements", decrypt_elements_ns); - PrintDurationLine("DecryptValueList(total)", total_ns); - } -} - // --------------------------------------------------------------------------- // Functions for encrypting and decrypting byte arrays. // --------------------------------------------------------------------------- @@ -208,19 +116,11 @@ std::vector BasicXorEncryptor::EncryptTypedElements( // Encrypt the elements by traversing the typed input buffer and encrypt each element separately: // - Create an output raw-bytes buffer to capture the encrypted elements as bytes. - // - For each element: read its raw bytes, encrypt them, write into the output buffer. + // - For each element: read its raw bytes, get a writable span on the output, and encrypt in-place. // - Finalize the output buffer into a contiguous byte vector. - auto total_start = std::chrono::steady_clock::now(); - std::vector final_buffer; size_t element_size = 0; - int64_t get_raw_element_ns = 0; - int64_t get_writable_element_ns = 0; - int64_t xor_encrypt_ns = 0; - int64_t encrypt_elements_loop_ns = 0; - int64_t finalize_buffer_ns = 0; - int64_t write_header_ns = 0; // Encrypt fixed-size elements if constexpr (is_fixed) { @@ -228,27 +128,15 @@ std::vector BasicXorEncryptor::EncryptTypedElements( TypedBufferRawBytesFixedSized output_buffer{ num_elements, prefix_length, RawBytesFixedSizedCodec{element_size}}; - auto loop_start = std::chrono::steady_clock::now(); - size_t output_index = 0; tcb::span raw_bytes; while (input_buffer.ElementsIteratorNext(raw_bytes)) { - // auto t1 = std::chrono::steady_clock::now(); - auto write_span = output_buffer.GetWritableRawElement(output_index, raw_bytes.size()); - // get_writable_element_ns += ElapsedNanosecondsSince(t1); - - // auto t2 = std::chrono::steady_clock::now(); + auto write_span = output_buffer.GetWritableRawElement(output_index, element_size); XorEncryptInto(raw_bytes, write_span); - // xor_encrypt_ns += ElapsedNanosecondsSince(t2); - output_index++; } - encrypt_elements_loop_ns = ElapsedNanosecondsSince(loop_start); - - auto finalize_start = std::chrono::steady_clock::now(); final_buffer = output_buffer.FinalizeAndTakeBuffer(); - finalize_buffer_ns = ElapsedNanosecondsSince(finalize_start); } // Encrypt variable-size elements @@ -257,58 +145,27 @@ std::vector BasicXorEncryptor::EncryptTypedElements( TypedBufferRawBytesVariableSized output_buffer{ num_elements, reserved_bytes_hint, true, prefix_length}; - auto loop_start = std::chrono::steady_clock::now(); - size_t output_index = 0; tcb::span raw_bytes; - - // ++++++ REMOVE ALL EXTRA INSTRUMENTATION AND TIMING CODE. - // - CAREFUL WITH NOT REMOVING CORRECT INLINED COMMENTS !!!! while (input_buffer.ElementsIteratorNext(raw_bytes)) { - // auto t1 = std::chrono::steady_clock::now(); auto write_span = output_buffer.GetWritableRawElement(output_index, raw_bytes.size()); - // get_writable_element_ns += ElapsedNanosecondsSince(t1); - - // auto t2 = std::chrono::steady_clock::now(); XorEncryptInto(raw_bytes, write_span); - // xor_encrypt_ns += ElapsedNanosecondsSince(t2); - output_index++; } - encrypt_elements_loop_ns = ElapsedNanosecondsSince(loop_start); - - auto finalize_start = std::chrono::steady_clock::now(); final_buffer = output_buffer.FinalizeAndTakeBuffer(); - finalize_buffer_ns = ElapsedNanosecondsSince(finalize_start); } // Write the header to the final buffer and return it. - auto header_start = std::chrono::steady_clock::now(); WriteHeader(final_buffer, {is_fixed, static_cast(num_elements), static_cast(element_size)}); - write_header_ns = ElapsedNanosecondsSince(header_start); - - int64_t total_ns = ElapsedNanosecondsSince(total_start); - - PrintBasicXorEncryptTypedElementsTimings( - is_fixed, num_elements, element_size, - encrypt_elements_loop_ns, - get_raw_element_ns, - get_writable_element_ns, - xor_encrypt_ns, - finalize_buffer_ns, - write_header_ns, - total_ns); return final_buffer; } std::vector BasicXorEncryptor::EncryptValueList( const TypedValuesBuffer& typed_buffer) { - auto total_start = std::chrono::steady_clock::now(); - // Printable context string for logging // std::string context_str = std::string("Context parameters:") // + "\n column_name: " + column_name_ diff --git a/src/processing/parquet_utils_test.cpp b/src/processing/parquet_utils_test.cpp index 29c92bb..316d8e3 100644 --- a/src/processing/parquet_utils_test.cpp +++ b/src/processing/parquet_utils_test.cpp @@ -698,12 +698,11 @@ TEST(ParquetUtils, Reinterpret_INT32) { auto* buf = std::get_if(&result); ASSERT_NE(nullptr, buf); - size_t i = 0; - for (auto val : *buf) { + for (size_t i = 0; i < buf->GetNumElements(); ++i) { + const auto val = buf->GetElement(i); EXPECT_EQ(values[i], val); - ++i; } - EXPECT_EQ(values.size(), i); + EXPECT_EQ(values.size(), buf->GetNumElements()); } TEST(ParquetUtils, Reinterpret_DOUBLE) { @@ -717,12 +716,11 @@ TEST(ParquetUtils, Reinterpret_DOUBLE) { auto* buf = std::get_if(&result); ASSERT_NE(nullptr, buf); - size_t i = 0; - for (auto val : *buf) { + for (size_t i = 0; i < buf->GetNumElements(); ++i) { + const auto val = buf->GetElement(i); EXPECT_DOUBLE_EQ(values[i], val); - ++i; } - EXPECT_EQ(values.size(), i); + EXPECT_EQ(values.size(), buf->GetNumElements()); } TEST(ParquetUtils, Reinterpret_INT96) { @@ -744,14 +742,13 @@ TEST(ParquetUtils, Reinterpret_INT96) { auto* buf = std::get_if(&result); ASSERT_NE(nullptr, buf); - size_t i = 0; - for (auto val : *buf) { + for (size_t i = 0; i < buf->GetNumElements(); ++i) { + const auto val = buf->GetElement(i); EXPECT_EQ(expected[i].lo, val.lo); EXPECT_EQ(expected[i].mid, val.mid); EXPECT_EQ(expected[i].hi, val.hi); - ++i; } - EXPECT_EQ(expected.size(), i); + EXPECT_EQ(expected.size(), buf->GetNumElements()); } TEST(ParquetUtils, Reinterpret_BYTE_ARRAY) { @@ -777,12 +774,11 @@ TEST(ParquetUtils, Reinterpret_BYTE_ARRAY) { auto* buf = std::get_if(&result); ASSERT_NE(nullptr, buf); - size_t i = 0; - for (auto val : *buf) { + for (size_t i = 0; i < buf->GetNumElements(); ++i) { + const auto val = buf->GetElement(i); EXPECT_EQ(expected[i], std::vector(val.begin(), val.end())); - ++i; } - EXPECT_EQ(expected.size(), i); + EXPECT_EQ(expected.size(), buf->GetNumElements()); } TEST(ParquetUtils, Reinterpret_FIXED_LEN_BYTE_ARRAY) { @@ -805,12 +801,11 @@ TEST(ParquetUtils, Reinterpret_FIXED_LEN_BYTE_ARRAY) { auto* buf = std::get_if(&result); ASSERT_NE(nullptr, buf); - size_t i = 0; - for (auto val : *buf) { + for (size_t i = 0; i < buf->GetNumElements(); ++i) { + const auto val = buf->GetElement(i); EXPECT_EQ(expected[i], std::vector(val.begin(), val.end())); - ++i; } - EXPECT_EQ(expected.size(), i); + EXPECT_EQ(expected.size(), buf->GetNumElements()); } TEST(ParquetUtils, Reinterpret_UnsupportedEncoding) { @@ -842,7 +837,10 @@ TEST(ParquetUtils, Reinterpret_InvalidDataSize) { auto* buf = std::get_if(&result); ASSERT_NE(nullptr, buf); EXPECT_THROW( - { for (auto val : *buf) { (void)val; } }, + { + tcb::span element; + while (buf->ElementsIteratorNext(element)) { (void)element; } + }, InvalidInputException); } @@ -876,7 +874,10 @@ TEST(ParquetUtils, Reinterpret_EmptyBytes_FixedSize) { auto* buf = std::get_if(&result); ASSERT_NE(nullptr, buf); size_t count = 0; - for (auto val : *buf) { (void)val; ++count; } + for (size_t i = 0; i < buf->GetNumElements(); ++i) { + (void)buf->GetElement(i); + ++count; + } EXPECT_EQ(0u, count); } @@ -888,7 +889,10 @@ TEST(ParquetUtils, Reinterpret_EmptyBytes_VariableSize) { auto* buf = std::get_if(&result); ASSERT_NE(nullptr, buf); size_t count = 0; - for (auto val : *buf) { (void)val; ++count; } + for (size_t i = 0; i < buf->GetNumElements(); ++i) { + (void)buf->GetElement(i); + ++count; + } EXPECT_EQ(0u, count); } @@ -910,7 +914,8 @@ TEST(ParquetUtils, RoundTrip_INT32) { TypedBufferI32 write_buf{values.size()}; size_t pos = 0; - for (auto val : *src) { + for (size_t i = 0; i < src->GetNumElements(); ++i) { + const auto val = src->GetElement(i); write_buf.SetElement(pos++, val); } @@ -944,7 +949,8 @@ TEST(ParquetUtils, RoundTrip_BYTE_ARRAY) { TypedBufferRawBytesVariableSized write_buf{payloads.size(), input_bytes.size(), true}; size_t pos = 0; - for (auto val : *src) { + for (size_t i = 0; i < src->GetNumElements(); ++i) { + const auto val = src->GetElement(i); write_buf.SetElement(pos++, val); } EXPECT_EQ(payloads.size(), pos); @@ -976,7 +982,8 @@ TEST(ParquetUtils, RoundTrip_FIXED_LEN_BYTE_ARRAY) { TypedBufferRawBytesFixedSized write_buf{ payloads.size(), 0, RawBytesFixedSizedCodec{static_cast(element_len)}}; size_t pos = 0; - for (auto val : *src) { + for (size_t i = 0; i < src->GetNumElements(); ++i) { + const auto val = src->GetElement(i); write_buf.SetElement(pos++, val); } EXPECT_EQ(payloads.size(), pos); diff --git a/src/processing/typed_buffer.h b/src/processing/typed_buffer.h index 7cf61c5..0e84dfc 100644 --- a/src/processing/typed_buffer.h +++ b/src/processing/typed_buffer.h @@ -21,7 +21,6 @@ #include #include #include -#include #include #include @@ -65,6 +64,11 @@ class ByteBuffer { size_t prefix_size = 0, Codec codec = Codec{}); + // Getters for immediately available properties. + size_t GetNumElements() const { return num_elements_; } + size_t GetElementSize() const { return element_size_; } + size_t GetRawBufferSize() const { return elements_span_size_; } + // Get and set elements by position with type access from Codec value_type GetElement(size_t position) const; tcb::span GetRawElement(size_t position) const; @@ -72,75 +76,16 @@ class ByteBuffer { void SetElement(size_t position, const value_type& element); void SetRawElement(size_t position, tcb::span raw); - // Getters for immediately available properties. - size_t GetNumElements() const { return num_elements_; } - size_t GetElementSize() const { return element_size_; } - size_t GetRawBufferSize() const { return elements_span_size_; } - - // Finalizes the write path and transfers the resulting buffer ownership. - std::vector FinalizeAndTakeBuffer(); - // Iterator for read-only elements returning raw bytes. bool ElementsIteratorNext(tcb::span& raw_bytes) const; - // Iterator for read-only elements returning a `value_type` - class ConstIterator { - public: - // Iterator traits consumed indirectly by STL iterator machinery. - using iterator_category = std::forward_iterator_tag; - using value_type = typename ByteBuffer::value_type; - using difference_type = std::ptrdiff_t; - using pointer = void; - using reference = value_type; - - // Basic forward-iterator operations over encoded elements in elements_span_. - ConstIterator(const ByteBuffer* buffer, size_t cursor_offset); - value_type operator*() const; - ConstIterator& operator++(); - bool operator==(const ConstIterator& other) const; - bool operator!=(const ConstIterator& other) const; - - protected: - size_t ReadAndValidateVariableElementSizeAtCursor() const; - tcb::span RawSpanAtCursor() const; - - const ByteBuffer* buffer_ = nullptr; - size_t cursor_offset_ = 0; - size_t elements_span_size_ = 0; - mutable size_t current_element_size_ = 0; - }; - // Methods used by the STL iterator machinery to iterate over the buffer. - ConstIterator begin() const; - ConstIterator end() const; - - // Iterator for read-only elements returning raw bytes. - // Subclass of ConstIterator that only overrides operator* to return raw spans. - class ConstRawIterator : public ConstIterator { - public: - using iterator_category = std::forward_iterator_tag; - using value_type = tcb::span; - using difference_type = std::ptrdiff_t; - using pointer = void; - using reference = value_type; - - using ConstIterator::ConstIterator; - tcb::span operator*() const; - }; - struct RawElementsView { - const ByteBuffer* buffer; - ConstRawIterator begin() const { return ConstRawIterator(buffer, buffer->prefix_size_); } - ConstRawIterator end() const { return ConstRawIterator(buffer, buffer->elements_span_.size()); } - }; - RawElementsView raw_elements() const; + // Finalizes the write path and transfers the resulting buffer ownership. + std::vector FinalizeAndTakeBuffer(); protected: // Helper for reserve heuristics in variable-size parsing. static size_t EstimateOffsetsReserveCountFromSample(tcb::span bytes); - // ++++ Needed after deprecating std iterators? - // Helper to validate the preconditions for reading the buffer with an iterator. - void ValidateIteratorReadPreconditions() const; - // Helper to get a writable span for an element during SetElement calls. tcb::span GetWritableSpanForElement(size_t position, size_t payload_size); @@ -172,8 +117,8 @@ class ByteBuffer { // Initialization methods and flags for read-only buffer void InitializeFromSpan() const; void EnsureInitializedFromSpan() const; - static size_t InitElementSize(const Codec& codec); mutable bool is_initialized_from_span_ = false; + static size_t InitElementSize(const Codec& codec); // Initialization methods and flags for write buffer void InitializeForWriteBuffer(size_t variable_size_reserved_bytes_hint); @@ -304,8 +249,15 @@ inline void ByteBuffer::EnsureInitializedFromSpan() const { InitializeFromSpan(); } -// ++++++ PROBABLY DEPRECATED NOW. MAY LEAVE IT FOR FUTURE USAGE. MARKE IT AS SO +++++++++ -// - Since num_elements_ is now known/an invariant, we no longer need this pretty method (sadly). +// ----------------------------------------------------------------------------- +// EstimateOffsetsReserveCountFromSample method +// +// - Reference method for estimating the number of elements in a span. +// - Since num_elements_ is now known and is an invariant, this method is no longer in the active codepath. +// - Nonetheless it captures a useful heuristic for estimating the number of elements in a span from +// the payload for future reference. +// ----------------------------------------------------------------------------- + template inline size_t ByteBuffer::EstimateOffsetsReserveCountFromSample(tcb::span bytes) { if (bytes.empty()) @@ -353,6 +305,11 @@ inline size_t ByteBuffer::EstimateOffsetsReserveCountFromSample(tcb::span // Element span reader methods // ----------------------------------------------------------------------------- +template +inline typename ByteBuffer::value_type ByteBuffer::GetElement(size_t position) const { + return codec_.Decode(GetRawElement(position)); +} + template inline tcb::span ByteBuffer::GetRawElement(size_t position) const { EnsureInitializedFromSpan(); @@ -382,24 +339,10 @@ inline tcb::span ByteBuffer::GetRawElement(size_t position return elements_span_.subspan(offset + kSizePrefixBytes, element_size); } -template -inline typename ByteBuffer::value_type ByteBuffer::GetElement(size_t position) const { - return codec_.Decode(GetRawElement(position)); -} - - - - - - - - - // ----------------------------------------------------------------------------- -// Element span iterator -- The streamlined version of the iterator. +// Element span iterator // ----------------------------------------------------------------------------- - template inline bool ByteBuffer::ElementsIteratorNext(tcb::span& raw_bytes) const { @@ -454,161 +397,6 @@ inline bool ByteBuffer::ElementsIteratorNext(tcb::span& ra return true; } - - - - - - - - - -// ++++++++++++++ STL ITERATORS ARE DEPRECATED. REMOVE THIS CODE. ++++++++++++++ - -// ----------------------------------------------------------------------------- -// Element span iterator -// -// Allows an alternative read of elements_span_ without need for lazy initialization of offsets_, -// so saving execution time when the traversal of the buffer is strictly sequential. -// This is the most common behavior when reading elements in single threaded mode. -// ----------------------------------------------------------------------------- - -template -inline ByteBuffer::ConstIterator::ConstIterator(const ByteBuffer* buffer, size_t cursor_offset) - : buffer_(buffer), - cursor_offset_(cursor_offset), - elements_span_size_(buffer != nullptr ? buffer->elements_span_size_ : 0u), - current_element_size_(kUnsetSize) {} - -template -inline size_t ByteBuffer::ConstIterator::ReadAndValidateVariableElementSizeAtCursor() const { - // Fixed-sized buffers should not call this method. - if constexpr (is_fixed_sized) { - throw InvalidInputException("ReadAndValidateVariableElementSizeAtCursor is not valid for fixed-size codecs"); - } - - // If the current element size has already been read, return it. - if (current_element_size_ != kUnsetSize) { - return current_element_size_; - } - - // Read the current element size and save it to current_element_size_ cached variable. - if ((elements_span_size_ - cursor_offset_) < kSizePrefixBytes) { - throw InvalidInputException("Malformed variable-size buffer: truncated length prefix"); - } - const size_t current_element_size = ReadSizeAt(buffer_->elements_span_, cursor_offset_); - const size_t payload_offset = cursor_offset_ + kSizePrefixBytes; - if ((elements_span_size_ - payload_offset) < current_element_size) { - throw InvalidInputException("Malformed variable-size buffer: truncated element payload"); - } - current_element_size_ = current_element_size; - return current_element_size_; -} - -template -inline tcb::span ByteBuffer::ConstIterator::RawSpanAtCursor() const { - if (buffer_ == nullptr || cursor_offset_ >= elements_span_size_) { - throw InvalidInputException("Cannot dereference ByteBuffer iterator at end position"); - } - if constexpr (is_fixed_sized) { - return buffer_->elements_span_.subspan(cursor_offset_, buffer_->element_size_); - } - const size_t current_element_size = ReadAndValidateVariableElementSizeAtCursor(); - const size_t payload_offset = cursor_offset_ + kSizePrefixBytes; - return buffer_->elements_span_.subspan(payload_offset, current_element_size); -} - -template -inline typename ByteBuffer::ConstIterator::value_type ByteBuffer::ConstIterator::operator*() const { - // Decode converts raw bytes into the codec's value_type (e.g. int32_t, float, string_view). - // This keeps the iterator's return type consistent with GetElement across all codecs. - return buffer_->codec_.Decode(RawSpanAtCursor()); -} - -template -inline typename ByteBuffer::ConstIterator& ByteBuffer::ConstIterator::operator++() { - if (buffer_ == nullptr || cursor_offset_ >= elements_span_size_) { - return *this; - } - if constexpr (is_fixed_sized) { - cursor_offset_ += buffer_->element_size_; - return *this; - } - const size_t current_element_size = ReadAndValidateVariableElementSizeAtCursor(); - cursor_offset_ += (kSizePrefixBytes + current_element_size); - current_element_size_ = kUnsetSize; - return *this; -} - -template -inline bool ByteBuffer::ConstIterator::operator==(const ConstIterator& other) const { - return buffer_ == other.buffer_ && cursor_offset_ == other.cursor_offset_; -} - -template -inline bool ByteBuffer::ConstIterator::operator!=(const ConstIterator& other) const { - return !(*this == other); -} - -template -inline tcb::span ByteBuffer::ConstRawIterator::operator*() const { - // Returns the raw bytes for the current element, consistent with GetRawElement. - return this->RawSpanAtCursor(); -} - -template -inline void ByteBuffer::ValidateIteratorReadPreconditions() const { - if (is_write_buffer_enabled_) { - throw InvalidInputException("Iterator is only available for read buffers"); - } - if (elements_span_size_ < prefix_size_) { - throw InvalidInputException("Malformed buffer: prefix_size exceeds span size"); - } - if constexpr (is_fixed_sized) { - if (element_size_ <= 0) { - throw InvalidInputException("Invalid fixed-size buffer: element_size must be greater than zero"); - } - const size_t readable_size = elements_span_size_ - prefix_size_; - if ((readable_size % element_size_) != 0) { - throw InvalidInputException("Malformed fixed-size buffer: buffer does not align with element_size"); - } - } -} - -template -inline typename ByteBuffer::ConstIterator ByteBuffer::begin() const { - ValidateIteratorReadPreconditions(); - return ConstIterator(this, prefix_size_); -} - -template -inline typename ByteBuffer::ConstIterator ByteBuffer::end() const { - ValidateIteratorReadPreconditions(); - return ConstIterator(this, elements_span_size_); -} - -template -inline typename ByteBuffer::RawElementsView ByteBuffer::raw_elements() const { - ValidateIteratorReadPreconditions(); - return RawElementsView{this}; -} - - - - - - - - - - - - - - - - - // ----------------------------------------------------------------------------- // Constructors and initializers for write buffer // ----------------------------------------------------------------------------- @@ -639,7 +427,7 @@ ByteBuffer::ByteBuffer( : num_elements_(num_elements), codec_(std::move(codec)), prefix_size_(prefix_size), - element_size_(InitElementSize(codec_)), + element_size_(kUnsetSize), is_write_buffer_enabled_(true) { static_assert(!is_fixed_sized, "ByteBuffer constructor for variable-size elements only."); InitializeForWriteBuffer(use_reserve_hint ? reserved_bytes_hint : 0); diff --git a/src/processing/typed_buffer_codecs.h b/src/processing/typed_buffer_codecs.h index 4794ecb..9e165fd 100644 --- a/src/processing/typed_buffer_codecs.h +++ b/src/processing/typed_buffer_codecs.h @@ -61,75 +61,6 @@ struct PlainValueCodec { } }; -struct StringFixedSizedCodec { - using value_type = std::string_view; - static constexpr bool is_fixed_sized = true; - - explicit StringFixedSizedCodec(size_t element_size_bytes) : element_size_bytes_(element_size_bytes) { - if (element_size_bytes_ <= 0) { - throw InvalidInputException("StringFixedSizedCodec requires element_size_bytes > 0"); - } - } - - static constexpr std::string_view type_name() noexcept { - return "string (fixed-length)"; - } - - size_t element_size() const noexcept { - return element_size_bytes_; - } - - value_type Decode(tcb::span read_span) const { - if (read_span.size() != element_size_bytes_) { - throw InvalidInputException("Decode: read_span size does not match element_size_bytes"); - } - return std::string_view( - reinterpret_cast(read_span.data()), - read_span.size()); - } - - void Encode(const value_type& value, tcb::span write_span) const { - if (write_span.size() != element_size_bytes_) { - throw InvalidInputException("Encode: write_span size does not match element_size_bytes"); - } - if (value.size() != write_span.size()) { - throw InvalidInputException("Encode: value size does not match write_span size"); - } - std::memcpy(write_span.data(), value.data(), write_span.size()); - } - - private: - size_t element_size_bytes_; -}; - -struct StringVariableSizedCodec { - using value_type = std::string_view; - static constexpr bool is_fixed_sized = false; - - static constexpr std::string_view type_name() noexcept { - return "string (variable-length)"; - } - - size_t element_size() const { - throw InvalidInputException("StringVariableSizedCodec does not have a fixed element size"); - } - - value_type Decode(tcb::span read_span) const noexcept { - return std::string_view( - reinterpret_cast(read_span.data()), - read_span.size()); - } - - void Encode(const value_type& value, tcb::span write_span) const { - // Exact match required to prevent short values to leave uninitialized trailing bytes in the buffer - // and prevent longer values from overflowing. - if (value.size() != write_span.size()) { - throw InvalidInputException("Encode: value size does not match write_span size"); - } - std::memcpy(write_span.data(), value.data(), write_span.size()); - } -}; - struct RawBytesFixedSizedCodec { using value_type = tcb::span; static constexpr bool is_fixed_sized = true; diff --git a/src/processing/typed_buffer_test.cpp b/src/processing/typed_buffer_test.cpp index 9819d7d..7dccff5 100644 --- a/src/processing/typed_buffer_test.cpp +++ b/src/processing/typed_buffer_test.cpp @@ -17,6 +17,7 @@ #include "typed_buffer.h" #include "typed_buffer_codecs.h" +#include "typed_buffer_testing_codecs.h" #include #include @@ -31,6 +32,8 @@ using dbps::processing::ByteBuffer; using dbps::processing::RawBytesFixedSizedCodec; using dbps::processing::RawBytesVariableSizedCodec; using dbps::processing::kUnsetSize; +using dbps::processing::testing::StringFixedSizedCodec; +using dbps::processing::testing::StringVariableSizedCodec; template class TypedBufferTestProxy : public ByteBuffer { @@ -261,7 +264,8 @@ TEST(TypedBufferTest, Iterate_ReadOnlyVariableSize_WithPrefixSize_SkipsPrefix) { ByteBuffer buffer(tcb::span(bytes), 2u, 2u); std::vector> collected; - for (const auto element : buffer) { + tcb::span element; + while (buffer.ElementsIteratorNext(element)) { collected.push_back(std::vector(element.begin(), element.end())); } @@ -278,7 +282,8 @@ TEST(TypedBufferTest, Iterate_ReadOnlyFixedSize_WithPrefixSize_SkipsPrefix) { RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 3u, 3u, RawBytesFixedSizedCodec{2u}); std::vector> collected; - for (const auto element : buffer) { + tcb::span element; + while (buffer.ElementsIteratorNext(element)) { collected.push_back(std::vector(element.begin(), element.end())); } @@ -293,7 +298,8 @@ TEST(TypedBufferTest, Iterate_ReadOnlyFixedSize_TraversesInOrder) { RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 3u, 0u, RawBytesFixedSizedCodec{2}); std::vector> collected; - for (const auto element : buffer) { + tcb::span element; + while (buffer.ElementsIteratorNext(element)) { collected.push_back(std::vector(element.begin(), element.end())); } @@ -328,7 +334,8 @@ TEST(TypedBufferTest, Iterate_ReadOnlyVariableSize_TraversesInOrder) { RawBytesVariableSizedBuffer buffer{tcb::span(bytes), 2u}; std::vector> collected; - for (const auto element : buffer) { + tcb::span element; + while (buffer.ElementsIteratorNext(element)) { collected.push_back(std::vector(element.begin(), element.end())); } @@ -372,7 +379,8 @@ TEST(TypedBufferTest, TransformFixedSize_ReadIterateWriteFinalize_RoundTrip) { // Create a new buffer to write the transformed elements into. RawBytesFixedSizedBuffer transformed_buffer(3u, 0, RawBytesFixedSizedCodec{kElementSize}); size_t position = 0; - for (const auto element : source_buffer) { + tcb::span element; + while (source_buffer.ElementsIteratorNext(element)) { std::vector source_element(element.begin(), element.end()); std::vector transformed_element = source_element; transformed_element[0] = static_cast(transformed_element[0] + 1u); @@ -428,7 +436,8 @@ TEST(TypedBufferTest, TransformVariableSize_ReadIterateWriteFinalize_RoundTrip) RawBytesVariableSizedBuffer transformed_buffer(3u, source_bytes.size(), true); size_t position = 0; - for (const auto element : source_buffer) { + tcb::span element; + while (source_buffer.ElementsIteratorNext(element)) { std::vector source_element(element.begin(), element.end()); std::vector transformed_element = source_element; transformed_element[0] = static_cast(transformed_element[0] + 1u); @@ -474,8 +483,8 @@ TEST(TypedBufferTest, Iterate_ReadOnlyEmptySpan_VisitsNoElements) { // Fixed-size empty span. RawBytesFixedSizedBuffer fixed_buffer(tcb::span(empty_bytes), 0u, 0u, RawBytesFixedSizedCodec{2}); size_t fixed_count = 0; - for (const auto element : fixed_buffer) { - (void)element; + tcb::span element; + while (fixed_buffer.ElementsIteratorNext(element)) { ++fixed_count; } EXPECT_EQ(fixed_count, 0u); @@ -483,8 +492,7 @@ TEST(TypedBufferTest, Iterate_ReadOnlyEmptySpan_VisitsNoElements) { // Variable-size empty span. RawBytesVariableSizedBuffer variable_buffer{tcb::span(empty_bytes), 0u}; size_t variable_count = 0; - for (const auto element : variable_buffer) { - (void)element; + while (variable_buffer.ElementsIteratorNext(element)) { ++variable_count; } EXPECT_EQ(variable_count, 0u); @@ -946,14 +954,14 @@ TEST(TypedBufferTest, SetElement_OnReadOnlyBuffer_Throws) { EXPECT_THROW((void)buffer.SetElement(0, tcb::span(replacement)), InvalidInputException); } -TEST(TypedBufferTest, Iterate_OnWriteBuffer_Throws) { +TEST(TypedBufferTest, ElementsIteratorNext_OnWriteBuffer_Throws) { RawBytesFixedSizedBuffer buffer(3u, 0, RawBytesFixedSizedCodec{2u}); - EXPECT_THROW((void)buffer.begin(), InvalidInputException); - EXPECT_THROW((void)buffer.end(), InvalidInputException); + tcb::span element; + EXPECT_THROW((void)buffer.ElementsIteratorNext(element), InvalidInputException); } // ----------------------------------------------------------------------------- -// NextRawElementIterator tests +// ElementsIteratorNext tests // ----------------------------------------------------------------------------- TEST(TypedBufferTest, ElementsIteratorNext_FixedSize_TraversesAllElements) { @@ -1050,17 +1058,17 @@ TEST(TypedBufferTest, GetSpanSize_ReturnsUnderlyingSpanByteCount) { } // ----------------------------------------------------------------------------- -// ConstRawIterator / raw_elements() tests +// Raw-byte iteration via ElementsIteratorNext tests // ----------------------------------------------------------------------------- TEST(TypedBufferTest, RawIterate_FixedSize_ReturnsRawBytesNotDecodedValues) { - using dbps::processing::StringFixedSizedCodec; std::vector bytes = {'H', 'i', '!', 'B', 'y', 'e'}; ByteBuffer buffer( tcb::span(bytes), 2u, 0u, StringFixedSizedCodec{3u}); std::vector decoded; - for (const auto value : buffer) { + for (size_t i = 0; i < buffer.GetNumElements(); ++i) { + const auto value = buffer.GetElement(i); decoded.push_back(value); } ASSERT_EQ(decoded.size(), 2u); @@ -1068,7 +1076,8 @@ TEST(TypedBufferTest, RawIterate_FixedSize_ReturnsRawBytesNotDecodedValues) { EXPECT_EQ(decoded[1], "Bye"); std::vector> raw; - for (const auto span : buffer.raw_elements()) { + tcb::span span; + while (buffer.ElementsIteratorNext(span)) { raw.push_back(std::vector(span.begin(), span.end())); } ASSERT_EQ(raw.size(), 2u); @@ -1077,7 +1086,6 @@ TEST(TypedBufferTest, RawIterate_FixedSize_ReturnsRawBytesNotDecodedValues) { } TEST(TypedBufferTest, RawIterate_VariableSize_ReturnsRawBytesNotDecodedValues) { - using dbps::processing::StringVariableSizedCodec; std::vector bytes; append_u32_le(bytes, 5u); bytes.insert(bytes.end(), {'H', 'e', 'l', 'l', 'o'}); @@ -1087,7 +1095,8 @@ TEST(TypedBufferTest, RawIterate_VariableSize_ReturnsRawBytesNotDecodedValues) { ByteBuffer buffer{tcb::span(bytes), 2u}; std::vector decoded; - for (const auto value : buffer) { + for (size_t i = 0; i < buffer.GetNumElements(); ++i) { + const auto value = buffer.GetElement(i); decoded.push_back(value); } ASSERT_EQ(decoded.size(), 2u); @@ -1095,7 +1104,8 @@ TEST(TypedBufferTest, RawIterate_VariableSize_ReturnsRawBytesNotDecodedValues) { EXPECT_EQ(decoded[1], "OK"); std::vector> raw; - for (const auto span : buffer.raw_elements()) { + tcb::span span; + while (buffer.ElementsIteratorNext(span)) { raw.push_back(std::vector(span.begin(), span.end())); } ASSERT_EQ(raw.size(), 2u); @@ -1103,6 +1113,10 @@ TEST(TypedBufferTest, RawIterate_VariableSize_ReturnsRawBytesNotDecodedValues) { EXPECT_EQ(raw[1], (std::vector{'O', 'K'})); } +// ----------------------------------------------------------------------------- +// Tests for border cases +// ----------------------------------------------------------------------------- + TEST(TypedBufferTest, ConstructVariableSize_EmptyBuffer_InitializesEmptyState) { std::vector bytes; RawBytesVariableSizedBuffer buffer{tcb::span(bytes), 0u}; diff --git a/src/processing/typed_buffer_testing_codecs.h b/src/processing/typed_buffer_testing_codecs.h new file mode 100644 index 0000000..5f2f985 --- /dev/null +++ b/src/processing/typed_buffer_testing_codecs.h @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "typed_buffer.h" + + +// ----------------------------------------------------------------------------- +// StringFixedSizedCodec and StringVariableSizedCodec +// +// These codecs are only test only, but help check on border conditions like zero-size and mult-byte values +// +// ----------------------------------------------------------------------------- + +namespace dbps::processing::testing { + +struct StringFixedSizedCodec { + using value_type = std::string_view; + static constexpr bool is_fixed_sized = true; + + explicit StringFixedSizedCodec(size_t element_size_bytes) : element_size_bytes_(element_size_bytes) { + if (element_size_bytes_ == 0) { + throw InvalidInputException("StringFixedSizedCodec requires element_size_bytes > 0"); + } + } + + static constexpr std::string_view type_name() noexcept { + return "string (fixed-length)"; + } + + size_t element_size() const noexcept { + return element_size_bytes_; + } + + value_type Decode(tcb::span read_span) const { + if (read_span.size() != element_size_bytes_) { + throw InvalidInputException("Decode: read_span size does not match element_size_bytes"); + } + return std::string_view( + reinterpret_cast(read_span.data()), + read_span.size()); + } + + void Encode(const value_type& value, tcb::span write_span) const { + if (write_span.size() != element_size_bytes_) { + throw InvalidInputException("Encode: write_span size does not match element_size_bytes"); + } + if (value.size() != write_span.size()) { + throw InvalidInputException("Encode: value size does not match write_span size"); + } + std::memcpy(write_span.data(), value.data(), write_span.size()); + } + +private: + size_t element_size_bytes_; +}; + +struct StringVariableSizedCodec { + using value_type = std::string_view; + static constexpr bool is_fixed_sized = false; + + static constexpr std::string_view type_name() noexcept { + return "string (variable-length)"; + } + + size_t element_size() const { + throw InvalidInputException("StringVariableSizedCodec does not have a fixed element size"); + } + + value_type Decode(tcb::span read_span) const noexcept { + return std::string_view( + reinterpret_cast(read_span.data()), + read_span.size()); + } + + void Encode(const value_type& value, tcb::span write_span) const { + // Exact match required to prevent short values leaving stale trailing bytes, + // and to prevent longer values from overflowing. + if (value.size() != write_span.size()) { + throw InvalidInputException("Encode: value size does not match write_span size"); + } + std::memcpy(write_span.data(), value.data(), write_span.size()); + } +}; + +using TypedBufferStringFixedSized = ByteBuffer; +using TypedBufferStringVariableSized = ByteBuffer; + +} // namespace dbps::processing::testing diff --git a/src/processing/typed_buffer_values.h b/src/processing/typed_buffer_values.h index 3db6547..7a71285 100644 --- a/src/processing/typed_buffer_values.h +++ b/src/processing/typed_buffer_values.h @@ -42,8 +42,6 @@ using TypedBufferI64 = ByteBuffer>; using TypedBufferFloat = ByteBuffer>; using TypedBufferDouble = ByteBuffer>; using TypedBufferInt96 = ByteBuffer>; -using TypedBufferStringFixedSized = ByteBuffer; -using TypedBufferStringVariableSized = ByteBuffer; using TypedBufferRawBytesFixedSized = ByteBuffer; using TypedBufferRawBytesVariableSized = ByteBuffer; @@ -53,8 +51,6 @@ using TypedValuesBuffer = std::variant< TypedBufferFloat, TypedBufferDouble, TypedBufferInt96, - TypedBufferStringFixedSized, - TypedBufferStringVariableSized, TypedBufferRawBytesFixedSized, TypedBufferRawBytesVariableSized >; @@ -70,8 +66,8 @@ inline std::string PrintableTypedValuesBuffer(const TypedValuesBuffer& buffer) { out << BufferType::type_name() << " (" << num_elements << " elements):\n"; - size_t i = 0; - for (const auto element : typed_buffer) { + for (size_t i = 0; i < num_elements; ++i) { + const auto element = typed_buffer.GetElement(i); if constexpr (std::is_same_v) { out << " [" << i << "] [" << element.lo << ", " << element.mid << ", " << element.hi << "]\n"; @@ -83,7 +79,6 @@ inline std::string PrintableTypedValuesBuffer(const TypedValuesBuffer& buffer) { } else { out << " [" << i << "] " << element << "\n"; } - ++i; } return out.str(); diff --git a/src/processing/typed_buffer_values_test.cpp b/src/processing/typed_buffer_values_test.cpp index 0329d0a..bcb4394 100644 --- a/src/processing/typed_buffer_values_test.cpp +++ b/src/processing/typed_buffer_values_test.cpp @@ -16,6 +16,7 @@ // under the License. #include "typed_buffer_values.h" +#include "typed_buffer_testing_codecs.h" #include #include @@ -28,6 +29,9 @@ #include "exceptions.h" using namespace dbps::processing; +using dbps::processing::testing::StringFixedSizedCodec; +using dbps::processing::testing::TypedBufferStringFixedSized; +using dbps::processing::testing::TypedBufferStringVariableSized; // ============================================================================= // INT32 @@ -83,7 +87,8 @@ TEST(TypedBufferValuesTest, Int32_Iterate) { TypedBufferI32 buffer{tcb::span(bytes), 3u}; std::vector collected; - for (const auto value : buffer) { + for (size_t i = 0; i < buffer.GetNumElements(); ++i) { + const auto value = buffer.GetElement(i); collected.push_back(value); } @@ -192,7 +197,8 @@ TEST(TypedBufferValuesTest, Int96_Iterate) { TypedBufferInt96 buffer{tcb::span(bytes), 2u}; std::vector collected; - for (const auto value : buffer) { + for (size_t i = 0; i < buffer.GetNumElements(); ++i) { + const auto value = buffer.GetElement(i); collected.push_back(value); } @@ -259,7 +265,8 @@ TEST(TypedBufferValuesTest, Float_Iterate) { TypedBufferFloat buffer{tcb::span(bytes), 3u}; std::vector collected; - for (const auto value : buffer) { + for (size_t i = 0; i < buffer.GetNumElements(); ++i) { + const auto value = buffer.GetElement(i); collected.push_back(value); } @@ -337,7 +344,8 @@ TEST(TypedBufferValuesTest, StringFixedSized_Iterate) { tcb::span(bytes), 2u, 0u, StringFixedSizedCodec{4}); std::vector collected; - for (const auto value : buffer) { + for (size_t i = 0; i < buffer.GetNumElements(); ++i) { + const auto value = buffer.GetElement(i); collected.emplace_back(value); } @@ -415,7 +423,8 @@ TEST(TypedBufferValuesTest, StringVariableSized_Iterate) { TypedBufferStringVariableSized buffer{tcb::span(bytes), 2u}; std::vector collected; - for (const auto value : buffer) { + for (size_t i = 0; i < buffer.GetNumElements(); ++i) { + const auto value = buffer.GetElement(i); collected.emplace_back(value); } @@ -477,7 +486,8 @@ TEST(TypedBufferValuesTest, StringVariableSized_EmptyStringsMixedWithNonEmpty) { size_t element_count = 0; std::vector collected; - for (const auto value : reader) { + for (size_t i = 0; i < reader.GetNumElements(); ++i) { + const auto value = reader.GetElement(i); collected.emplace_back(value); ++element_count; } From 55c9baa7b7b48aa42bb12a5a61944851a1befa90 Mon Sep 17 00:00:00 2001 From: Alejandro Valerio Date: Fri, 13 Mar 2026 15:40:59 -0600 Subject: [PATCH 15/18] - Removing macOS hidden files. --- src/.!1578!.DS_Store | 0 src/.!1595!.DS_Store | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 src/.!1578!.DS_Store delete mode 100644 src/.!1595!.DS_Store diff --git a/src/.!1578!.DS_Store b/src/.!1578!.DS_Store deleted file mode 100644 index e69de29..0000000 diff --git a/src/.!1595!.DS_Store b/src/.!1595!.DS_Store deleted file mode 100644 index e69de29..0000000 From 3ad364d19e5202f889f1e2d3c332845e03df65c0 Mon Sep 17 00:00:00 2001 From: Alejandro Valerio Date: Mon, 16 Mar 2026 17:03:31 -0600 Subject: [PATCH 16/18] - Removing Compression call from encryption sequencer final result (low-compression ratio for encrypted payloads). - Small updates from code review. --- .gitignore | 1 + src/common/bytes_utils_test.cpp | 45 ++++ src/processing/encryption_sequencer.cpp | 19 +- src/processing/encryption_sequencer_test.cpp | 231 ++++++++++++++++++ .../encryptors/basic_xor_encryptor.cpp | 4 +- src/processing/parquet_utils.cpp | 16 +- src/processing/parquet_utils.h | 29 --- src/processing/parquet_utils_test.cpp | 138 ++++++++--- src/processing/typed_buffer.h | 18 +- 9 files changed, 414 insertions(+), 87 deletions(-) diff --git a/.gitignore b/.gitignore index 130724f..a627f88 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ _deps/ # macOS .DS_Store +**/.!*.DS_Store # Editor/IDE config .vscode/ diff --git a/src/common/bytes_utils_test.cpp b/src/common/bytes_utils_test.cpp index e4a8a0e..87a6a38 100644 --- a/src/common/bytes_utils_test.cpp +++ b/src/common/bytes_utils_test.cpp @@ -239,4 +239,49 @@ TEST(BytesUtils, AttributesMap_AddBool) { std::map bad_attrs{{"page_v2_is_compressed", "maybe"}}; EXPECT_THROW(AddBoolAttribute(out, bad_attrs, "page_v2_is_compressed"), InvalidInputException); +} + +TEST(BytesUtils, StringToBytes_AsciiText) { + const std::string input = "dbps"; + const std::vector result = StringToBytes(input); + + EXPECT_EQ((std::vector{'d', 'b', 'p', 's'}), result); +} + +TEST(BytesUtils, StringToBytes_EmptyString) { + const std::string input; + const std::vector result = StringToBytes(input); + + EXPECT_TRUE(result.empty()); +} + +TEST(BytesUtils, StringToBytes_PreservesRawBytesAndNulls) { + std::string input; + input.push_back('D'); + input.push_back('B'); + input.push_back('P'); + input.push_back('S'); + input.push_back('\0'); + input.push_back('X'); + input.push_back('Y'); + input.push_back(static_cast(0xFF)); + input.push_back(static_cast(0x80)); + input.push_back('\0'); + input.push_back('Z'); + + const std::vector result = StringToBytes(input); + const std::vector expected = { + static_cast('D'), + static_cast('B'), + static_cast('P'), + static_cast('S'), + static_cast(0x00), + static_cast('X'), + static_cast('Y'), + static_cast(0xFF), + static_cast(0x80), + static_cast(0x00), + static_cast('Z')}; + + EXPECT_EQ(expected, result); } \ No newline at end of file diff --git a/src/processing/encryption_sequencer.cpp b/src/processing/encryption_sequencer.cpp index 15464c2..f5f3e0b 100644 --- a/src/processing/encryption_sequencer.cpp +++ b/src/processing/encryption_sequencer.cpp @@ -147,10 +147,10 @@ bool DataBatchEncryptionSequencer::DecodeAndEncrypt(tcb::span pla // Encrypt the typed values buffer and level bytes, then join them into a single encrypted byte vector. auto encrypted_value_bytes = encryptor_->EncryptValueList(typed_buffer); auto encrypted_level_bytes = encryptor_->EncryptBlock(level_bytes); - auto joined_encrypted_bytes = JoinWithLengthPrefix(encrypted_level_bytes, encrypted_value_bytes); - - // Compress the joined encrypted bytes - encrypted_result_ = Compress(joined_encrypted_bytes, encrypted_compression_); + + // Encrypted payloads mostly have a low-compression ratio, so the gains in size from compression are minimal or negative. + // Therefore, the final joined encrypted bytes are returned as-is without compression. + encrypted_result_ = JoinWithLengthPrefix(encrypted_level_bytes, encrypted_value_bytes); // Set the encryption type to per-value encryption_metadata_[encryption_mode_key] = ENCRYPTION_MODE_PER_VALUE; @@ -226,16 +226,15 @@ bool DataBatchEncryptionSequencer::DecryptAndEncode(tcb::span cip error_message_ = "Failed to get encryption_mode from encryption_metadata"; return false; } - std::string encryption_mode = encryption_mode_opt.value(); + const std::string& encryption_mode = encryption_mode_opt.value(); // Per-value encryption if (encryption_mode == ENCRYPTION_MODE_PER_VALUE) { - // Decompress the encrypted bytes - auto decompressed_encrypted_bytes = Decompress(ciphertext, encrypted_compression_); - + // Split the joined encrypted bytes, then decrypt the level and value bytes separately. - auto [encrypted_level_bytes, encrypted_value_bytes] = - SplitWithLengthPrefix(tcb::span(decompressed_encrypted_bytes)); + // The ciphertext payload is already the joined bytes without compression. + auto [encrypted_level_bytes, encrypted_value_bytes] = SplitWithLengthPrefix(ciphertext); + auto level_bytes = encryptor_->DecryptBlock(encrypted_level_bytes); auto typed_buffer = encryptor_->DecryptValueList(encrypted_value_bytes); diff --git a/src/processing/encryption_sequencer_test.cpp b/src/processing/encryption_sequencer_test.cpp index 9af024e..cf9559d 100644 --- a/src/processing/encryption_sequencer_test.cpp +++ b/src/processing/encryption_sequencer_test.cpp @@ -597,3 +597,234 @@ TEST(EncryptionSequencer, ConvertEncodingAttributesToValues_Negative) { EXPECT_FALSE(sequencer3.TestConvertEncodingAttributesToValues()); EXPECT_EQ(sequencer3.error_stage_, "encoding_attribute_conversion"); } + +// ----------------------------------------------------------------------------- +// DATA_PAGE end-to-end coverage through DataBatchEncryptionSequencer. +// ----------------------------------------------------------------------------- + +TEST(EncryptionSequencer, DataPageV1NullableByteArray_PerValueRoundTrip) { + std::vector byte_array_elements = { + {'a', 'l', 'p', 'h', 'a'}, + {'b', 'e', 't', 'a'}, + {'g', 'a', 'm', 'm', 'a', '3'}}; + auto value_bytes = CombineRawBytesIntoValueBytesForTesting( + byte_array_elements, Type::BYTE_ARRAY, std::nullopt, Encoding::PLAIN); + + // Nullable DATA_PAGE_V1 levels: one RLE run with 3 present values. + // level bytes = [u32 len=2][0x06, 0x01] + std::vector level_bytes; + append_u32_le(level_bytes, 2u); + level_bytes.push_back(0x06); // run_len = 3 + level_bytes.push_back(0x01); // def level value = 1 (present) + + auto page_payload = Join(level_bytes, value_bytes); + auto plaintext = Compress(page_payload, CompressionCodec::SNAPPY); + + std::map attribs = { + {"page_type", "DATA_PAGE_V1"}, + {"data_page_num_values", "3"}, + {"data_page_max_definition_level", "1"}, + {"data_page_max_repetition_level", "0"}, + {"page_v1_repetition_level_encoding", "RLE"}, + {"page_v1_definition_level_encoding", "RLE"}}; + + DataBatchEncryptionSequencer sequencer( + "byte_array_col_v1", + Type::BYTE_ARRAY, + std::nullopt, + CompressionCodec::SNAPPY, + Encoding::PLAIN, + attribs, + CompressionCodec::UNCOMPRESSED, + "test_key", + "test_user", + "{}", + {}); + + ASSERT_TRUE(sequencer.DecodeAndEncrypt(plaintext)) + << sequencer.error_stage_ << " - " << sequencer.error_message_; + ASSERT_TRUE(sequencer.encryption_metadata_.count("encrypt_mode_data_page") == 1); + EXPECT_EQ(sequencer.encryption_metadata_.at("encrypt_mode_data_page"), "per_value"); + + ASSERT_TRUE(sequencer.DecryptAndEncode(sequencer.encrypted_result_)) + << sequencer.error_stage_ << " - " << sequencer.error_message_; + EXPECT_EQ(sequencer.decrypted_result_, plaintext); +} + +TEST(EncryptionSequencer, DataPageV2NullableByteArray_PerValueRoundTrip) { + std::vector byte_array_elements = { + {'x', 'x'}, + {'y', 'y', 'y'}, + {'z', 'z', 'z', 'z'}}; + auto value_bytes = CombineRawBytesIntoValueBytesForTesting( + byte_array_elements, Type::BYTE_ARRAY, std::nullopt, Encoding::PLAIN); + + // DATA_PAGE_V2: logical rows=5, nulls=2, so num_elements in value bytes is 3. + std::vector level_bytes = {0x00}; // one byte of definition-level section + auto plaintext = Join(level_bytes, value_bytes); + + std::map attribs = { + {"page_type", "DATA_PAGE_V2"}, + {"data_page_num_values", "5"}, + {"data_page_max_definition_level", "1"}, + {"data_page_max_repetition_level", "0"}, + {"page_v2_definition_levels_byte_length", "1"}, + {"page_v2_repetition_levels_byte_length", "0"}, + {"page_v2_num_nulls", "2"}, + {"page_v2_is_compressed", "false"}}; + + DataBatchEncryptionSequencer sequencer( + "byte_array_col_v2", + Type::BYTE_ARRAY, + std::nullopt, + CompressionCodec::UNCOMPRESSED, + Encoding::PLAIN, + attribs, + CompressionCodec::UNCOMPRESSED, + "test_key_v2", + "test_user", + "{}", + {}); + + ASSERT_TRUE(sequencer.DecodeAndEncrypt(plaintext)) + << sequencer.error_stage_ << " - " << sequencer.error_message_; + ASSERT_TRUE(sequencer.encryption_metadata_.count("encrypt_mode_data_page") == 1); + EXPECT_EQ(sequencer.encryption_metadata_.at("encrypt_mode_data_page"), "per_value"); + + ASSERT_TRUE(sequencer.DecryptAndEncode(sequencer.encrypted_result_)) + << sequencer.error_stage_ << " - " << sequencer.error_message_; + EXPECT_EQ(sequencer.decrypted_result_, plaintext); +} + +TEST(EncryptionSequencer, DataPageV1NullableFloat_PerValueRoundTrip) { + std::vector float_values = {1.25f, -2.5f, 100.75f}; + std::vector value_bytes; + value_bytes.reserve(float_values.size() * sizeof(float)); + for (float v : float_values) { + append_f32_le(value_bytes, v); + } + + // Nullable DATA_PAGE_V1 levels: one RLE run with 3 present values. + std::vector level_bytes; + append_u32_le(level_bytes, 2u); + level_bytes.push_back(0x06); // run_len = 3 + level_bytes.push_back(0x01); // def level value = 1 (present) + + auto page_payload = Join(level_bytes, value_bytes); + auto plaintext = Compress(page_payload, CompressionCodec::SNAPPY); + + std::map attribs = { + {"page_type", "DATA_PAGE_V1"}, + {"data_page_num_values", "3"}, + {"data_page_max_definition_level", "1"}, + {"data_page_max_repetition_level", "0"}, + {"page_v1_repetition_level_encoding", "RLE"}, + {"page_v1_definition_level_encoding", "RLE"}}; + + DataBatchEncryptionSequencer sequencer( + "float_col_v1", + Type::FLOAT, + std::nullopt, + CompressionCodec::SNAPPY, + Encoding::PLAIN, + attribs, + CompressionCodec::UNCOMPRESSED, + "test_key_float_v1", + "test_user", + "{}", + {}); + + ASSERT_TRUE(sequencer.DecodeAndEncrypt(plaintext)) + << sequencer.error_stage_ << " - " << sequencer.error_message_; + ASSERT_TRUE(sequencer.encryption_metadata_.count("encrypt_mode_data_page") == 1); + EXPECT_EQ(sequencer.encryption_metadata_.at("encrypt_mode_data_page"), "per_value"); + + ASSERT_TRUE(sequencer.DecryptAndEncode(sequencer.encrypted_result_)) + << sequencer.error_stage_ << " - " << sequencer.error_message_; + EXPECT_EQ(sequencer.decrypted_result_, plaintext); +} + +TEST(EncryptionSequencer, DataPageV2NullableFloat_PerValueRoundTrip) { + std::vector float_values = {9.5f, 0.0f, -7.25f}; + std::vector value_bytes; + value_bytes.reserve(float_values.size() * sizeof(float)); + for (float v : float_values) { + append_f32_le(value_bytes, v); + } + + // DATA_PAGE_V2: logical rows=5, nulls=2, so num_elements in value bytes is 3. + std::vector level_bytes = {0x00}; + auto plaintext = Join(level_bytes, value_bytes); + + std::map attribs = { + {"page_type", "DATA_PAGE_V2"}, + {"data_page_num_values", "5"}, + {"data_page_max_definition_level", "1"}, + {"data_page_max_repetition_level", "0"}, + {"page_v2_definition_levels_byte_length", "1"}, + {"page_v2_repetition_levels_byte_length", "0"}, + {"page_v2_num_nulls", "2"}, + {"page_v2_is_compressed", "false"}}; + + DataBatchEncryptionSequencer sequencer( + "float_col_v2", + Type::FLOAT, + std::nullopt, + CompressionCodec::UNCOMPRESSED, + Encoding::PLAIN, + attribs, + CompressionCodec::UNCOMPRESSED, + "test_key_float_v2", + "test_user", + "{}", + {}); + + ASSERT_TRUE(sequencer.DecodeAndEncrypt(plaintext)) + << sequencer.error_stage_ << " - " << sequencer.error_message_; + ASSERT_TRUE(sequencer.encryption_metadata_.count("encrypt_mode_data_page") == 1); + EXPECT_EQ(sequencer.encryption_metadata_.at("encrypt_mode_data_page"), "per_value"); + + ASSERT_TRUE(sequencer.DecryptAndEncode(sequencer.encrypted_result_)) + << sequencer.error_stage_ << " - " << sequencer.error_message_; + EXPECT_EQ(sequencer.decrypted_result_, plaintext); +} + +TEST(EncryptionSequencer, DataPageV1MalformedDefinitionPayload_Throws) { + std::vector byte_array_elements = { + {'a', 'l', 'p', 'h', 'a'}, + {'b', 'e', 't', 'a'}, + {'g', 'a', 'm', 'm', 'a', '3'}}; + auto value_bytes = CombineRawBytesIntoValueBytesForTesting( + byte_array_elements, Type::BYTE_ARRAY, std::nullopt, Encoding::PLAIN); + + // Malformed DATA_PAGE_V1 definition payload: truncated varint run header. + std::vector level_bytes; + append_u32_le(level_bytes, 1u); + level_bytes.push_back(0x80); // continuation bit set, missing next byte + + auto page_payload = Join(level_bytes, value_bytes); + auto plaintext = Compress(page_payload, CompressionCodec::SNAPPY); + + std::map attribs = { + {"page_type", "DATA_PAGE_V1"}, + {"data_page_num_values", "3"}, + {"data_page_max_definition_level", "1"}, + {"data_page_max_repetition_level", "0"}, + {"page_v1_repetition_level_encoding", "RLE"}, + {"page_v1_definition_level_encoding", "RLE"}}; + + DataBatchEncryptionSequencer sequencer( + "byte_array_col_bad_v1", + Type::BYTE_ARRAY, + std::nullopt, + CompressionCodec::SNAPPY, + Encoding::PLAIN, + attribs, + CompressionCodec::UNCOMPRESSED, + "test_key_bad_v1", + "test_user", + "{}", + {}); + + EXPECT_THROW((void)sequencer.DecodeAndEncrypt(plaintext), InvalidInputException); +} diff --git a/src/processing/encryptors/basic_xor_encryptor.cpp b/src/processing/encryptors/basic_xor_encryptor.cpp index e4b0b9d..2f9a580 100644 --- a/src/processing/encryptors/basic_xor_encryptor.cpp +++ b/src/processing/encryptors/basic_xor_encryptor.cpp @@ -27,6 +27,8 @@ using namespace dbps::external; // Functions for encrypting and decrypting byte arrays. // --------------------------------------------------------------------------- +// XorEncryptInto uses a writable span `out` to encrypt the data in-place. +// This is a performance optimization to avoid copying the data to a buffer and then returning it. void BasicXorEncryptor::XorEncryptInto(tcb::span data, tcb::span out) { size_t data_size = data.size(); size_t out_size = out.size(); @@ -103,8 +105,6 @@ std::vector BasicXorEncryptor::EncryptTypedElements( const TypedBuffer& input_buffer) { constexpr bool is_fixed = TypedBuffer::is_fixed_sized; constexpr size_t prefix_length = is_fixed ? kFixedHeaderLength : kVariableHeaderLength; - - // GetNumElements is read from Parquet metadata and level bytes, not calculated from payload. const size_t num_elements = input_buffer.GetNumElements(); // If there are no elements, return an empty buffer with the header. diff --git a/src/processing/parquet_utils.cpp b/src/processing/parquet_utils.cpp index 4f6a7ee..794e844 100644 --- a/src/processing/parquet_utils.cpp +++ b/src/processing/parquet_utils.cpp @@ -66,6 +66,8 @@ uint32_t ReadV1RunHeaderUleb128(tcb::span bytes, size_t& offset) // Decodes a DATA_PAGE_V1 definition-level payload (hybrid RLE/bit-packed) and // returns the number of present (non-null) values in the page. // +// Reference: https://parquet.apache.org/docs/file-format/data-pages/encodings/#RLE +// // Inputs: // - def_payload: bytes of the V1 definition-level stream payload only // (without the outer [u32 length] prefix). @@ -75,7 +77,7 @@ uint32_t ReadV1RunHeaderUleb128(tcb::span bytes, size_t& offset) // Output: // - present_count = number of decoded definition levels equal to max_def_level. // -size_t CountPresentFromDefinitionLevelsV1(tcb::span def_payload, int32_t num_values, int32_t max_def_level) { +size_t CountPresentValuesFromDefinitionLevelsV1(tcb::span def_payload, int32_t num_values, int32_t max_def_level) { if (num_values < 0) { throw InvalidInputException("Invalid V1 definition levels: num_values must be non-negative, got " + std::to_string(num_values)); @@ -223,6 +225,8 @@ tcb::span ReadDefinitionLevelBytesV1(tcb::span lev // Helper function to calculate level bytes length // ----------------------------------------------------------------------------- +// Calculates the total length of level bytes based on encoding attributes. +// Assumes the input encoding attributes are already validated with the required keys and expected value types. int CalculateLevelBytesLength(tcb::span raw, const AttributesMap& encoding_attribs) { @@ -305,7 +309,9 @@ LevelAndValueBytes DecompressAndSplit( decompressed_bytes, encoding_attributes); auto [level_bytes, value_bytes] = Split(decompressed_bytes, leading_bytes_to_strip); - // For DATA_PAGE_V1, calculate num_elements by parsing the level bytes. + // For DATA_PAGE_V1, data_page_num_values is the count of logical rows (includes nulls). + // The V1 header does not carry num_nulls, so we cannot derive present values as in V2. + // To get the number of encoded physical values in value_bytes, we must parse definition levels. size_t num_elements = 0; const int32_t num_values = std::get(encoding_attributes.at("data_page_num_values")); int32_t max_def_level = std::get(encoding_attributes.at("data_page_max_definition_level")); @@ -317,7 +323,7 @@ LevelAndValueBytes DecompressAndSplit( // If max_def_level > 0, there are definition levels bytes. So parse it and count the present values. else { auto def_bytes_payload = ReadDefinitionLevelBytesV1(level_bytes, max_rep_level); - num_elements = CountPresentFromDefinitionLevelsV1(def_bytes_payload, num_values, max_def_level); + num_elements = CountPresentValuesFromDefinitionLevelsV1(def_bytes_payload, num_values, max_def_level); } return LevelAndValueBytes{std::move(level_bytes), std::move(value_bytes), num_elements}; @@ -341,7 +347,9 @@ LevelAndValueBytes DecompressAndSplit( value_bytes = std::vector(compressed_value_bytes_span.begin(), compressed_value_bytes_span.end()); } - // For DATA_PAGE_V2, get num_elements from num_values and num_nulls in encoding_attributes. + // For DATA_PAGE_V2, data_page_num_values is the count of logical rows, not present values. data_page_num_values includes nulls. + // num_nulls is the count of nulls in the page. + // So num_elements (the present values) is num_values - num_nulls. int32_t num_values = std::get(encoding_attributes.at("data_page_num_values")); int32_t num_nulls = std::get(encoding_attributes.at("page_v2_num_nulls")); if (num_nulls > num_values) { diff --git a/src/processing/parquet_utils.h b/src/processing/parquet_utils.h index f6a6bd6..dd48290 100644 --- a/src/processing/parquet_utils.h +++ b/src/processing/parquet_utils.h @@ -38,35 +38,6 @@ struct LevelAndValueBytes { using namespace dbps::external; -// ----------------------------------------------------------------------------- -// Helper functions for lower-level Parquet metadata and level bytes parsing. -// ----------------------------------------------------------------------------- - -/** - * Calculates the total length of level bytes based on encoding attributes. - * Assumes the input encoding attributes are already validated with the required keys and expected value types. - * - * @param raw Raw binary data (currently unused but kept for future V1 implementation) - * @param encoding_attribs Converted encoding attributes map - * @return Total length of level bytes. Throws exceptions if calculation fails or page type is unsupported - */ -int CalculateLevelBytesLength(tcb::span raw, - const AttributesMap& encoding_attribs); - -/** - * Decode DATA_PAGE_V1 definition-level payload bytes (hybrid RLE/bit-packed) - * and return the number of present (non-null) values. - * - * @param def_payload Definition-level payload bytes only (without [u32 length] prefix) - * @param num_values Total logical values in page (includes nulls) - * @param max_def_level Maximum definition level for the column - * @return Count of decoded definition levels equal to max_def_level - */ -size_t CountPresentFromDefinitionLevelsV1( - tcb::span def_payload, - int32_t num_values, - int32_t max_def_level); - // ----------------------------------------------------------------------------- // Functions to decompress and split a Parquet page into level and value bytes. // ----------------------------------------------------------------------------- diff --git a/src/processing/parquet_utils_test.cpp b/src/processing/parquet_utils_test.cpp index 316d8e3..f98e88d 100644 --- a/src/processing/parquet_utils_test.cpp +++ b/src/processing/parquet_utils_test.cpp @@ -32,12 +32,30 @@ using namespace dbps::external; using namespace dbps::compression; using namespace dbps::processing; +// Forward declaration for internal helper tested here without exposing it in parquet_utils.h. +uint32_t ReadV1RunHeaderUleb128(tcb::span bytes, size_t& offset); + +size_t CountPresentValuesFromDefinitionLevelsV1( + tcb::span def_payload, + int32_t num_values, + int32_t max_def_level); + +int CalculateLevelBytesLength(tcb::span raw, + const AttributesMap& encoding_attribs); + // ----------------------------------------------------------------------------- // Helper functions to generate Parquet DATA_PAGE_V1 level bytes payloads for testing. // ----------------------------------------------------------------------------- namespace { +// Encodes an unsigned integer as ULEB128 bytes (base-128 varint) for test payload construction. +// Each output byte stores 7 data bits; the MSB is a continuation flag. +// - Example: 6 -> {0x06} (single byte, continuation flag not set) +// - Example: 300 -> {0xAC, 0x02}: +// 300 = 2*128 + 44, so the first 7-bit chunk is 44 (0x2C) and the remaining value is 2. +// First byte is 0x2C | 0x80 = 0xAC (set continuation bit because more chunks follow), +// second byte is 0x02 (final chunk, continuation bit cleared). std::vector EncodeUleb128(uint32_t value) { std::vector out; do { @@ -51,6 +69,7 @@ std::vector EncodeUleb128(uint32_t value) { return out; } +// Builds a DATA_PAGE_V1 RLE run payload: varint run header + repeated level value bytes. std::vector MakeRleDefPayload(uint32_t run_len, uint32_t level, int bit_width) { std::vector out = EncodeUleb128(run_len << 1); // RLE run const size_t byte_width = static_cast((bit_width + 7) / 8); @@ -60,6 +79,7 @@ std::vector MakeRleDefPayload(uint32_t run_len, uint32_t level, int bit return out; } +// Packs level values into Parquet's LSB-first bit-packed byte layout for a given bit width. std::vector PackBitPackedLevels( const std::vector& levels, int bit_width) { @@ -77,6 +97,7 @@ std::vector PackBitPackedLevels( return out; } +// Builds a DATA_PAGE_V1 bit-packed run payload: varint run header + packed level bytes. std::vector MakeBitPackedDefPayload( const std::vector& levels, int bit_width) { @@ -88,6 +109,7 @@ std::vector MakeBitPackedDefPayload( return out; } +// Prepends a 4-byte little-endian length prefix used by DATA_PAGE_V1 level streams. std::vector WrapLengthPrefixed(const std::vector& payload) { std::vector out; append_u32_le(out, static_cast(payload.size())); @@ -228,54 +250,54 @@ TEST(ParquetUtils, CalculateLevelBytesLength_NegativeTotalSize) { } // ----------------------------------------------------------------------------- -// Tests for CountPresentFromDefinitionLevelsV1 function. +// Tests for CountPresentValuesFromDefinitionLevelsV1 function. // ----------------------------------------------------------------------------- -TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_RleAllPresent) { +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_RleAllPresent) { auto def_payload = MakeRleDefPayload(10, 1, 1); - EXPECT_EQ(10u, CountPresentFromDefinitionLevelsV1(def_payload, 10, 1)); + EXPECT_EQ(10u, CountPresentValuesFromDefinitionLevelsV1(def_payload, 10, 1)); } -TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_RleAllNull) { +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_RleAllNull) { auto def_payload = MakeRleDefPayload(10, 0, 1); - EXPECT_EQ(0u, CountPresentFromDefinitionLevelsV1(def_payload, 10, 1)); + EXPECT_EQ(0u, CountPresentValuesFromDefinitionLevelsV1(def_payload, 10, 1)); } -TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_BitPackedMixed) { +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_BitPackedMixed) { std::vector levels = {1, 0, 1, 0, 1, 0, 1, 0}; auto def_payload = MakeBitPackedDefPayload(levels, 1); - EXPECT_EQ(4u, CountPresentFromDefinitionLevelsV1(def_payload, 8, 1)); + EXPECT_EQ(4u, CountPresentValuesFromDefinitionLevelsV1(def_payload, 8, 1)); } -TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_MixedRuns) { +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_MixedRuns) { auto rle_part = MakeRleDefPayload(4, 1, 1); // 4 present std::vector levels = {0, 1, 0, 1, 0, 0, 0, 0}; // +2 present auto bp_part = MakeBitPackedDefPayload(levels, 1); rle_part.insert(rle_part.end(), bp_part.begin(), bp_part.end()); - EXPECT_EQ(6u, CountPresentFromDefinitionLevelsV1(rle_part, 12, 1)); + EXPECT_EQ(6u, CountPresentValuesFromDefinitionLevelsV1(rle_part, 12, 1)); } -TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_InvalidRunLength) { +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_InvalidRunLength) { auto def_payload = MakeRleDefPayload(9, 1, 1); // run_len > num_values - EXPECT_THROW(CountPresentFromDefinitionLevelsV1(def_payload, 8, 1), InvalidInputException); + EXPECT_THROW(CountPresentValuesFromDefinitionLevelsV1(def_payload, 8, 1), InvalidInputException); } -TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_InvalidLevelExceedsMax) { +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_InvalidLevelExceedsMax) { auto def_payload = MakeRleDefPayload(1, 2, 1); // level 2 > max_def_level 1 - EXPECT_THROW(CountPresentFromDefinitionLevelsV1(def_payload, 1, 1), InvalidInputException); + EXPECT_THROW(CountPresentValuesFromDefinitionLevelsV1(def_payload, 1, 1), InvalidInputException); } -TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_TruncatedVarint) { +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_TruncatedVarint) { std::vector def_payload = {0x80}; // continuation bit set, no next byte - EXPECT_THROW(CountPresentFromDefinitionLevelsV1(def_payload, 1, 1), InvalidInputException); + EXPECT_THROW(CountPresentValuesFromDefinitionLevelsV1(def_payload, 1, 1), InvalidInputException); } -TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_TruncatedRleValue) { +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_TruncatedRleValue) { auto def_payload = EncodeUleb128(2); // run_len = 1, but missing repeated value byte - EXPECT_THROW(CountPresentFromDefinitionLevelsV1(def_payload, 1, 1), InvalidInputException); + EXPECT_THROW(CountPresentValuesFromDefinitionLevelsV1(def_payload, 1, 1), InvalidInputException); } -TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_BitPackedCanonical_0to7_88C6FA) { +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_BitPackedCanonical_0to7_88C6FA) { // Canonical example from Parquet Encodings.md: // values 0..7 with bit_width=3 are packed as bytes 0x88, 0xC6, 0xFA. // Bit-packed header for one group of 8 values is varint((1 << 1) | 1) = 0x03. @@ -286,42 +308,42 @@ TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_BitPackedCanonical_0to7_88 // encoding contract and can leave unread trailing bytes by design. std::vector def_payload = {0x03, 0x88, 0xC6, 0xFA}; - EXPECT_EQ(1u, CountPresentFromDefinitionLevelsV1(def_payload, 8, 7)); // only value 7 + EXPECT_EQ(1u, CountPresentValuesFromDefinitionLevelsV1(def_payload, 8, 7)); // only value 7 // Separate bit_width=2 payload for max_def_level=3. // This keeps payload bit-width consistent with decoder configuration and // avoids mixing a 3-bit packed stream with a 2-bit decode expectation. std::vector levels_bw2 = {0, 1, 2, 3, 0, 1, 2, 0}; // one value at level 3 auto def_payload_bw2 = MakeBitPackedDefPayload(levels_bw2, 2); - EXPECT_EQ(1u, CountPresentFromDefinitionLevelsV1(def_payload_bw2, 8, 3)); + EXPECT_EQ(1u, CountPresentValuesFromDefinitionLevelsV1(def_payload_bw2, 8, 3)); } -TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_ManualBytes_RleRunLen4_Level1) { +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_ManualBytes_RleRunLen4_Level1) { // Manual payload: // - header 0x08 => RLE run, run_len = 0x08 >> 1 = 4 // - repeated value byte = 0x01 (bit_width=1, level=1) std::vector def_payload = {0x08, 0x01}; - EXPECT_EQ(4u, CountPresentFromDefinitionLevelsV1(def_payload, 4, 1)); + EXPECT_EQ(4u, CountPresentValuesFromDefinitionLevelsV1(def_payload, 4, 1)); } -TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_ManualBytes_BitPackedAlternating_0xAA) { +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_ManualBytes_BitPackedAlternating_0xAA) { // Manual payload: // - header 0x03 => bit-packed, num_groups = 1, run_len = 8 // - packed byte 0xAA => bits (LSB->MSB): 0,1,0,1,0,1,0,1 std::vector def_payload = {0x03, 0xAA}; - EXPECT_EQ(4u, CountPresentFromDefinitionLevelsV1(def_payload, 8, 1)); + EXPECT_EQ(4u, CountPresentValuesFromDefinitionLevelsV1(def_payload, 8, 1)); } -TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_ManualBytes_MixedRleAndBitPacked) { +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_ManualBytes_MixedRleAndBitPacked) { // Manual payload: // - 0x06,0x01 => RLE run_len=3, level=1 // - 0x03,0x0F => bit-packed 8 values, bits: 1,1,1,1,0,0,0,0 // Total values = 3 + 8 = 11; present count at max_def_level=1 is 3 + 4 = 7. std::vector def_payload = {0x06, 0x01, 0x03, 0x0F}; - EXPECT_EQ(7u, CountPresentFromDefinitionLevelsV1(def_payload, 11, 1)); + EXPECT_EQ(7u, CountPresentValuesFromDefinitionLevelsV1(def_payload, 11, 1)); } -TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_BitWidth1_ExhaustiveOneGroup) { +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_BitWidth1_ExhaustiveOneGroup) { // Exhaustive check for all 8-value bit-packed patterns at bit_width=1. // Payload form: [0x03][packed-byte], where 0x03 means one bit-packed group (8 values). for (int packed = 0; packed <= 0xFF; ++packed) { @@ -332,46 +354,82 @@ TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_BitWidth1_ExhaustiveOneGro ones += static_cast((packed >> bit) & 0x01); } - EXPECT_EQ(ones, CountPresentFromDefinitionLevelsV1(def_payload, 8, 1)) + EXPECT_EQ(ones, CountPresentValuesFromDefinitionLevelsV1(def_payload, 8, 1)) << "packed=0x" << std::hex << packed; } } -TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_RejectsZeroRleRunLength) { +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_RejectsZeroRleRunLength) { // Header 0 means RLE with run_len = 0, which is invalid. std::vector def_payload = {0x00, 0x00}; - EXPECT_THROW(CountPresentFromDefinitionLevelsV1(def_payload, 1, 1), InvalidInputException); + EXPECT_THROW(CountPresentValuesFromDefinitionLevelsV1(def_payload, 1, 1), InvalidInputException); } -TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_RejectsZeroBitPackedGroups) { +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_RejectsZeroBitPackedGroups) { // Header 1 means bit-packed with num_groups = 0, which is invalid. std::vector def_payload = {0x01}; - EXPECT_THROW(CountPresentFromDefinitionLevelsV1(def_payload, 8, 1), InvalidInputException); + EXPECT_THROW(CountPresentValuesFromDefinitionLevelsV1(def_payload, 8, 1), InvalidInputException); } -TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_BitPackedFinalRunAllowsPadding) { +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_BitPackedFinalRunAllowsPadding) { // Corner case: a final bit-packed run is encoded as a full 8-value group, // while logical num_values ends mid-group. Decode only the logical values // and ignore padded trailing values in the last group. // Payload: header=0x03 (1 bit-packed group => 8 values), packed=0x07 (bits 1,1,1,0,0,0,0,0). std::vector def_payload = {0x03, 0x07}; - EXPECT_EQ(3u, CountPresentFromDefinitionLevelsV1(def_payload, 3, 1)); + EXPECT_EQ(3u, CountPresentValuesFromDefinitionLevelsV1(def_payload, 3, 1)); } -TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_RejectsTrailingBytesAfterDecoding) { +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_RejectsTrailingBytesAfterDecoding) { // One full bit-packed group (8 values) plus extra trailing byte that must be rejected. std::vector def_payload = {0x03, 0xAA, 0xFF}; - EXPECT_THROW(CountPresentFromDefinitionLevelsV1(def_payload, 8, 1), InvalidInputException); + EXPECT_THROW(CountPresentValuesFromDefinitionLevelsV1(def_payload, 8, 1), InvalidInputException); } -TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_RejectsNonPositiveMaxDefLevel) { +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_RejectsNonPositiveMaxDefLevel) { auto def_payload = MakeRleDefPayload(1, 0, 1); - EXPECT_THROW(CountPresentFromDefinitionLevelsV1(def_payload, 1, 0), InvalidInputException); + EXPECT_THROW(CountPresentValuesFromDefinitionLevelsV1(def_payload, 1, 0), InvalidInputException); } -TEST(ParquetUtils, CountPresentFromDefinitionLevelsV1_RejectsNegativeNumValues) { +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_RejectsNegativeNumValues) { auto def_payload = MakeRleDefPayload(1, 1, 1); - EXPECT_THROW(CountPresentFromDefinitionLevelsV1(def_payload, -1, 1), InvalidInputException); + EXPECT_THROW(CountPresentValuesFromDefinitionLevelsV1(def_payload, -1, 1), InvalidInputException); +} + +// ----------------------------------------------------------------------------- +// Tests for ReadV1RunHeaderUleb128 helper function. +// ----------------------------------------------------------------------------- + +TEST(ParquetUtils, ReadV1RunHeaderUleb128_SingleByteHeader) { + std::vector bytes = {0x06}; // value 6 + size_t offset = 0; + + const uint32_t header = ReadV1RunHeaderUleb128(bytes, offset); + EXPECT_EQ(6u, header); + EXPECT_EQ(1u, offset); +} + +TEST(ParquetUtils, ReadV1RunHeaderUleb128_MultiByteHeaderAndOffsetAdvance) { + std::vector bytes = {0xAA, 0xAC, 0x02}; // second varint = 300 + size_t offset = 1; + + const uint32_t header = ReadV1RunHeaderUleb128(bytes, offset); + EXPECT_EQ(300u, header); + EXPECT_EQ(3u, offset); +} + +TEST(ParquetUtils, ReadV1RunHeaderUleb128_TruncatedVarintThrows) { + std::vector bytes = {0x80}; // continuation bit set, but no following byte + size_t offset = 0; + + EXPECT_THROW(ReadV1RunHeaderUleb128(bytes, offset), InvalidInputException); +} + +TEST(ParquetUtils, ReadV1RunHeaderUleb128_VarintTooLargeThrows) { + std::vector bytes = {0x80, 0x80, 0x80, 0x80, 0x80}; + size_t offset = 0; + + EXPECT_THROW(ReadV1RunHeaderUleb128(bytes, offset), InvalidInputException); } // ----------------------------------------------------------------------------- diff --git a/src/processing/typed_buffer.h b/src/processing/typed_buffer.h index 0e84dfc..4868ca9 100644 --- a/src/processing/typed_buffer.h +++ b/src/processing/typed_buffer.h @@ -92,9 +92,14 @@ class ByteBuffer { // Variables for span elements reading tcb::span elements_span_; size_t elements_span_size_; - const size_t num_elements_; Codec codec_; + // Variable for the number of elements in the buffer. + // - `num_elements_` is a `const` and passed during construction of both read-only and write buffers. + // - It indicates the expected number of elements in the buffer payload declared upfront. + // - This is treated as an invariant, so if the payload count mismatches, exceptions are thrown. + const size_t num_elements_; + // Variables for element span iterator. mutable const uint8_t* element_iterator_current_ptr_; const uint8_t* element_iterator_end_ptr_; @@ -152,8 +157,12 @@ ByteBuffer::ByteBuffer( elements_span_size_(elements_span.size()), num_elements_(num_elements), codec_(std::move(codec)), + // `element_iterator_current_ptr_` is initialized to the start of the span. + // - it points to the start of the span + the prefix size if there is one. element_iterator_current_ptr_( elements_span.data() + std::min(prefix_size, elements_span_size_)), + // `element_iterator_end_ptr_` is initialized to the end of the span. + // - it points to the start of the span + the size. element_iterator_end_ptr_(elements_span.data() + elements_span_size_), element_iterator_count_(0), prefix_size_(prefix_size), @@ -165,7 +174,11 @@ ByteBuffer::ByteBuffer( template inline size_t ByteBuffer::InitElementSize(const Codec& codec) { if constexpr (is_fixed_sized) { - return codec.element_size(); + auto codec_element_size = codec.element_size(); + if (codec_element_size <= 0) { + throw InvalidInputException("Invalid fixed-size buffer: element_size must be greater than zero"); + } + return codec_element_size; } // For variable-size elements, element size is undefined. return kUnsetSize; @@ -199,6 +212,7 @@ inline void ByteBuffer::InitializeFromSpan() const { } // Check if the num_elements passed at contruction time coincides with the calculated from the payload size. + // This is a division of integer values, however it results in a correct integer result because of the modulo guard above. const size_t num_elements_on_payload = readable_size / element_size_; if (num_elements_on_payload != num_elements_) { throw InvalidInputException("Malformed fixed-size buffer: num_elements on payload != num_elements_ expected."); From f7c57e425cfd6c40a5ce93825d1642c1b5fb8427 Mon Sep 17 00:00:00 2001 From: Alejandro Valerio Date: Mon, 16 Mar 2026 17:32:35 -0600 Subject: [PATCH 17/18] - Comment update. --- src/processing/typed_buffer.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/processing/typed_buffer.h b/src/processing/typed_buffer.h index 4868ca9..f22f636 100644 --- a/src/processing/typed_buffer.h +++ b/src/processing/typed_buffer.h @@ -94,8 +94,8 @@ class ByteBuffer { size_t elements_span_size_; Codec codec_; - // Variable for the number of elements in the buffer. - // - `num_elements_` is a `const` and passed during construction of both read-only and write buffers. + // Attribute for the number of elements in the buffer (a const) + // - `num_elements_` is a const and passed during construction of both read-only and write buffers. // - It indicates the expected number of elements in the buffer payload declared upfront. // - This is treated as an invariant, so if the payload count mismatches, exceptions are thrown. const size_t num_elements_; From 145f85cefaa3358bcae91167768337b8247e7975 Mon Sep 17 00:00:00 2001 From: Alejandro Valerio Date: Mon, 16 Mar 2026 19:26:07 -0600 Subject: [PATCH 18/18] - Updating comments after code review. --- src/processing/encryption_sequencer.cpp | 5 ----- src/processing/typed_buffer.h | 6 ++++-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/processing/encryption_sequencer.cpp b/src/processing/encryption_sequencer.cpp index f5f3e0b..0bee26e 100644 --- a/src/processing/encryption_sequencer.cpp +++ b/src/processing/encryption_sequencer.cpp @@ -147,9 +147,6 @@ bool DataBatchEncryptionSequencer::DecodeAndEncrypt(tcb::span pla // Encrypt the typed values buffer and level bytes, then join them into a single encrypted byte vector. auto encrypted_value_bytes = encryptor_->EncryptValueList(typed_buffer); auto encrypted_level_bytes = encryptor_->EncryptBlock(level_bytes); - - // Encrypted payloads mostly have a low-compression ratio, so the gains in size from compression are minimal or negative. - // Therefore, the final joined encrypted bytes are returned as-is without compression. encrypted_result_ = JoinWithLengthPrefix(encrypted_level_bytes, encrypted_value_bytes); // Set the encryption type to per-value @@ -232,9 +229,7 @@ bool DataBatchEncryptionSequencer::DecryptAndEncode(tcb::span cip if (encryption_mode == ENCRYPTION_MODE_PER_VALUE) { // Split the joined encrypted bytes, then decrypt the level and value bytes separately. - // The ciphertext payload is already the joined bytes without compression. auto [encrypted_level_bytes, encrypted_value_bytes] = SplitWithLengthPrefix(ciphertext); - auto level_bytes = encryptor_->DecryptBlock(encrypted_level_bytes); auto typed_buffer = encryptor_->DecryptValueList(encrypted_value_bytes); diff --git a/src/processing/typed_buffer.h b/src/processing/typed_buffer.h index f22f636..21f55e0 100644 --- a/src/processing/typed_buffer.h +++ b/src/processing/typed_buffer.h @@ -155,14 +155,16 @@ ByteBuffer::ByteBuffer( Codec codec) : elements_span_(elements_span), elements_span_size_(elements_span.size()), + // `num_elements_` is the expected number of elements in the buffer declared upfront. + // - if the actual payload count mismatches, exceptions are thrown. num_elements_(num_elements), codec_(std::move(codec)), // `element_iterator_current_ptr_` is initialized to the start of the span. - // - it points to the start of the span + the prefix size if there is one. + // - it is calculated to point to the start of the span + the prefix size if there is one. element_iterator_current_ptr_( elements_span.data() + std::min(prefix_size, elements_span_size_)), // `element_iterator_end_ptr_` is initialized to the end of the span. - // - it points to the start of the span + the size. + // - it is calculated to point to the start of the span + the size. element_iterator_end_ptr_(elements_span.data() + elements_span_size_), element_iterator_count_(0), prefix_size_(prefix_size),