- Added support for Parquet utils to read/decode num_elements from headers+level bytes without actually reading the value bytes.

- num_elements for DICTIONARY_PAGE and DATA_PAGE_V2 pages read from the headers values directly (easy)
- num_elements for DATA_PAGE_V1 pages read from the level bytes and RLE bit decoding (hard!)
- Added unittests for RLE bit decoding and count present values from definition levels for DATA_PAGE_V1 pages.
- Updated all scripts and unittests for new Parquet parsing requirements.

Author: avalerio-tkd

src/common/dbpa_local_test.cpp

@@ -50,7 +50,7 @@ TEST_F(LocalDataBatchProtectionAgentTest, SuccessfulEncryption) {
                                Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, std::nullopt));
 
     std::vector<uint8_t> test_data = BuildByteArrayValueBytesForTesting("test_ABC");
-    std::map<std::string, std::string> encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}};
+    std::map<std::string, std::string> encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}};
     auto result = agent.Encrypt(test_data, encoding_attributes);
 
     ASSERT_NE(result, nullptr);
@@ -74,7 +74,7 @@ TEST_F(LocalDataBatchProtectionAgentTest, SuccessfulEncryptionCompressedDictiona
         0x03, 0x32, 0x92, 0x12, 0xF3, 0x80, 0x10, 0x00, 0xC7, 0xB8,
         0x50, 0xFC, 0x13, 0x00, 0x00, 0x00
     };
-    std::map<std::string, std::string> encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}};
+    std::map<std::string, std::string> encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}};
     auto result = agent.Encrypt(test_data_gzip, encoding_attributes);
 
     ASSERT_NE(result, nullptr);
@@ -93,7 +93,7 @@ TEST_F(LocalDataBatchProtectionAgentTest, SuccessfulDecryption) {
                                Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, DBPS_ENCRYPTION_METADATA));
 
     std::vector<uint8_t> test_data = BuildByteArrayValueBytesForTesting("test_EFG");
-    std::map<std::string, std::string> encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}};
+    std::map<std::string, std::string> encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}};
     auto result = agent.Decrypt(test_data, encoding_attributes);
 
     ASSERT_NE(result, nullptr);
@@ -113,7 +113,7 @@ TEST_F(LocalDataBatchProtectionAgentTest, RoundTripEncryptDecrypt) {
 
     // Original data to encrypt
     std::vector<uint8_t> original_data = BuildByteArrayValueBytesForTesting("roundtrip_XYZ");
-    std::map<std::string, std::string> encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}};
+    std::map<std::string, std::string> encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}};
 
     // Encrypt the data
     auto encrypt_result = encrypt_agent.Encrypt(original_data, encoding_attributes);
@@ -156,7 +156,7 @@ TEST_F(LocalDataBatchProtectionAgentTest, EncryptWithoutInit) {
     LocalDataBatchProtectionAgent agent;
 
     std::vector<uint8_t> test_data = {1, 2, 3, 4};
-    std::map<std::string, std::string> encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}};
+    std::map<std::string, std::string> encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}};
     auto result = agent.Encrypt(test_data, encoding_attributes);
 
     ASSERT_NE(result, nullptr);
@@ -171,7 +171,7 @@ TEST_F(LocalDataBatchProtectionAgentTest, DecryptWithoutInit) {
     LocalDataBatchProtectionAgent agent;
 
     std::vector<uint8_t> test_data = {1, 2, 3, 4};
-    std::map<std::string, std::string> encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}};
+    std::map<std::string, std::string> encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}};
     auto result = agent.Decrypt(test_data, encoding_attributes);
 
     ASSERT_NE(result, nullptr);
@@ -203,7 +203,7 @@ TEST_F(LocalDataBatchProtectionAgentTest, MissingPageEncoding) {
                                Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, std::nullopt));
 
     std::vector<uint8_t> test_data = {1, 2, 3, 4};
-    std::map<std::string, std::string> encoding_attributes = {{"page_type", "DICTIONARY_PAGE"}};
+    std::map<std::string, std::string> encoding_attributes = {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}};
     auto result = agent.Encrypt(test_data, encoding_attributes);
 
     ASSERT_NE(result, nullptr);

src/processing/encryption_sequencer.cpp

@@ -62,7 +62,7 @@ namespace {
         int64_t join_with_length_prefix_ns,
         int64_t compress_ns) {
 
-        std::cout << "+++++ DecodeAndEncrypt timings (microseconds + nanoseconds) +++++" << std::endl;
+        std::cout << "----- DecodeAndEncrypt timings (microseconds + nanoseconds) -----" << std::endl;
         PrintDurationLine("DecompressAndSplit", decompress_and_split_ns);
         PrintDurationLine("ParseValueBytesIntoTypedList", parse_value_bytes_into_typed_list_ns);
         PrintDurationLine("EncryptValueList", encrypt_value_list_ns);
@@ -174,7 +174,7 @@ bool DataBatchEncryptionSequencer::DecodeAndEncrypt(tcb::span<const uint8_t> pla
 
         // Decompress and split plaintext into level and value bytes
         auto stage_start = std::chrono::steady_clock::now();
-        auto [level_bytes, value_bytes] = DecompressAndSplit(
+        auto [level_bytes, value_bytes, num_elements] = DecompressAndSplit(
             plaintext, compression_, encoding_attributes_converted_);
         decompress_and_split_ns = std::chrono::duration_cast<std::chrono::nanoseconds>(
             std::chrono::steady_clock::now() - stage_start).count();
@@ -355,7 +355,7 @@ bool DataBatchEncryptionSequencer::ConvertEncodingAttributesToValues() {
             add_int("page_v2_num_nulls");
             add_bool("page_v2_is_compressed");
         } else if (page_type == "DICTIONARY_PAGE") {
-            // DICTIONARY_PAGE has no specific encoding attributes
+            add_int("dict_page_num_values");
         } else {
             throw InvalidInputException("Unexpected page type: " + page_type);
         }