Skip to content

Commit a10fbf1

Browse files
avalerio-tkdavalerio-tkd
committed
- Remove UNDEFINED enum value from Type enum
- Updated unittests to use new BuildByteArrayValueBytes misc function when creating BYTE_ARRAY test data - Removed support for Type::UNDEFINED along the codebase.
1 parent 344e719 commit a10fbf1

13 files changed

Lines changed: 93 additions & 180 deletions

‎src/common/dbpa_local_test.cpp‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
// under the License.
1717

1818
#include "dbpa_local.h"
19+
#include "../processing/parquet_utils.h"
1920
#include <gtest/gtest.h>
2021
#include <memory>
2122
#include <vector>
@@ -45,9 +46,9 @@ TEST_F(LocalDataBatchProtectionAgentTest, SuccessfulEncryption) {
4546
std::string app_context = R"({"user_id": "test_user"})";
4647

4748
EXPECT_NO_THROW(agent.init("test_column", configuration_map, app_context, "test_key",
48-
Type::UNDEFINED, std::nullopt, CompressionCodec::UNCOMPRESSED, std::nullopt));
49+
Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, std::nullopt));
4950

50-
std::vector<uint8_t> test_data = {1, 2, 3, 4};
51+
std::vector<uint8_t> test_data = BuildByteArrayValueBytes("test_ABC");
5152
std::map<std::string, std::string> encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}};
5253
auto result = agent.Encrypt(test_data, encoding_attributes);
5354

@@ -88,9 +89,9 @@ TEST_F(LocalDataBatchProtectionAgentTest, SuccessfulDecryption) {
8889
std::string app_context = R"({"user_id": "test_user"})";
8990

9091
EXPECT_NO_THROW(agent.init("test_column", configuration_map, app_context, "test_key",
91-
Type::UNDEFINED, std::nullopt, CompressionCodec::UNCOMPRESSED, DBPS_ENCRYPTION_METADATA));
92+
Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, DBPS_ENCRYPTION_METADATA));
9293

93-
std::vector<uint8_t> test_data = {1, 2, 3, 4};
94+
std::vector<uint8_t> test_data = BuildByteArrayValueBytes("test_EFG");
9495
std::map<std::string, std::string> encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}};
9596
auto result = agent.Decrypt(test_data, encoding_attributes);
9697

@@ -107,10 +108,10 @@ TEST_F(LocalDataBatchProtectionAgentTest, RoundTripEncryptDecrypt) {
107108
std::string app_context = R"({"user_id": "test_user"})";
108109

109110
EXPECT_NO_THROW(encrypt_agent.init("test_column", configuration_map, app_context, "test_key",
110-
Type::UNDEFINED, std::nullopt, CompressionCodec::UNCOMPRESSED, std::nullopt));
111+
Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, std::nullopt));
111112

112113
// Original data to encrypt
113-
std::vector<uint8_t> original_data = {1, 2, 3, 4, 5};
114+
std::vector<uint8_t> original_data = BuildByteArrayValueBytes("roundtrip_XYZ");
114115
std::map<std::string, std::string> encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}};
115116

116117
// Encrypt the data
@@ -133,7 +134,7 @@ TEST_F(LocalDataBatchProtectionAgentTest, RoundTripEncryptDecrypt) {
133134
// Create a new agent for decryption with the encryption_metadata from the encryption result
134135
LocalDataBatchProtectionAgent decrypt_agent;
135136
EXPECT_NO_THROW(decrypt_agent.init("test_column", configuration_map, app_context, "test_key",
136-
Type::UNDEFINED, std::nullopt, CompressionCodec::UNCOMPRESSED, encryption_metadata));
137+
Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, encryption_metadata));
137138

138139
// Decrypt the ciphertext
139140
auto decrypt_result = decrypt_agent.Decrypt(ciphertext, encoding_attributes);

‎src/common/enum_utils.cpp‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ constexpr std::optional<E> from_string_impl(
5252
// For dbps::external::Type
5353
namespace {
5454
using T = ::dbps::external::Type::type;
55-
inline constexpr std::array<std::pair<T, std::string_view>, 9> kTypePairs{{
55+
inline constexpr std::array<std::pair<T, std::string_view>, 8> kTypePairs{{
5656
{T::BOOLEAN, "BOOLEAN"},
5757
{T::INT32, "INT32"},
5858
{T::INT64, "INT64"},
@@ -61,7 +61,6 @@ inline constexpr std::array<std::pair<T, std::string_view>, 9> kTypePairs{{
6161
{T::DOUBLE, "DOUBLE"},
6262
{T::BYTE_ARRAY, "BYTE_ARRAY"},
6363
{T::FIXED_LEN_BYTE_ARRAY, "FIXED_LEN_BYTE_ARRAY"},
64-
{T::UNDEFINED, "UNDEFINED"},
6564
}};
6665
} // anon
6766

‎src/common/enum_utils_test.cpp‎

Lines changed: 6 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@ TEST(EnumUtils, TypeToStringConversion) {
3434
ASSERT_EQ("DOUBLE", std::string(to_string(Type::DOUBLE)));
3535
ASSERT_EQ("BYTE_ARRAY", std::string(to_string(Type::BYTE_ARRAY)));
3636
ASSERT_EQ("FIXED_LEN_BYTE_ARRAY", std::string(to_string(Type::FIXED_LEN_BYTE_ARRAY)));
37-
ASSERT_EQ("UNDEFINED", std::string(to_string(Type::UNDEFINED)));
3837
}
3938

4039
TEST(EnumUtils, TypeFromStringConversion) {
@@ -69,10 +68,6 @@ TEST(EnumUtils, TypeFromStringConversion) {
6968
result = to_datatype_enum("FIXED_LEN_BYTE_ARRAY");
7069
ASSERT_TRUE(result.has_value());
7170
ASSERT_EQ(Type::FIXED_LEN_BYTE_ARRAY, result.value());
72-
73-
result = to_datatype_enum("UNDEFINED");
74-
ASSERT_TRUE(result.has_value());
75-
ASSERT_EQ(Type::UNDEFINED, result.value());
7671
}
7772

7873
TEST(EnumUtils, TypeInvalidFromString) {
@@ -246,7 +241,7 @@ TEST(EnumUtils, RoundTripTypeConversion) {
246241
// Test all Type enum values
247242
Type::type types[] = {
248243
Type::BOOLEAN, Type::INT32, Type::INT64, Type::INT96,
249-
Type::FLOAT, Type::DOUBLE, Type::BYTE_ARRAY, Type::FIXED_LEN_BYTE_ARRAY, Type::UNDEFINED
244+
Type::FLOAT, Type::DOUBLE, Type::BYTE_ARRAY, Type::FIXED_LEN_BYTE_ARRAY
250245
};
251246

252247
for (auto type : types) {
@@ -347,7 +342,7 @@ TEST(EnumUtils, TypeEnumCompleteness) {
347342
// Define all known Type enum values
348343
Type::type all_types[] = {
349344
Type::BOOLEAN, Type::INT32, Type::INT64, Type::INT96,
350-
Type::FLOAT, Type::DOUBLE, Type::BYTE_ARRAY, Type::FIXED_LEN_BYTE_ARRAY, Type::UNDEFINED
345+
Type::FLOAT, Type::DOUBLE, Type::BYTE_ARRAY, Type::FIXED_LEN_BYTE_ARRAY
351346
};
352347

353348
// Test that every enum value can be converted to string and back
@@ -406,12 +401,12 @@ TEST(EnumUtils, StringUniqueness) {
406401
// Collect all Type strings
407402
Type::type all_types[] = {
408403
Type::BOOLEAN, Type::INT32, Type::INT64, Type::INT96,
409-
Type::FLOAT, Type::DOUBLE, Type::BYTE_ARRAY, Type::FIXED_LEN_BYTE_ARRAY, Type::UNDEFINED
404+
Type::FLOAT, Type::DOUBLE, Type::BYTE_ARRAY, Type::FIXED_LEN_BYTE_ARRAY
410405
};
411406
for (auto type : all_types) {
412407
type_strings.insert(std::string(to_string(type)));
413408
}
414-
ASSERT_EQ(9, type_strings.size()); // All strings should be unique
409+
ASSERT_EQ(8, type_strings.size()); // All strings should be unique
415410

416411
// Collect all CompressionCodec strings
417412
CompressionCodec::type all_codecs[] = {
@@ -444,7 +439,7 @@ TEST(EnumUtils, CrossEnumStringCollision) {
444439
// Collect all strings from all enums
445440
Type::type all_types[] = {
446441
Type::BOOLEAN, Type::INT32, Type::INT64, Type::INT96,
447-
Type::FLOAT, Type::DOUBLE, Type::BYTE_ARRAY, Type::FIXED_LEN_BYTE_ARRAY, Type::UNDEFINED
442+
Type::FLOAT, Type::DOUBLE, Type::BYTE_ARRAY, Type::FIXED_LEN_BYTE_ARRAY
448443
};
449444
for (auto type : all_types) {
450445
all_strings.insert(std::string(to_string(type)));
@@ -467,12 +462,8 @@ TEST(EnumUtils, CrossEnumStringCollision) {
467462
all_strings.insert(std::string(to_string(encoding)));
468463
}
469464

470-
// Verify two equally named enums are handled correctly.
471-
ASSERT_EQ("UNDEFINED", std::string(to_string(Type::UNDEFINED)));
472465
ASSERT_EQ("UNDEFINED", std::string(to_string(Encoding::UNDEFINED)));
473466

474-
// Total should be 9 + 10 + 11 = 30 unique strings, but we have 1 collision
475-
// (Type::UNDEFINED and Encoding::UNDEFINED both map to "UNDEFINED")
476-
// So we expect 29 unique strings
467+
// Total should be 8 + 10 + 11 = 29 unique strings
477468
ASSERT_EQ(29, all_strings.size());
478469
}

‎src/common/enums.h‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,7 @@ struct Type {
3131
FLOAT = 4,
3232
DOUBLE = 5,
3333
BYTE_ARRAY = 6,
34-
FIXED_LEN_BYTE_ARRAY = 7,
35-
UNDEFINED = 8
34+
FIXED_LEN_BYTE_ARRAY = 7
3635
};
3736
};
3837

‎src/processing/encryption_sequencer_test.cpp‎

Lines changed: 5 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
#include "encryption_sequencer.h"
1919
#include "compression_utils.h"
20+
#include "parquet_utils.h"
2021
#include "../common/enums.h"
2122
#include "../common/bytes_utils.h"
2223
#include <iostream>
@@ -30,40 +31,11 @@ using namespace dbps::compression;
3031

3132
using namespace dbps::external;
3233

33-
// TODO: Move this to a common test utility file.
34-
// Methods that will pad byte arrays of strings (or pure bytes) with preceding
35-
// bytes that specify the array length. Needed because this is how Parquet
36-
// encodings represent their data.
37-
std::vector<uint8_t> EncodeStringByteArray(const std::vector<std::string>& strings) {
38-
std::vector<uint8_t> result;
39-
for (const auto& str : strings) {
40-
uint32_t len = str.size();
41-
// Add 4-byte length prefix (little-endian)
42-
result.push_back(len & 0xFF);
43-
result.push_back((len >> 8) & 0xFF);
44-
result.push_back((len >> 16) & 0xFF);
45-
result.push_back((len >> 24) & 0xFF);
46-
// Add string data
47-
result.insert(result.end(), str.begin(), str.end());
48-
}
49-
return result;
50-
}
51-
52-
std::vector<uint8_t> EncodePlainByteArray(const std::vector<uint8_t>& payload) {
53-
std::vector<uint8_t> out;
54-
uint32_t len = static_cast<uint32_t>(payload.size());
55-
out.push_back(static_cast<uint8_t>( len & 0xFF));
56-
out.push_back(static_cast<uint8_t>((len >> 8) & 0xFF));
57-
out.push_back(static_cast<uint8_t>((len >> 16) & 0xFF));
58-
out.push_back(static_cast<uint8_t>((len >> 24) & 0xFF));
59-
out.insert(out.end(), payload.begin(), payload.end());
60-
return out;
61-
}
62-
6334
// Test data constants - pure binary data
64-
const std::vector<uint8_t> HELLO_WORLD_DATA = EncodeStringByteArray({"Hello, World!"});
65-
const std::vector<uint8_t> BINARY_DATA = EncodePlainByteArray({0x00, 0x01, 0x02, 0x03, 0x04, 0x05});
66-
const std::vector<uint8_t> SINGLE_CHAR_DATA = EncodeStringByteArray({"A"});
35+
const std::vector<uint8_t> HELLO_WORLD_DATA = BuildByteArrayValueBytes("Hello, World!");
36+
const std::vector<uint8_t> BINARY_DATA = BuildByteArrayValueBytes(
37+
std::string("\x00\x01\x02\x03\x04\x05", 6));
38+
const std::vector<uint8_t> SINGLE_CHAR_DATA = BuildByteArrayValueBytes("A");
6739
const std::vector<uint8_t> EMPTY_DATA = {};
6840
const std::vector<uint8_t> FIXED_LEN_BYTE_ARRAY_DATA = {
6941
'A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P',

‎src/processing/parquet_utils.cpp‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -127,8 +127,6 @@ inline static size_t GetFixedElemSizeOrThrow(Type::type datatype, const std::opt
127127
}
128128
return static_cast<size_t>(datatype_length.value());
129129
}
130-
case Type::UNDEFINED:
131-
return 1;
132130
case Type::BOOLEAN:
133131
throw InvalidInputException("BOOLEAN is bit-sized; not fixed byte-sized");
134132
case Type::BYTE_ARRAY:
@@ -254,6 +252,23 @@ std::vector<uint8_t> CombineRawBytesIntoValueBytes(
254252
return out;
255253
}
256254

255+
std::vector<uint8_t> BuildByteArrayValueBytes(const std::string& payload) {
256+
std::vector<RawValueBytes> elements;
257+
elements.emplace_back(payload.begin(), payload.end());
258+
return CombineRawBytesIntoValueBytes(
259+
elements, Type::BYTE_ARRAY, std::nullopt, Encoding::PLAIN);
260+
}
261+
262+
std::vector<std::string> ParseByteArrayListValueBytes(const std::vector<uint8_t>& bytes) {
263+
TypedListValues list = ParseValueBytesIntoTypedList(
264+
bytes, Type::BYTE_ARRAY, std::nullopt, Encoding::PLAIN);
265+
const auto* values = std::get_if<std::vector<std::string>>(&list);
266+
if (!values) {
267+
throw InvalidInputException("Expected BYTE_ARRAY values");
268+
}
269+
return *values;
270+
}
271+
257272
LevelAndValueBytes DecompressAndSplit(
258273
const std::vector<uint8_t>& plaintext,
259274
CompressionCodec::type compression,

‎src/processing/parquet_utils.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,16 @@ std::vector<uint8_t> CombineRawBytesIntoValueBytes(
6666
const std::optional<int>& datatype_length,
6767
Encoding::type encoding);
6868

69+
/**
70+
* Build BYTE_ARRAY value bytes for a single string payload.
71+
*/
72+
std::vector<uint8_t> BuildByteArrayValueBytes(const std::string& payload);
73+
74+
/**
75+
* Parse BYTE_ARRAY value bytes into a list of string payloads.
76+
*/
77+
std::vector<std::string> ParseByteArrayListValueBytes(const std::vector<uint8_t>& bytes);
78+
6979
/**
7080
* Decompresses and splits a Parquet page into level and value bytes.
7181
* Handles DATA_PAGE_V1, DATA_PAGE_V2 (including optional compression on value bytes),

‎src/processing/parquet_utils_test.cpp‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -347,6 +347,35 @@ TEST(ParquetUtils, CombineRawBytesIntoValueBytes_BYTE_ARRAY) {
347347
EXPECT_EQ(out, expected);
348348
}
349349

350+
TEST(ParquetUtils, BuildByteArrayValueBytes_SinglePayload) {
351+
std::vector<uint8_t> out = BuildByteArrayValueBytes("abc");
352+
ASSERT_EQ(out.size(), 7u);
353+
EXPECT_EQ(read_u32_le(out, 0), 3u);
354+
EXPECT_EQ(out[4], static_cast<uint8_t>('a'));
355+
EXPECT_EQ(out[5], static_cast<uint8_t>('b'));
356+
EXPECT_EQ(out[6], static_cast<uint8_t>('c'));
357+
}
358+
359+
TEST(ParquetUtils, ParseByteArrayListValueBytes_SinglePayload) {
360+
std::vector<uint8_t> bytes = BuildByteArrayValueBytes("hello");
361+
std::vector<std::string> out = ParseByteArrayListValueBytes(bytes);
362+
ASSERT_EQ(out.size(), 1u);
363+
EXPECT_EQ(out[0], "hello");
364+
}
365+
366+
TEST(ParquetUtils, ParseByteArrayListValueBytes_MultiplePayloads) {
367+
std::vector<RawValueBytes> elems = {
368+
{'a','b'},
369+
{'x','y','z'}
370+
};
371+
std::vector<uint8_t> bytes = CombineRawBytesIntoValueBytes(
372+
elems, Type::BYTE_ARRAY, std::nullopt, Encoding::PLAIN);
373+
std::vector<std::string> out = ParseByteArrayListValueBytes(bytes);
374+
ASSERT_EQ(out.size(), 2u);
375+
EXPECT_EQ(out[0], "ab");
376+
EXPECT_EQ(out[1], "xyz");
377+
}
378+
350379
TEST(ParquetUtils, CombineRawBytesIntoValueBytes_FIXED_LEN_BYTE_ARRAY_SizeMismatch) {
351380
// Expect length 3, but provide a 2-byte element -> should throw
352381
std::vector<RawValueBytes> elems = {

‎src/processing/typed_list_values.cpp‎

Lines changed: 0 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -135,17 +135,6 @@ TypedListValues BuildTypedListFromRawBytes(
135135
}
136136
return out;
137137
}
138-
case Type::UNDEFINED: {
139-
std::vector<uint8_t> out;
140-
for (size_t i = 0; i < elements_bytes.size(); ++i) {
141-
const std::vector<uint8_t>& bytes = elements_bytes[i];
142-
if (bytes.size() != 1) {
143-
throw std::runtime_error("DecryptTypedListValues: invalid UNDEFINED element size");
144-
}
145-
out.push_back(bytes[0]);
146-
}
147-
return out;
148-
}
149138
default:
150139
throw std::runtime_error("DecryptTypedListValues: unsupported datatype");
151140
}
@@ -203,7 +192,6 @@ const char* GetTypeName() {
203192
else if constexpr (std::is_same_v<T, std::vector<std::array<uint32_t, 3>>>) return "INT96";
204193
else if constexpr (std::is_same_v<T, std::vector<std::string>>)
205194
return "string (BYTE_ARRAY/FIXED_LEN_BYTE_ARRAY)";
206-
else if constexpr (std::is_same_v<T, std::vector<uint8_t>>) return "UNDEFINED (raw bytes)";
207195
else if constexpr (std::is_same_v<T, std::monostate>) return "empty/error";
208196
else return "unknown";
209197
}
@@ -227,28 +215,6 @@ std::string TypedListToString(const TypedListValues& list) {
227215
<< values[i][1] << ", " << values[i][2] << "]\n";
228216
}
229217
}
230-
else if constexpr (std::is_same_v<T, std::vector<uint8_t>>) {
231-
// Special case for UNDEFINED - raw bytes as hex
232-
out << "Decoded UNDEFINED type (raw bytes):\n";
233-
out << " Hex: ";
234-
for (size_t i = 0; i < values.size(); ++i) {
235-
out << std::hex << std::setw(2) << std::setfill('0')
236-
<< static_cast<int>(values[i]);
237-
if (i < values.size() - 1) out << " ";
238-
}
239-
out << std::dec << "\n"; // Reset to decimal
240-
241-
// Also show as string if printable
242-
out << " String: \"";
243-
for (uint8_t byte : values) {
244-
if (byte >= 32 && byte < 127) {
245-
out << static_cast<char>(byte);
246-
} else {
247-
out << ".";
248-
}
249-
}
250-
out << "\"\n";
251-
}
252218
else if constexpr (std::is_same_v<T, std::vector<std::string>>) {
253219
// String values with quotes and the length of the string.
254220
out << "Decoded " << GetTypeName<T>() << " values:\n";

‎src/processing/typed_list_values.h‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,7 @@ using TypedListValues = std::variant<
3232
std::vector<float>,
3333
std::vector<double>,
3434
std::vector<std::array<uint32_t, 3>>, // For INT96
35-
std::vector<std::string>, // For BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY
36-
std::vector<uint8_t> // For UNDEFINED, a plain untyped byte sequence.
35+
std::vector<std::string> // For BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY
3736
>;
3837

3938
/**

0 commit comments

Comments
 (0)