Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 12 additions & 8 deletions src/server/encryption_sequencer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -129,8 +129,9 @@ bool DataBatchEncryptionSequencer::ConvertAndEncrypt(const std::vector<uint8_t>&

/*
* Note on try-catch block:
* - When fully done, ConvertAndEncrypt will support per-value encryption for all cases.
* - This try-catch block is transitional to allow features to be developed incrementally until all features are
* - When fully done, ConvertAndEncrypt will support per-value encryption for all cases, except for
* (1) BOOLEAN datatype and (2) RLE_DICTIONARY encoding.
* - This try-catch block allows features to be developed incrementally until all features are
* complete: Compressions, Encodings, Page types, Datatypes.
* - During development if a feature is not yet supported, UnsupportedExceptions are caught and the fallback to
* per-block encryption is used.
Expand Down Expand Up @@ -160,18 +161,21 @@ bool DataBatchEncryptionSequencer::ConvertAndEncrypt(const std::vector<uint8_t>&
// Allow fallback to per-block encryption, only for explicitly unsupported conditions. See note above.
catch (const DBPSUnsupportedException& e) {

// Compression: Only UNCOMPRESSED and SNAPPY are supported
// Compression: Only UNCOMPRESSED and SNAPPY are currently supported
const bool is_compression_supported = (compression_ == CompressionCodec::UNCOMPRESSED ||
compression_ == CompressionCodec::SNAPPY);

// Format: Only PLAIN is supported
const bool is_format_supported = (format_ == Format::PLAIN);
// Format: Only PLAIN is currently supported
// RLE_DICTIONARY is permanently unsupported for per-value encryption since the values are not present in the
// `plaintext` data, only references to them.
const bool is_format_supported = (format_ == Format::PLAIN && format_ != Format::RLE_DICTIONARY);

// Page type: All are supported (DATA_PAGE_V1, DATA_PAGE_V2, DICTIONARY_PAGE)
const bool is_page_supported = true;

// Datatype: All datatypes are supported.
const bool is_datatype_supported = true;

// Datatype: All datatypes are supported except BOOLEAN.
// BOOLEAN is permanently unsupported for per-value encryption and always defaults to per-block encryption.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

in here, I'd either (a) remove the 'permanently' wording and just leave it as "boolean defaults to per-value' OR (b) add a reason as to why it's permanently unsupported.

I believe (a) is the way to go.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done. Thanks.

const bool is_datatype_supported = (datatype_ != Type::BOOLEAN);

if (is_compression_supported && is_format_supported && is_page_supported && is_datatype_supported) {
// All conditions are supported, therefore an DBPSUnsupportedException exception should not have happened.
Expand Down
66 changes: 66 additions & 0 deletions src/server/encryption_sequencer_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -417,6 +417,72 @@ TEST(EncryptionSequencer, ResultStorage) {

}

// Test BOOLEAN type uses per-block encryption (not per-value)
TEST(EncryptionSequencer, BooleanTypeUsesPerBlockEncryption) {
// BOOLEAN is permanently unsupported for per-value operations
// and always defaults to per-block encryption
std::vector<uint8_t> boolean_data = {0xB4, 0xFF, 0x00}; // some boolean bit-packed data

DataBatchEncryptionSequencer sequencer(
"bool_column",
Type::BOOLEAN,
std::nullopt,
CompressionCodec::UNCOMPRESSED,
Format::PLAIN,
{{"page_type", "DICTIONARY_PAGE"}},
CompressionCodec::UNCOMPRESSED,
"test_key",
"test_user",
"{}",
{}
);

bool result = sequencer.ConvertAndEncrypt(boolean_data);
ASSERT_TRUE(result) << "BOOLEAN encryption failed: " << sequencer.error_stage_ << " - " << sequencer.error_message_;

// Verify it used per-block encryption mode

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

typo: "if", I believe.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed the wording.

ASSERT_TRUE(sequencer.encryption_metadata_.count("encrypt_mode_dict_page") > 0);

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

assertion makes sense, but maybe just do == 1 ?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done.

EXPECT_EQ(sequencer.encryption_metadata_.at("encrypt_mode_dict_page"), "per_block");

// Verify round-trip works
bool decrypt_result = sequencer.ConvertAndDecrypt(sequencer.encrypted_result_);
ASSERT_TRUE(decrypt_result) << "BOOLEAN decryption failed: " << sequencer.error_stage_ << " - " << sequencer.error_message_;
EXPECT_EQ(sequencer.decrypted_result_, boolean_data);
}

// Test RLE_DICTIONARY format uses per-block encryption (not per-value)
TEST(EncryptionSequencer, RleDictionaryFormatUsesPerBlockEncryption) {
// RLE_DICTIONARY is permanently unsupported for per-value operations
// since the values are not present in the data, only references to them
std::vector<uint8_t> rle_dict_data = {0x02, 0x00, 0x00, 0x00, 0x01}; // some RLE dictionary encoded data

DataBatchEncryptionSequencer sequencer(
"dict_column",
Type::INT32,
std::nullopt,
CompressionCodec::UNCOMPRESSED,
Format::RLE_DICTIONARY,
{{"page_type", "DICTIONARY_PAGE"}},
CompressionCodec::UNCOMPRESSED,
"test_key",
"test_user",
"{}",
{}
);

bool result = sequencer.ConvertAndEncrypt(rle_dict_data);
ASSERT_TRUE(result) << "RLE_DICTIONARY encryption failed: " << sequencer.error_stage_ << " - " << sequencer.error_message_;

// Verify it used per-block encryption mode
ASSERT_TRUE(sequencer.encryption_metadata_.count("encrypt_mode_dict_page") > 0);
EXPECT_EQ(sequencer.encryption_metadata_.at("encrypt_mode_dict_page"), "per_block");

// Verify round-trip works
bool decrypt_result = sequencer.ConvertAndDecrypt(sequencer.encrypted_result_);
ASSERT_TRUE(decrypt_result) << "RLE_DICTIONARY decryption failed: " << sequencer.error_stage_ << " - " << sequencer.error_message_;
EXPECT_EQ(sequencer.decrypted_result_, rle_dict_data);
}

// Test FIXED_LEN_BYTE_ARRAY validation
TEST(EncryptionSequencer, FixedLenByteArrayValidation) {

Expand Down
15 changes: 14 additions & 1 deletion src/server/parquet_utils.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,8 @@ inline static size_t GetFixedElemSizeOrThrow(Type::type datatype, const std::opt
}
case Type::UNDEFINED:
return 1;
case Type::BOOLEAN:
throw InvalidInputException("BOOLEAN is bit-sized; not fixed byte-sized");
case Type::BYTE_ARRAY:
throw InvalidInputException("BYTE_ARRAY is variable-length; not fixed-size");
default:
Expand All @@ -146,13 +148,19 @@ std::vector<RawValueBytes> SliceValueBytesIntoRawBytes(
// RLE_DICTIONARY is not supported for per-value operations since the values themselves are not present in the data,
// only references to them.
if (format == Format::RLE_DICTIONARY) {
throw DBPSUnsupportedException("Unsupported format: RLE_DICTIONARY is not supported for per-value operations");
throw DBPSUnsupportedException("Unsupported format: RLE_DICTIONARY is not supported for per-value operations "

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nice clarification.

"since values are not present in the data, only references to them.");
}

if (format != Format::PLAIN) {
throw DBPSUnsupportedException("On SliceValueBytesIntoRawBytes, unsupported format: " + std::string(to_string(format)));
}

// BOOLEAN: boolean values are bit-encoded and not expanded as bytes
if (datatype == Type::BOOLEAN) {
throw DBPSUnsupportedException("On SliceValueBytesIntoRawBytes, BOOLEAN datatype is not supported for converting to bytes.");
}

// Variable-length BYTE_ARRAY: parse [4-byte len][bytes...] elements in order.
// This is the Parquet specific encoding for BYTE_ARRAY.
if (datatype == Type::BYTE_ARRAY) {
Expand Down Expand Up @@ -211,6 +219,11 @@ std::vector<uint8_t> CombineRawBytesIntoValueBytes(
throw DBPSUnsupportedException("On CombineRawBytesIntoValueBytes, unsupported format: " + std::string(to_string(format)));
}

// BOOLEAN: boolean values are bit-encoded and not expanded as bytes
if (datatype == Type::BOOLEAN) {
throw DBPSUnsupportedException("On CombineRawBytesIntoValueBytes, BOOLEAN datatype is not supported for converting to bytes.");
}

if (datatype == Type::BYTE_ARRAY) {
std::vector<uint8_t> out;
size_t total = 0;
Expand Down
42 changes: 42 additions & 0 deletions src/server/parquet_utils_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,13 @@ TEST(ParquetUtils, ParseValueBytesIntoTypedList_UnsupportedFormat) {
DBPSUnsupportedException);
}

TEST(ParquetUtils, ParseValueBytesIntoTypedList_BOOLEAN_Throws) {
// BOOLEAN type is not supported for per-value parsing
std::vector<uint8_t> bytes = {0xB4}; // 8 boolean values bit-packed
EXPECT_THROW(ParseValueBytesIntoTypedList(bytes, Type::BOOLEAN, std::nullopt, Format::PLAIN),
DBPSUnsupportedException);
}

TEST(ParquetUtils, ParseValueBytesIntoTypedList_InvalidDataSize) {
std::vector<uint8_t> bytes = {0x01, 0x02, 0x03}; // 3 bytes, not divisible by sizeof(int32_t)
EXPECT_THROW(ParseValueBytesIntoTypedList(bytes, Type::INT32, std::nullopt, Format::PLAIN),
Expand All @@ -236,6 +243,22 @@ TEST(ParquetUtils, SliceValueBytesIntoRawBytes_INT32) {
EXPECT_EQ(out[1], (std::vector<uint8_t>{0x0D,0x0C,0x0B,0x0A}));
}

TEST(ParquetUtils, SliceValueBytesIntoRawBytes_BOOLEAN_Throws) {
// BOOLEAN is bit-packed and not supported for per-value slicing
std::vector<uint8_t> bytes = {0xB4}; // 8 boolean values bit-packed
EXPECT_THROW(
SliceValueBytesIntoRawBytes(bytes, Type::BOOLEAN, std::nullopt, Format::PLAIN),
DBPSUnsupportedException);
}

TEST(ParquetUtils, SliceValueBytesIntoRawBytes_BOOLEAN_MultipleBytes_Throws) {
// Multiple bytes of boolean data
std::vector<uint8_t> bytes = {0xFF, 0x00, 0xAA, 0x55}; // 32 boolean values
EXPECT_THROW(
SliceValueBytesIntoRawBytes(bytes, Type::BOOLEAN, std::nullopt, Format::PLAIN),
DBPSUnsupportedException);
}

TEST(ParquetUtils, SliceValueBytesIntoRawBytes_INT96) {
std::vector<uint8_t> bytes = {
0x01,0x02,0x03,0x04,
Expand Down Expand Up @@ -290,6 +313,25 @@ TEST(ParquetUtils, CombineRawBytesIntoValueBytes_INT32) {
EXPECT_EQ(out, (std::vector<uint8_t>{0x04,0x03,0x02,0x01, 0x0D,0x0C,0x0B,0x0A}));
}

TEST(ParquetUtils, CombineRawBytesIntoValueBytes_BOOLEAN_Throws) {
// BOOLEAN is bit-packed and not supported for per-value combining
std::vector<RawValueBytes> elems = {
{0x01}, // single byte representing boolean value(s)
{0x00}
};
EXPECT_THROW(
CombineRawBytesIntoValueBytes(elems, Type::BOOLEAN, std::nullopt, Format::PLAIN),
DBPSUnsupportedException);
}

TEST(ParquetUtils, CombineRawBytesIntoValueBytes_BOOLEAN_EmptyInput_Throws) {
// Even empty input should throw for BOOLEAN type
std::vector<RawValueBytes> elems;
EXPECT_THROW(
CombineRawBytesIntoValueBytes(elems, Type::BOOLEAN, std::nullopt, Format::PLAIN),
DBPSUnsupportedException);
}

TEST(ParquetUtils, CombineRawBytesIntoValueBytes_BYTE_ARRAY) {
std::vector<RawValueBytes> elems = {
{'h','i'},
Expand Down