diff --git a/CMakeLists.txt b/CMakeLists.txt index 6b50f72..9a20c4d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -82,6 +82,13 @@ FetchContent_Declare( GIT_TAG master ) +# Snappy is a fast compression/decompression library +FetchContent_Declare( + snappy + GIT_REPOSITORY https://github.com/google/snappy.git + GIT_TAG 1.2.2 +) + # Fetch dependencies # Disable dependency tests for cppcodec @@ -92,6 +99,19 @@ set(BUILD_TESTING OFF CACHE BOOL "" FORCE) FetchContent_MakeAvailable(cppcodec) set(BUILD_TESTING ON CACHE BOOL "" FORCE) +# Disable Snappy tests to avoid conflicts with our GoogleTest +# Snappy includes its own GoogleTest which conflicts with our FetchContent GoogleTest +set(SNAPPY_BUILD_TESTS OFF CACHE BOOL "" FORCE) +FetchContent_MakeAvailable(snappy) +set(SNAPPY_BUILD_TESTS ON CACHE BOOL "" FORCE) + +# Suppress warnings from Snappy's source code +if(TARGET snappy) + target_compile_options(snappy PRIVATE + -Wno-sign-compare + ) +endif() + FetchContent_MakeAvailable(crow httplib nlohmann_json cxxopts googletest jwt-cpp) # Make GoogleTest helper macros available (gtest_discover_tests) @@ -118,13 +138,15 @@ add_library(dbps_server_lib STATIC src/server/encryption_sequencer.cpp src/server/auth_utils.cpp src/server/decoding_utils.cpp + src/server/compression_utils.cpp ) -target_link_libraries(dbps_server_lib PUBLIC dbps_common_lib) +target_link_libraries(dbps_server_lib PUBLIC dbps_common_lib snappy) target_include_directories(dbps_server_lib PUBLIC src/server ${CMAKE_BINARY_DIR}/_deps/cppcodec-src ${CMAKE_BINARY_DIR}/_deps/jwt-cpp-src/include ${CMAKE_BINARY_DIR}/_deps/nlohmann_json-src/include + ${CMAKE_BINARY_DIR}/_deps/snappy-src ) # Find and link OpenSSL (required by jwt-cpp) @@ -200,9 +222,12 @@ target_link_libraries(dbpa_remote_testapp dbps_remote_lib dbps_client_lib dbps_common_lib + dbps_server_lib ) target_include_directories(dbpa_remote_testapp PRIVATE ${CMAKE_BINARY_DIR}/_deps/cxxopts-src/include + src/server + ${CMAKE_BINARY_DIR}/_deps/snappy-src ) # ============================================================================= @@ -244,6 +269,15 @@ if(BUILD_TESTS) ) target_include_directories(decoding_utils_test PRIVATE src/server) + # Compression utils tests + add_executable(compression_utils_test src/server/compression_utils_test.cpp) + target_link_libraries(compression_utils_test + dbps_server_lib + dbps_common_lib + gtest_main + ) + target_include_directories(compression_utils_test PRIVATE src/server) + # Auth utils tests add_executable(auth_utils_test src/server/auth_utils_test.cpp) target_link_libraries(auth_utils_test @@ -413,6 +447,7 @@ if(BUILD_TESTS) enum_utils_test encryption_sequencer_test decoding_utils_test + compression_utils_test auth_utils_test dbpa_interface_test dbpa_utils_test @@ -429,6 +464,7 @@ if(BUILD_TESTS) gtest_discover_tests(enum_utils_test) gtest_discover_tests(encryption_sequencer_test) gtest_discover_tests(decoding_utils_test) + gtest_discover_tests(compression_utils_test) gtest_discover_tests(auth_utils_test) gtest_discover_tests(dbpa_interface_test) gtest_discover_tests(dbpa_utils_test) diff --git a/src/scripts/dbpa_remote_testapp.cpp b/src/scripts/dbpa_remote_testapp.cpp index 9ada009..85c3be6 100644 --- a/src/scripts/dbpa_remote_testapp.cpp +++ b/src/scripts/dbpa_remote_testapp.cpp @@ -28,10 +28,12 @@ #include "../common/dbpa_remote.h" #include "../client/httplib_client.h" #include "../common/enums.h" +#include "../server/compression_utils.h" #include "tcb/span.hpp" using namespace dbps::external; using namespace dbps::enum_utils; +using namespace dbps::compression; template using span = tcb::span; @@ -143,7 +145,7 @@ class DBPARemoteTestApp { "demo_fixed_len_key_001", // column_key_id Type::FIXED_LEN_BYTE_ARRAY, // datatype 8, // datatype_length (8 bytes per element) - CompressionCodec::UNCOMPRESSED, // compression_type + CompressionCodec::SNAPPY, // compression_type (input will be Snappy-compressed) VALID_ENCRYPTION_METADATA // column_encryption_metadata ); @@ -464,11 +466,16 @@ class DBPARemoteTestApp { } std::cout << "Test data: 3 fixed-length strings (8 bytes each)" << std::endl; - std::cout << "Total size: " << fixed_length_data.size() << " bytes" << std::endl; + std::cout << "Original size: " << fixed_length_data.size() << " bytes" << std::endl; + // Compress the test data using Snappy before sending to server + std::vector fixed_length_data_compressed = Compress(fixed_length_data, CompressionCodec::SNAPPY); + std::cout << "Compressed size: " << fixed_length_data_compressed.size() << " bytes" << std::endl; + // Test encryption with FIXED_LEN_BYTE_ARRAY and datatype_length + // The server will decompress the fixed_length_data_compressed, encrypt it, and compress the encrypted result std::map fixed_len_encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}}; - auto encrypt_result = fixed_len_agent_->Encrypt(span(fixed_length_data), fixed_len_encoding_attributes); + auto encrypt_result = fixed_len_agent_->Encrypt(span(fixed_length_data_compressed), fixed_len_encoding_attributes); if (!encrypt_result || !encrypt_result->success()) { std::cout << "ERROR: FIXED_LEN_BYTE_ARRAY encryption failed" << std::endl; @@ -493,13 +500,22 @@ class DBPARemoteTestApp { std::cout << "OK: FIXED_LEN_BYTE_ARRAY decrypted successfully" << std::endl; + // The server returns compressed plaintext (same compression as input) + // Decompress it before comparing with original data + auto decrypted_compressed_span = decrypt_result->plaintext(); + std::vector decrypted_compressed(decrypted_compressed_span.begin(), decrypted_compressed_span.end()); + std::vector decrypted_data = Decompress(decrypted_compressed, CompressionCodec::SNAPPY); + std::cout << "Decrypted compressed size: " << decrypted_compressed.size() << " bytes" << std::endl; + std::cout << "Decrypted uncompressed size: " << decrypted_data.size() << " bytes" << std::endl; + // Verify data integrity - auto decrypted_data = decrypt_result->plaintext(); if (decrypted_data.size() == fixed_length_data.size() && - std::equal(decrypted_data.begin(), decrypted_data.end(), fixed_length_data.begin())) { + std::equal(decrypted_data.begin(), decrypted_data.end(), fixed_length_data.begin(), fixed_length_data.end())) { std::cout << "OK: FIXED_LEN_BYTE_ARRAY data integrity verified" << std::endl; } else { std::cout << "ERROR: FIXED_LEN_BYTE_ARRAY data integrity check failed" << std::endl; + std::cout << " Expected size: " << fixed_length_data.size() << " bytes" << std::endl; + std::cout << " Got size: " << decrypted_data.size() << " bytes" << std::endl; return false; } diff --git a/src/server/compression_utils.cpp b/src/server/compression_utils.cpp new file mode 100644 index 0000000..0c6b90a --- /dev/null +++ b/src/server/compression_utils.cpp @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "compression_utils.h" +#include + +using namespace dbps::external; +using namespace dbps::enum_utils; + +namespace dbps::compression { + +std::vector Compress(const std::vector& bytes, CompressionCodec::type compression) { + if (compression == CompressionCodec::UNCOMPRESSED) { + return bytes; + } + + if (compression == CompressionCodec::SNAPPY) { + if (bytes.empty()) { + return bytes; + } + // `compressed` is a std::string because Snappy's API requires it. + // It is used as a binary buffer (not text), and immediately converted back to std::vector to preserve binary semantics. + std::string compressed; + snappy::Compress(reinterpret_cast(bytes.data()), bytes.size(), &compressed); + return std::vector(compressed.begin(), compressed.end()); + } + + // Note for future implementations: If compression fails because of invalid or corrupt input, + // then throw an InvalidInputException. + throw DBPSUnsupportedException( + "Unsupported compression codec: " + std::string(to_string(compression))); +} + +std::vector Decompress(const std::vector& bytes, CompressionCodec::type compression) { + if (compression == CompressionCodec::UNCOMPRESSED) { + return bytes; + } + + if (compression == CompressionCodec::SNAPPY) { + if (bytes.empty()) { + return bytes; + } + // `decompressed` is a std::string because Snappy's API requires it. + // It is used as a binary buffer (not text), and immediately converted back to std::vector to preserve binary semantics. + std::string decompressed; + bool success = snappy::Uncompress(reinterpret_cast(bytes.data()), bytes.size(), &decompressed); + if (!success) { + throw InvalidInputException("Failed to decompress data: invalid or corrupt Snappy-compressed input"); + } + return std::vector(decompressed.begin(), decompressed.end()); + } + + // Note for future implementations: If decompression fails because of invalid or corrupt input, + // then throw an InvalidInputException. + throw DBPSUnsupportedException( + "Unsupported compression codec: " + std::string(to_string(compression))); +} + +} // namespace dbps::compression diff --git a/src/server/compression_utils.h b/src/server/compression_utils.h index 1edf60e..f735916 100644 --- a/src/server/compression_utils.h +++ b/src/server/compression_utils.h @@ -26,6 +26,8 @@ using namespace dbps::external; using namespace dbps::enum_utils; +namespace dbps::compression { + /** * Compress bytes using the compression codec. * @@ -34,15 +36,7 @@ using namespace dbps::enum_utils; * @return Compressed bytes, or original bytes if UNCOMPRESSED * @throws DBPSUnsupportedException if the compression codec is not supported */ -inline std::vector Compress(const std::vector& bytes, CompressionCodec::type compression) { - if (compression == CompressionCodec::UNCOMPRESSED) { - return bytes; - } - // Note for future implementations: If compression fails because of invalid or corrupt input, - // then throw an InvalidInputException. - throw DBPSUnsupportedException( - "Unsupported compression codec: " + std::string(to_string(compression))); -} +std::vector Compress(const std::vector& bytes, CompressionCodec::type compression); /** * Decompress bytes using the compression codec. @@ -52,13 +46,6 @@ inline std::vector Compress(const std::vector& bytes, Compress * @return Decompressed bytes, or original bytes if UNCOMPRESSED * @throws DBPSUnsupportedException if the compression codec is not supported */ -inline std::vector Decompress(const std::vector& bytes, CompressionCodec::type compression) { - if (compression == CompressionCodec::UNCOMPRESSED) { - return bytes; - } - // Note for future implementations: If decompression fails because of invalid or corrupt input, - // then throw an InvalidInputException. - throw DBPSUnsupportedException( - "Unsupported compression codec: " + std::string(to_string(compression))); -} +std::vector Decompress(const std::vector& bytes, CompressionCodec::type compression); +} // namespace dbps::compression diff --git a/src/server/compression_utils_test.cpp b/src/server/compression_utils_test.cpp new file mode 100644 index 0000000..0def963 --- /dev/null +++ b/src/server/compression_utils_test.cpp @@ -0,0 +1,163 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "compression_utils.h" +#include "exceptions.h" +#include +#include +#include + +using namespace dbps::external; +using namespace dbps::compression; + +TEST(CompressionUtils, Compress_Uncompressed) { + std::vector input = {0x01, 0x02, 0x03, 0x04, 0x05}; + std::vector result = Compress(input, CompressionCodec::UNCOMPRESSED); + EXPECT_EQ(input, result); +} + +TEST(CompressionUtils, Decompress_Uncompressed) { + std::vector input = {0x01, 0x02, 0x03, 0x04, 0x05}; + std::vector result = Decompress(input, CompressionCodec::UNCOMPRESSED); + EXPECT_EQ(input, result); +} + +TEST(CompressionUtils, Compress_Empty) { + std::vector input; + std::vector result = Compress(input, CompressionCodec::SNAPPY); + EXPECT_EQ(input, result); +} + +TEST(CompressionUtils, Decompress_Empty) { + std::vector input; + std::vector result = Decompress(input, CompressionCodec::SNAPPY); + EXPECT_EQ(input, result); +} + +TEST(CompressionUtils, CompressDecompress_Snappy_RoundTrip) { + std::vector original = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}; + std::vector compressed = Compress(original, CompressionCodec::SNAPPY); + std::vector decompressed = Decompress(compressed, CompressionCodec::SNAPPY); + EXPECT_EQ(original, decompressed); + // Verify that compression actually occurred (compressed data should differ from original) + EXPECT_NE(original, compressed); +} + +TEST(CompressionUtils, Decompress_InvalidData) { + std::vector invalid_data = {0xFF, 0xFF, 0xFF, 0xFF}; + EXPECT_THROW(Decompress(invalid_data, CompressionCodec::SNAPPY), InvalidInputException); +} + +TEST(CompressionUtils, Compress_UnsupportedCodec) { + std::vector input = {0x01, 0x02, 0x03}; + EXPECT_THROW(Compress(input, CompressionCodec::GZIP), DBPSUnsupportedException); +} + +TEST(CompressionUtils, Decompress_UnsupportedCodec) { + std::vector input = {0x01, 0x02, 0x03}; + EXPECT_THROW(Decompress(input, CompressionCodec::GZIP), DBPSUnsupportedException); +} + +TEST(CompressionUtils, CompressDecompress_Snappy_SingleByte) { + std::vector original = {0x42}; + std::vector compressed = Compress(original, CompressionCodec::SNAPPY); + std::vector decompressed = Decompress(compressed, CompressionCodec::SNAPPY); + EXPECT_EQ(original, decompressed); + // Verify that compression actually occurred (compressed data should differ from original) + EXPECT_NE(original, compressed); +} + +TEST(CompressionUtils, CompressDecompress_Snappy_LargeData) { + std::vector original; + original.resize(10000); + for (size_t i = 0; i < original.size(); ++i) { + original[i] = static_cast(i % 256); + } + std::vector compressed = Compress(original, CompressionCodec::SNAPPY); + std::vector decompressed = Decompress(compressed, CompressionCodec::SNAPPY); + EXPECT_EQ(original, decompressed); + // Verify that compression actually occurred (compressed data should differ from original) + EXPECT_NE(original, compressed); +} + +TEST(CompressionUtils, CompressDecompress_Snappy_RepeatingPattern) { + std::vector original; + original.resize(1000, 0xAA); + std::vector compressed = Compress(original, CompressionCodec::SNAPPY); + std::vector decompressed = Decompress(compressed, CompressionCodec::SNAPPY); + EXPECT_EQ(original, decompressed); + EXPECT_LT(compressed.size(), original.size()); +} + +TEST(CompressionUtils, CompressDecompress_Snappy_TextData) { + std::string text = "The quick brown fox jumps over the lazy dog. This is a test string."; + std::vector original(text.begin(), text.end()); + std::vector compressed = Compress(original, CompressionCodec::SNAPPY); + std::vector decompressed = Decompress(compressed, CompressionCodec::SNAPPY); + EXPECT_EQ(original, decompressed); +} + +TEST(CompressionUtils, CompressDecompress_Snappy_BinaryData) { + std::vector original = {0x00, 0xFF, 0xAA, 0x55, 0x11, 0xEE, 0x99, 0x66}; + std::vector compressed = Compress(original, CompressionCodec::SNAPPY); + std::vector decompressed = Decompress(compressed, CompressionCodec::SNAPPY); + EXPECT_EQ(original, decompressed); +} + +TEST(CompressionUtils, CompressDecompress_Snappy_MultipleRoundTrips) { + std::vector original = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}; + std::vector data = original; + for (int i = 0; i < 5; ++i) { + data = Compress(data, CompressionCodec::SNAPPY); + data = Decompress(data, CompressionCodec::SNAPPY); + } + EXPECT_EQ(original, data); +} + +TEST(CompressionUtils, Compress_Snappy_CompressionRatio) { + std::vector original; + original.resize(5000, 0x42); + std::vector compressed = Compress(original, CompressionCodec::SNAPPY); + EXPECT_LE(compressed.size(), original.size()); + EXPECT_GT(compressed.size(), 0); +} + +TEST(CompressionUtils, Decompress_InvalidData_Empty) { + std::vector empty_data = {}; + std::vector result = Decompress(empty_data, CompressionCodec::SNAPPY); + EXPECT_EQ(empty_data, result); +} + +TEST(CompressionUtils, Decompress_InvalidData_TooShort) { + std::vector invalid_data = {0x01}; + EXPECT_THROW(Decompress(invalid_data, CompressionCodec::SNAPPY), InvalidInputException); +} + +TEST(CompressionUtils, Compress_UnsupportedCodec_AllTypes) { + std::vector input = {0x01, 0x02, 0x03}; + EXPECT_THROW(Compress(input, CompressionCodec::BROTLI), DBPSUnsupportedException); + EXPECT_THROW(Compress(input, CompressionCodec::ZSTD), DBPSUnsupportedException); + EXPECT_THROW(Compress(input, CompressionCodec::LZ4), DBPSUnsupportedException); +} + +TEST(CompressionUtils, Decompress_UnsupportedCodec_AllTypes) { + std::vector input = {0x01, 0x02, 0x03}; + EXPECT_THROW(Decompress(input, CompressionCodec::BROTLI), DBPSUnsupportedException); + EXPECT_THROW(Decompress(input, CompressionCodec::ZSTD), DBPSUnsupportedException); + EXPECT_THROW(Decompress(input, CompressionCodec::LZ4), DBPSUnsupportedException); +} + diff --git a/src/server/encryption_sequencer.cpp b/src/server/encryption_sequencer.cpp index 6aa416e..f670d79 100644 --- a/src/server/encryption_sequencer.cpp +++ b/src/server/encryption_sequencer.cpp @@ -29,6 +29,7 @@ using namespace dbps::external; using namespace dbps::enum_utils; +using namespace dbps::compression; namespace { constexpr const char* DBPS_VERSION_KEY = "dbps_agent_version";