Merge pull request #171 from protegrity/av_snappy_09

Adding Snappy to API server flow

Author: avalerio-tkd

CMakeLists.txt

@@ -82,6 +82,13 @@ FetchContent_Declare(
   GIT_TAG master
 )
 
+# Snappy is a fast compression/decompression library
+FetchContent_Declare(
+  snappy
+  GIT_REPOSITORY https://github.com/google/snappy.git
+  GIT_TAG 1.2.2
+)
+
 # Fetch dependencies
 
 # Disable dependency tests for cppcodec
@@ -92,6 +99,19 @@ set(BUILD_TESTING OFF CACHE BOOL "" FORCE)
 FetchContent_MakeAvailable(cppcodec)
 set(BUILD_TESTING ON CACHE BOOL "" FORCE)
 
+# Disable Snappy tests to avoid conflicts with our GoogleTest
+# Snappy includes its own GoogleTest which conflicts with our FetchContent GoogleTest
+set(SNAPPY_BUILD_TESTS OFF CACHE BOOL "" FORCE)
+FetchContent_MakeAvailable(snappy)
+set(SNAPPY_BUILD_TESTS ON CACHE BOOL "" FORCE)
+
+# Suppress warnings from Snappy's source code
+if(TARGET snappy)
+    target_compile_options(snappy PRIVATE
+        -Wno-sign-compare
+    )
+endif()
+
 FetchContent_MakeAvailable(crow httplib nlohmann_json cxxopts googletest jwt-cpp)
 
 # Make GoogleTest helper macros available (gtest_discover_tests)
@@ -118,13 +138,15 @@ add_library(dbps_server_lib STATIC
   src/server/encryption_sequencer.cpp
   src/server/auth_utils.cpp
   src/server/decoding_utils.cpp
+  src/server/compression_utils.cpp
 )
-target_link_libraries(dbps_server_lib PUBLIC dbps_common_lib)
+target_link_libraries(dbps_server_lib PUBLIC dbps_common_lib snappy)
 target_include_directories(dbps_server_lib PUBLIC
   src/server
   ${CMAKE_BINARY_DIR}/_deps/cppcodec-src
   ${CMAKE_BINARY_DIR}/_deps/jwt-cpp-src/include
   ${CMAKE_BINARY_DIR}/_deps/nlohmann_json-src/include
+  ${CMAKE_BINARY_DIR}/_deps/snappy-src
 )
 
 # Find and link OpenSSL (required by jwt-cpp)
@@ -200,9 +222,12 @@ target_link_libraries(dbpa_remote_testapp
   dbps_remote_lib
   dbps_client_lib
   dbps_common_lib
+  dbps_server_lib
 )
 target_include_directories(dbpa_remote_testapp PRIVATE
   ${CMAKE_BINARY_DIR}/_deps/cxxopts-src/include
+  src/server
+  ${CMAKE_BINARY_DIR}/_deps/snappy-src
 )
 
 # =============================================================================
@@ -244,6 +269,15 @@ if(BUILD_TESTS)
   )
   target_include_directories(decoding_utils_test PRIVATE src/server)
 
+  # Compression utils tests
+  add_executable(compression_utils_test src/server/compression_utils_test.cpp)
+  target_link_libraries(compression_utils_test
+    dbps_server_lib
+    dbps_common_lib
+    gtest_main
+  )
+  target_include_directories(compression_utils_test PRIVATE src/server)
+
   # Auth utils tests
   add_executable(auth_utils_test src/server/auth_utils_test.cpp)
   target_link_libraries(auth_utils_test
@@ -413,6 +447,7 @@ if(BUILD_TESTS)
       enum_utils_test
       encryption_sequencer_test
       decoding_utils_test
+      compression_utils_test
       auth_utils_test
       dbpa_interface_test
       dbpa_utils_test
@@ -429,6 +464,7 @@ if(BUILD_TESTS)
   gtest_discover_tests(enum_utils_test)
   gtest_discover_tests(encryption_sequencer_test)
   gtest_discover_tests(decoding_utils_test)
+  gtest_discover_tests(compression_utils_test)
   gtest_discover_tests(auth_utils_test)
   gtest_discover_tests(dbpa_interface_test)
   gtest_discover_tests(dbpa_utils_test)

src/scripts/dbpa_remote_testapp.cpp

@@ -28,10 +28,12 @@
 #include "../common/dbpa_remote.h"
 #include "../client/httplib_client.h"
 #include "../common/enums.h"
+#include "../server/compression_utils.h"
 #include "tcb/span.hpp"
 
 using namespace dbps::external;
 using namespace dbps::enum_utils;
+using namespace dbps::compression;
 
 template <typename T>
 using span = tcb::span<T>;
@@ -143,7 +145,7 @@ class DBPARemoteTestApp {
                 "demo_fixed_len_key_001",      // column_key_id
                 Type::FIXED_LEN_BYTE_ARRAY,   // datatype
                 8,                            // datatype_length (8 bytes per element)
-                CompressionCodec::UNCOMPRESSED, // compression_type
+                CompressionCodec::SNAPPY,      // compression_type (input will be Snappy-compressed)
                 VALID_ENCRYPTION_METADATA       // column_encryption_metadata
             );
 
@@ -464,11 +466,16 @@ class DBPARemoteTestApp {
             }
 
             std::cout << "Test data: 3 fixed-length strings (8 bytes each)" << std::endl;
-            std::cout << "Total size: " << fixed_length_data.size() << " bytes" << std::endl;
+            std::cout << "Original size: " << fixed_length_data.size() << " bytes" << std::endl;
 
+            // Compress the test data using Snappy before sending to server
+            std::vector<uint8_t> fixed_length_data_compressed = Compress(fixed_length_data, CompressionCodec::SNAPPY);
+            std::cout << "Compressed size: " << fixed_length_data_compressed.size() << " bytes" << std::endl;
+
             // Test encryption with FIXED_LEN_BYTE_ARRAY and datatype_length
+            // The server will decompress the fixed_length_data_compressed, encrypt it, and compress the encrypted result
             std::map<std::string, std::string> fixed_len_encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}};
-            auto encrypt_result = fixed_len_agent_->Encrypt(span<const uint8_t>(fixed_length_data), fixed_len_encoding_attributes);
+            auto encrypt_result = fixed_len_agent_->Encrypt(span<const uint8_t>(fixed_length_data_compressed), fixed_len_encoding_attributes);
 
             if (!encrypt_result || !encrypt_result->success()) {
                 std::cout << "ERROR: FIXED_LEN_BYTE_ARRAY encryption failed" << std::endl;
@@ -493,13 +500,22 @@ class DBPARemoteTestApp {
 
             std::cout << "OK: FIXED_LEN_BYTE_ARRAY decrypted successfully" << std::endl;
 
+            // The server returns compressed plaintext (same compression as input)
+            // Decompress it before comparing with original data
+            auto decrypted_compressed_span = decrypt_result->plaintext();
+            std::vector<uint8_t> decrypted_compressed(decrypted_compressed_span.begin(), decrypted_compressed_span.end());
+            std::vector<uint8_t> decrypted_data = Decompress(decrypted_compressed, CompressionCodec::SNAPPY);
+            std::cout << "Decrypted compressed size: " << decrypted_compressed.size() << " bytes" << std::endl;
+            std::cout << "Decrypted uncompressed size: " << decrypted_data.size() << " bytes" << std::endl;
+            
             // Verify data integrity
-            auto decrypted_data = decrypt_result->plaintext();
             if (decrypted_data.size() == fixed_length_data.size() && 
-                std::equal(decrypted_data.begin(), decrypted_data.end(), fixed_length_data.begin())) {
+                std::equal(decrypted_data.begin(), decrypted_data.end(), fixed_length_data.begin(), fixed_length_data.end())) {
                 std::cout << "OK: FIXED_LEN_BYTE_ARRAY data integrity verified" << std::endl;
             } else {
                 std::cout << "ERROR: FIXED_LEN_BYTE_ARRAY data integrity check failed" << std::endl;
+                std::cout << "  Expected size: " << fixed_length_data.size() << " bytes" << std::endl;
+                std::cout << "  Got size: " << decrypted_data.size() << " bytes" << std::endl;
                 return false;
             }
 

src/server/compression_utils.cpp

@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "compression_utils.h"
+#include <snappy.h>
+
+using namespace dbps::external;
+using namespace dbps::enum_utils;
+
+namespace dbps::compression {
+
+std::vector<uint8_t> Compress(const std::vector<uint8_t>& bytes, CompressionCodec::type compression) {
+    if (compression == CompressionCodec::UNCOMPRESSED) {
+        return bytes;
+    }
+    
+    if (compression == CompressionCodec::SNAPPY) {
+        if (bytes.empty()) {
+            return bytes;
+        }
+        // `compressed` is a std::string because Snappy's API requires it.
+        // It is used as a binary buffer (not text), and immediately converted back to std::vector<uint8_t> to preserve binary semantics.
+        std::string compressed;
+        snappy::Compress(reinterpret_cast<const char*>(bytes.data()), bytes.size(), &compressed);
+        return std::vector<uint8_t>(compressed.begin(), compressed.end());
+    }
+    
+    // Note for future implementations: If compression fails because of invalid or corrupt input,
+    // then throw an InvalidInputException.
+    throw DBPSUnsupportedException(
+        "Unsupported compression codec: " + std::string(to_string(compression)));
+}
+
+std::vector<uint8_t> Decompress(const std::vector<uint8_t>& bytes, CompressionCodec::type compression) {
+    if (compression == CompressionCodec::UNCOMPRESSED) {
+        return bytes;
+    }
+    
+    if (compression == CompressionCodec::SNAPPY) {
+        if (bytes.empty()) {
+            return bytes;
+        }
+        // `decompressed` is a std::string because Snappy's API requires it.
+        // It is used as a binary buffer (not text), and immediately converted back to std::vector<uint8_t> to preserve binary semantics.
+        std::string decompressed;
+        bool success = snappy::Uncompress(reinterpret_cast<const char*>(bytes.data()), bytes.size(), &decompressed);
+        if (!success) {
+            throw InvalidInputException("Failed to decompress data: invalid or corrupt Snappy-compressed input");
+        }
+        return std::vector<uint8_t>(decompressed.begin(), decompressed.end());
+    }
+    
+    // Note for future implementations: If decompression fails because of invalid or corrupt input,
+    // then throw an InvalidInputException.
+    throw DBPSUnsupportedException(
+        "Unsupported compression codec: " + std::string(to_string(compression)));
+}
+
+} // namespace dbps::compression

src/server/compression_utils.h

@@ -26,6 +26,8 @@
 using namespace dbps::external;
 using namespace dbps::enum_utils;
 
+namespace dbps::compression {
+
 /**
  * Compress bytes using the compression codec.
  * 
@@ -34,15 +36,7 @@ using namespace dbps::enum_utils;
  * @return Compressed bytes, or original bytes if UNCOMPRESSED
  * @throws DBPSUnsupportedException if the compression codec is not supported
  */
-inline std::vector<uint8_t> Compress(const std::vector<uint8_t>& bytes, CompressionCodec::type compression) {
-    if (compression == CompressionCodec::UNCOMPRESSED) {
-        return bytes;
-    }
-    // Note for future implementations: If compression fails because of invalid or corrupt input,
-    // then throw an InvalidInputException.
-    throw DBPSUnsupportedException(
-        "Unsupported compression codec: " + std::string(to_string(compression)));
-}
+std::vector<uint8_t> Compress(const std::vector<uint8_t>& bytes, CompressionCodec::type compression);
 
 /**
  * Decompress bytes using the compression codec.
@@ -52,13 +46,6 @@ inline std::vector<uint8_t> Compress(const std::vector<uint8_t>& bytes, Compress
  * @return Decompressed bytes, or original bytes if UNCOMPRESSED
  * @throws DBPSUnsupportedException if the compression codec is not supported
  */
-inline std::vector<uint8_t> Decompress(const std::vector<uint8_t>& bytes, CompressionCodec::type compression) {
-    if (compression == CompressionCodec::UNCOMPRESSED) {
-        return bytes;
-    }
-    // Note for future implementations: If decompression fails because of invalid or corrupt input,
-    // then throw an InvalidInputException.
-    throw DBPSUnsupportedException(
-        "Unsupported compression codec: " + std::string(to_string(compression)));
-}
+std::vector<uint8_t> Decompress(const std::vector<uint8_t>& bytes, CompressionCodec::type compression);
 
+} // namespace dbps::compression