Adding ByteBuffer class for reading/writing elements more memory efficiently.

CMakeLists.txt

@@ -133,6 +133,16 @@ target_include_directories(dbps_common_lib PUBLIC
 )
 target_link_libraries(dbps_common_lib PUBLIC tcb_span)
 
+# Byte buffer processing library
+add_library(dbps_byte_buffer_lib STATIC
+  src/processing/byte_buffer.cpp
+)
+target_include_directories(dbps_byte_buffer_lib PUBLIC
+  src/processing
+  src/common
+)
+target_link_libraries(dbps_byte_buffer_lib PUBLIC tcb_span)
+
 # Server components library
 add_library(dbps_server_lib STATIC 
   src/processing/encryption_sequencer.cpp
@@ -205,7 +215,7 @@ target_include_directories(dbps_local_lib PUBLIC
 )
 
 # Ensure PIC on static libs that are linked into shared libraries
-set_target_properties(dbps_common_lib dbps_server_lib dbps_local_lib
+set_target_properties(dbps_common_lib dbps_byte_buffer_lib dbps_server_lib dbps_local_lib
   PROPERTIES POSITION_INDEPENDENT_CODE ON)
 
 # =============================================================================
@@ -325,6 +335,14 @@ if(BUILD_TESTS)
   )
   target_include_directories(compression_utils_test PRIVATE src/processing)
 
+  # Byte buffer tests
+  add_executable(byte_buffer_test src/processing/byte_buffer_test.cpp)
+  target_link_libraries(byte_buffer_test
+    dbps_byte_buffer_lib
+    gtest_main
+  )
+  target_include_directories(byte_buffer_test PRIVATE src/processing src/common)
+
   # Basic encryptor tests
   add_executable(basic_encryptor_test src/processing/encryptors/basic_encryptor_test.cpp)
   target_link_libraries(basic_encryptor_test
@@ -482,6 +500,7 @@ endif()
 add_custom_target(libraries
   DEPENDS 
     dbps_common_lib
+    dbps_byte_buffer_lib
     dbps_server_lib
     dbps_client_lib
     dbps_remote_lib
@@ -517,6 +536,7 @@ if(BUILD_TESTS)
       parquet_utils_test
       bytes_utils_test
       compression_utils_test
+      byte_buffer_test
       basic_encryptor_test
       auth_utils_test
       dbpa_interface_test
@@ -539,6 +559,7 @@ if(BUILD_TESTS)
   gtest_discover_tests(parquet_utils_test)
   gtest_discover_tests(bytes_utils_test)
   gtest_discover_tests(compression_utils_test)
+  gtest_discover_tests(byte_buffer_test)
   gtest_discover_tests(basic_encryptor_test)
   gtest_discover_tests(auth_utils_test)
   gtest_discover_tests(dbpa_interface_test)

src/common/bytes_utils.h

@@ -25,30 +25,35 @@
 #include <string>
 #include <variant>
 #include <cassert>
+#include <tcb/span.hpp>
 #include "exceptions.h"
 
 // Utility functions for little-endian number reading and writing.
 
 inline void append_u32_le(std::vector<uint8_t>& out, uint32_t v) {
-    out.push_back(static_cast<uint8_t>(v & 0xFF));
-    out.push_back(static_cast<uint8_t>((v >> 8) & 0xFF));
-    out.push_back(static_cast<uint8_t>((v >> 16) & 0xFF));
-    out.push_back(static_cast<uint8_t>((v >> 24) & 0xFF));
+    const size_t offset = out.size();
+    out.resize(offset + 4);
+    out[offset + 0] = static_cast<uint8_t>(v & 0xFF);
+    out[offset + 1] = static_cast<uint8_t>((v >> 8) & 0xFF);
+    out[offset + 2] = static_cast<uint8_t>((v >> 16) & 0xFF);
+    out[offset + 3] = static_cast<uint8_t>((v >> 24) & 0xFF);
 }
 
 inline void append_i32_le(std::vector<uint8_t>& out, int32_t v) {
     append_u32_le(out, static_cast<uint32_t>(v));
 }
 
 inline void append_u64_le(std::vector<uint8_t>& out, uint64_t v) {
-    out.push_back(static_cast<uint8_t>(v & 0xFF));
-    out.push_back(static_cast<uint8_t>((v >> 8) & 0xFF));
-    out.push_back(static_cast<uint8_t>((v >> 16) & 0xFF));
-    out.push_back(static_cast<uint8_t>((v >> 24) & 0xFF));
-    out.push_back(static_cast<uint8_t>((v >> 32) & 0xFF));
-    out.push_back(static_cast<uint8_t>((v >> 40) & 0xFF));
-    out.push_back(static_cast<uint8_t>((v >> 48) & 0xFF));
-    out.push_back(static_cast<uint8_t>((v >> 56) & 0xFF));
+    const size_t offset = out.size();
+    out.resize(offset + 8);
+    out[offset + 0] = static_cast<uint8_t>(v & 0xFF);
+    out[offset + 1] = static_cast<uint8_t>((v >> 8) & 0xFF);
+    out[offset + 2] = static_cast<uint8_t>((v >> 16) & 0xFF);
+    out[offset + 3] = static_cast<uint8_t>((v >> 24) & 0xFF);
+    out[offset + 4] = static_cast<uint8_t>((v >> 32) & 0xFF);
+    out[offset + 5] = static_cast<uint8_t>((v >> 40) & 0xFF);
+    out[offset + 6] = static_cast<uint8_t>((v >> 48) & 0xFF);
+    out[offset + 7] = static_cast<uint8_t>((v >> 56) & 0xFF);
 }
 
 inline void append_i64_le(std::vector<uint8_t>& out, int64_t v) {
@@ -74,6 +79,13 @@ inline uint32_t read_u32_le(const std::vector<uint8_t>& in, size_t offset) {
         (static_cast<uint32_t>(in[offset + 3]) << 24);
 }
 
+inline uint32_t read_u32_le(tcb::span<const uint8_t> in, size_t offset) {
+    return static_cast<uint32_t>(in[offset]) |
+        (static_cast<uint32_t>(in[offset + 1]) << 8) |
+        (static_cast<uint32_t>(in[offset + 2]) << 16) |
+        (static_cast<uint32_t>(in[offset + 3]) << 24);
+}
+
 // Utility functions for splitting and joining byte vectors.
 
 struct SplitBytesPair {