Skip to content

Adding support for endianness in TypedBuffer codecs #233

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
avalerio-tkd merged 4 commits into from
Mar 19, 2026
Merged

Adding support for endianness in TypedBuffer codecs #233

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e0c093b
- Adding endianness protection to the codecs.
avalerio-tkd Mar 19, 2026
c1f393f
- Reimplemented BytesToString more efficiently.
avalerio-tkd Mar 19, 2026
2ed0c92
- Added unitests for write_le and read_le functions with expected buf…
avalerio-tkd Mar 19, 2026
be0c4a5
- Updated comments on code duplication.
avalerio-tkd Mar 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
122 changes: 110 additions & 12 deletions src/common/bytes_utils.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@

inline constexpr size_t kSizePrefixBytes = sizeof(uint32_t);

// Utility functions for little-endian number reading and writing.
// Utility functions for little-endian number reading and writing (vectors and spans)

inline void append_u32_le(std::vector<uint8_t>& out, uint32_t v) {
const size_t offset = out.size();
Expand Down Expand Up @@ -81,13 +81,6 @@ inline void write_u32_le_at(std::vector<uint8_t>& buf, size_t offset, uint32_t v
buf[offset + 3] = static_cast<uint8_t>((v >> 24) & 0xFF);
}

inline void write_u32_le(uint8_t* p, uint32_t v) {
p[0] = static_cast<uint8_t>(v);
p[1] = static_cast<uint8_t>(v >> 8);
p[2] = static_cast<uint8_t>(v >> 16);
p[3] = static_cast<uint8_t>(v >> 24);
}

inline uint32_t read_u32_le(const std::vector<uint8_t>& in, size_t offset) {
return static_cast<uint32_t>(in[offset]) |
(static_cast<uint32_t>(in[offset + 1]) << 8) |
Expand All @@ -102,10 +95,110 @@ inline uint32_t read_u32_le(tcb::span<const uint8_t> in, size_t offset) {
(static_cast<uint32_t>(in[offset + 3]) << 24);
}

// Utility functions for reading and writing with templated types.

template <class T>
inline T read_le(const uint8_t* p) {
Comment thread
argmarco-tkd marked this conversation as resolved.
if constexpr (std::is_same_v<T, int32_t>) {
const uint32_t v =
(static_cast<uint32_t>(p[0]) ) |
(static_cast<uint32_t>(p[1]) << 8) |
(static_cast<uint32_t>(p[2]) << 16) |
(static_cast<uint32_t>(p[3]) << 24);
return static_cast<int32_t>(v);
} else if constexpr (std::is_same_v<T, int64_t>) {
const uint64_t v =
(static_cast<uint64_t>(p[0]) ) |
(static_cast<uint64_t>(p[1]) << 8) |
(static_cast<uint64_t>(p[2]) << 16) |
(static_cast<uint64_t>(p[3]) << 24) |
(static_cast<uint64_t>(p[4]) << 32) |
(static_cast<uint64_t>(p[5]) << 40) |
(static_cast<uint64_t>(p[6]) << 48) |
(static_cast<uint64_t>(p[7]) << 56);
return static_cast<int64_t>(v);
} else if constexpr (std::is_same_v<T, float>) {
const uint32_t bits =
(static_cast<uint32_t>(p[0]) ) |
(static_cast<uint32_t>(p[1]) << 8) |
(static_cast<uint32_t>(p[2]) << 16) |
(static_cast<uint32_t>(p[3]) << 24);
float value;
std::memcpy(&value, &bits, sizeof(value));
return value;
} else if constexpr (std::is_same_v<T, double>) {
const uint64_t bits =
(static_cast<uint64_t>(p[0]) ) |
(static_cast<uint64_t>(p[1]) << 8) |
(static_cast<uint64_t>(p[2]) << 16) |
(static_cast<uint64_t>(p[3]) << 24) |
(static_cast<uint64_t>(p[4]) << 32) |
(static_cast<uint64_t>(p[5]) << 40) |
(static_cast<uint64_t>(p[6]) << 48) |
(static_cast<uint64_t>(p[7]) << 56);
double value;
std::memcpy(&value, &bits, sizeof(value));
return value;
} else {
throw InvalidInputException("read_le<T>: unsupported type");
}
}

template <class T>
inline void write_le(const T& value, uint8_t* p) {
if constexpr (std::is_same_v<T, int32_t>) {
const uint32_t v = static_cast<uint32_t>(value);
p[0] = static_cast<uint8_t>( v & 0xFF);
p[1] = static_cast<uint8_t>((v >> 8) & 0xFF);
p[2] = static_cast<uint8_t>((v >> 16) & 0xFF);
p[3] = static_cast<uint8_t>((v >> 24) & 0xFF);
} else if constexpr (std::is_same_v<T, int64_t>) {
const uint64_t v = static_cast<uint64_t>(value);
p[0] = static_cast<uint8_t>( v & 0xFF);
p[1] = static_cast<uint8_t>((v >> 8) & 0xFF);
p[2] = static_cast<uint8_t>((v >> 16) & 0xFF);
p[3] = static_cast<uint8_t>((v >> 24) & 0xFF);
p[4] = static_cast<uint8_t>((v >> 32) & 0xFF);
p[5] = static_cast<uint8_t>((v >> 40) & 0xFF);
p[6] = static_cast<uint8_t>((v >> 48) & 0xFF);
p[7] = static_cast<uint8_t>((v >> 56) & 0xFF);
} else if constexpr (std::is_same_v<T, float>) {
uint32_t bits;
std::memcpy(&bits, &value, sizeof(bits));
p[0] = static_cast<uint8_t>( bits & 0xFF);
p[1] = static_cast<uint8_t>((bits >> 8) & 0xFF);
p[2] = static_cast<uint8_t>((bits >> 16) & 0xFF);
p[3] = static_cast<uint8_t>((bits >> 24) & 0xFF);
} else if constexpr (std::is_same_v<T, double>) {
uint64_t bits;
std::memcpy(&bits, &value, sizeof(bits));
p[0] = static_cast<uint8_t>( bits & 0xFF);
p[1] = static_cast<uint8_t>((bits >> 8) & 0xFF);
p[2] = static_cast<uint8_t>((bits >> 16) & 0xFF);
p[3] = static_cast<uint8_t>((bits >> 24) & 0xFF);
p[4] = static_cast<uint8_t>((bits >> 32) & 0xFF);
p[5] = static_cast<uint8_t>((bits >> 40) & 0xFF);
p[6] = static_cast<uint8_t>((bits >> 48) & 0xFF);
p[7] = static_cast<uint8_t>((bits >> 56) & 0xFF);
} else {
throw InvalidInputException("write_le<T>: unsupported type");
}
}

// Utility functions for little-endian number reading and writing (pointers)

inline void write_u32_le(uint8_t* p, uint32_t v) {
p[0] = static_cast<uint8_t>(v);
p[1] = static_cast<uint8_t>(v >> 8);
p[2] = static_cast<uint8_t>(v >> 16);
p[3] = static_cast<uint8_t>(v >> 24);
}

inline uint32_t read_u32_le(const uint8_t* p) {
uint32_t v;
std::memcpy(&v, p, sizeof(v));
return v;
return static_cast<uint32_t>(p[0]) |
(static_cast<uint32_t>(p[1]) << 8) |
(static_cast<uint32_t>(p[2]) << 16) |
(static_cast<uint32_t>(p[3]) << 24);
Comment thread
argmarco-tkd marked this conversation as resolved.
}

// Utility functions for splitting and joining byte vectors.
Expand Down Expand Up @@ -303,7 +396,12 @@ inline std::string AddStringAttribute(
return value;
}

// Helper function to convert string to binary data
// Helper function to convert string to binary data and vice versa

inline std::vector<uint8_t> StringToBytes(const std::string& str) {
return std::vector<uint8_t>(str.begin(), str.end());
}

inline std::string BytesToString(tcb::span<const uint8_t> span) {
return std::string(reinterpret_cast<const char*>(span.data()), span.size());
}
Comment thread
argmarco-tkd marked this conversation as resolved.
84 changes: 84 additions & 0 deletions src/common/bytes_utils_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
#include "bytes_utils.h"
#include "exceptions.h"

#include <array>
#include <cmath>
#include <vector>
#include <variant>
#include <gtest/gtest.h>
Expand Down Expand Up @@ -284,4 +286,86 @@ TEST(BytesUtils, StringToBytes_PreservesRawBytesAndNulls) {
static_cast<uint8_t>('Z')};

EXPECT_EQ(expected, result);
}

TEST(BytesUtils, BytesToString_ConvertsAsciiEmptyAndRawBytes) {
{
const std::vector<uint8_t> bytes = {'d', 'b', 'p', 's'};
const std::string result = BytesToString(tcb::span<const uint8_t>(bytes));
EXPECT_EQ(result, "dbps");
}

{
const std::vector<uint8_t> bytes;
const std::string result = BytesToString(tcb::span<const uint8_t>(bytes));
EXPECT_TRUE(result.empty());
}

{
const std::vector<uint8_t> bytes = {
static_cast<uint8_t>('D'),
static_cast<uint8_t>('B'),
static_cast<uint8_t>('P'),
static_cast<uint8_t>('S'),
static_cast<uint8_t>(0x00),
static_cast<uint8_t>('X'),
static_cast<uint8_t>('Y'),
static_cast<uint8_t>(0xFF),
static_cast<uint8_t>(0x80),
static_cast<uint8_t>(0x00),
static_cast<uint8_t>('Z')};
const std::string result = BytesToString(tcb::span<const uint8_t>(bytes));
const std::string expected = std::string{
'D', 'B', 'P', 'S', '\0', 'X', 'Y',
static_cast<char>(0xFF), static_cast<char>(0x80), '\0', 'Z'};
EXPECT_EQ(result.size(), expected.size());
EXPECT_EQ(result, expected);
}
}

TEST(BytesUtils, ReadU32Le_FromPointer_DecodesLittleEndianBytes) {
const std::array<uint8_t, 4> bytes = {0x78, 0x56, 0x34, 0x12};
const uint32_t value = read_u32_le(bytes.data());

EXPECT_EQ(value, 0x12345678u);
}

TEST(BytesUtils, ReadLeWriteLe_Int32_RoundTrip) {
const int32_t original = -2147483000;
std::array<uint8_t, sizeof(int32_t)> bytes{};
Comment thread
argmarco-tkd marked this conversation as resolved.

write_le<int32_t>(original, bytes.data());
const int32_t decoded = read_le<int32_t>(bytes.data());

EXPECT_EQ(decoded, original);
}

TEST(BytesUtils, ReadLeWriteLe_Int64_RoundTrip) {
const int64_t original = -9223372036854000000LL;
std::array<uint8_t, sizeof(int64_t)> bytes{};

write_le<int64_t>(original, bytes.data());
const int64_t decoded = read_le<int64_t>(bytes.data());

EXPECT_EQ(decoded, original);
}

TEST(BytesUtils, ReadLeWriteLe_Float_RoundTrip) {
const float original = -12345.625f;
std::array<uint8_t, sizeof(float)> bytes{};

write_le<float>(original, bytes.data());
const float decoded = read_le<float>(bytes.data());

EXPECT_FLOAT_EQ(decoded, original);
}

TEST(BytesUtils, ReadLeWriteLe_Double_RoundTrip) {
const double original = 9876543210.125;
std::array<uint8_t, sizeof(double)> bytes{};

write_le<double>(original, bytes.data());
const double decoded = read_le<double>(bytes.data());

EXPECT_DOUBLE_EQ(decoded, original);
}
63 changes: 52 additions & 11 deletions src/processing/typed_buffer_codecs.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,19 @@
#include <cstring>
#include <string_view>
#include <tcb/span.hpp>
#include "bytes_utils.h"
#include "exceptions.h"

namespace dbps::processing {

// The values in Int96 of low/mid/hi are stored in little endian order.
// The order of low/mid/hi in the C++ struct should be kept, otherwise the codec will yield incorrect values.
struct Int96 {
Comment thread
argmarco-tkd marked this conversation as resolved.
int32_t lo;
int32_t mid;
int32_t hi;
};

template <class T, const char* TypeName>
struct PlainValueCodec {
using value_type = T;
Expand All @@ -43,21 +52,53 @@ struct PlainValueCodec {
return sizeof(T);
}

// TODO: Make explicit endianness conversions to prevent architecture/in-memory representation incompatibility issues.
value_type Decode(tcb::span<const uint8_t> read_span) const {
inline value_type Decode(tcb::span<const uint8_t> read_span) const {
if (read_span.size() != sizeof(T)) {
throw InvalidInputException("Decode: read_span size does not match sizeof(T)");
}
T value;
std::memcpy(&value, read_span.data(), sizeof(T));
return value;
return read_le<T>(read_span.data());
}

void Encode(const value_type& value, tcb::span<uint8_t> write_span) const {
inline void Encode(const value_type& value, tcb::span<uint8_t> write_span) const {
if (write_span.size() != sizeof(T)) {
throw InvalidInputException("Encode: write_span size does not match sizeof(T)");
}
std::memcpy(write_span.data(), &value, sizeof(T));
write_le<T>(value, write_span.data());
}
};

struct Int96Codec {
Comment thread
argmarco-tkd marked this conversation as resolved.
using value_type = Int96;
static constexpr bool is_fixed_sized = true;
static constexpr size_t kI32Size = sizeof(int32_t);

static constexpr std::string_view type_name() noexcept {
return "INT96";
}

static constexpr size_t element_size() noexcept {
return sizeof(Int96);
}

inline value_type Decode(tcb::span<const uint8_t> read_span) const {
if (read_span.size() != sizeof(Int96)) {
throw InvalidInputException("Decode: read_span size does not match Int96 element size");
}
const uint8_t* p = read_span.data();
return Int96{
read_le<int32_t>(p + 0 * kI32Size),
read_le<int32_t>(p + 1 * kI32Size),
read_le<int32_t>(p + 2 * kI32Size)};
}

inline void Encode(const value_type& value, tcb::span<uint8_t> write_span) const {
if (write_span.size() != sizeof(Int96)) {
throw InvalidInputException("Encode: write_span size does not match Int96 element size");
}
uint8_t* p = write_span.data();
write_le<int32_t>(value.lo, p + 0 * kI32Size);
write_le<int32_t>(value.mid, p + 1 * kI32Size);
write_le<int32_t>(value.hi, p + 2 * kI32Size);
}
};

Expand All @@ -79,11 +120,11 @@ struct RawBytesFixedSizedCodec {
return element_size_bytes_;
}

value_type Decode(tcb::span<const uint8_t> read_span) const noexcept {
inline value_type Decode(tcb::span<const uint8_t> read_span) const noexcept {
return read_span;
}

void Encode(const value_type& value, tcb::span<uint8_t> write_span) const {
inline void Encode(const value_type& value, tcb::span<uint8_t> write_span) const {
if (value.size() != write_span.size()) {
throw InvalidInputException("Encode: value size does not match write_span size");
}
Expand All @@ -106,11 +147,11 @@ struct RawBytesVariableSizedCodec {
throw InvalidInputException("RawBytesVariableSizedCodec does not have a fixed element size");
}

value_type Decode(tcb::span<const uint8_t> read_span) const noexcept {
inline value_type Decode(tcb::span<const uint8_t> read_span) const noexcept {
return read_span;
}

void Encode(const value_type& value, tcb::span<uint8_t> write_span) const {
inline void Encode(const value_type& value, tcb::span<uint8_t> write_span) const {
if (value.size() != write_span.size()) {
throw InvalidInputException("Encode: value size does not match write_span size");
}
Expand Down
8 changes: 4 additions & 4 deletions src/processing/typed_buffer_testing_codecs.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ struct StringFixedSizedCodec {
return element_size_bytes_;
}

value_type Decode(tcb::span<const uint8_t> read_span) const {
inline value_type Decode(tcb::span<const uint8_t> read_span) const {
if (read_span.size() != element_size_bytes_) {
throw InvalidInputException("Decode: read_span size does not match element_size_bytes");
}
Expand All @@ -59,7 +59,7 @@ struct StringFixedSizedCodec {
read_span.size());
}

void Encode(const value_type& value, tcb::span<uint8_t> write_span) const {
inline void Encode(const value_type& value, tcb::span<uint8_t> write_span) const {
if (write_span.size() != element_size_bytes_) {
throw InvalidInputException("Encode: write_span size does not match element_size_bytes");
}
Expand All @@ -85,13 +85,13 @@ struct StringVariableSizedCodec {
throw InvalidInputException("StringVariableSizedCodec does not have a fixed element size");
}

value_type Decode(tcb::span<const uint8_t> read_span) const noexcept {
inline value_type Decode(tcb::span<const uint8_t> read_span) const noexcept {
return std::string_view(
reinterpret_cast<const char*>(read_span.data()),
read_span.size());
}

void Encode(const value_type& value, tcb::span<uint8_t> write_span) const {
inline void Encode(const value_type& value, tcb::span<uint8_t> write_span) const {
// Exact match required to prevent short values leaving stale trailing bytes,
// and to prevent longer values from overflowing.
if (value.size() != write_span.size()) {
Expand Down
Loading
Loading