Feature: custom config for coding and jam-codec compatibility

CMakeLists.txt

@@ -13,8 +13,8 @@ endif()
 include(${CMAKE_CURRENT_LIST_DIR}/cmake/HunterGate.cmake)
 
 HunterGate(
-    URL  https://github.com/qdrvm/hunter/archive/refs/tags/v0.25.3-qdrvm21.zip
-    SHA1 5b52ab9a309771f172ca609a46e26dde60a8edd7
+    URL  https://github.com/qdrvm/hunter/archive/refs/tags/v0.25.3-qdrvm25.tar.gz
+    SHA1 bf5742041306c4b2c8b65b9c2d2af712a36ac3f9
 )
 
 project(Scale LANGUAGES CXX VERSION 1.1.0)
@@ -25,6 +25,9 @@ set(CMAKE_CXX_EXTENSIONS OFF)
 
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
+option(JAM_COMPATIBLE "Build compatible with JAM-codec" OFF)
+option(CUSTOM_CONFIG_SUPPORT "Support custom config of streams" OFF)
+
 option(BUILD_TESTS "Whether to include the test suite in build" OFF)
 
 hunter_add_package(Boost)
@@ -33,6 +36,13 @@ find_package(Boost CONFIG REQUIRED)
 hunter_add_package(qtils)
 find_package(qtils CONFIG REQUIRED)
 
+if (JAM_COMPATIBLE)
+    add_compile_definitions(JAM_COMPATIBILITY_ENABLED)
+endif ()
+if (CUSTOM_CONFIG_SUPPORT)
+    add_compile_definitions(CUSTOM_CONFIG_ENABLED)
+endif ()
+
 add_subdirectory(src)
 
 if (BUILD_TESTS)

include/scale/detail/compact_integer.hpp

@@ -0,0 +1,199 @@
+/**
+ * Copyright Quadrivium LLC
+ * All Rights Reserved
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#pragma once
+
+#include <array>
+#include <concepts>
+#include <cstddef>
+#include <cstdint>
+
+#include <boost/multiprecision/cpp_int.hpp>
+
+#include <scale/detail/fixed_width_integer.hpp>
+#include <scale/outcome/outcome_throw.hpp>
+#include <scale/scale_error.hpp>
+#include <scale/types.hpp>
+
+namespace scale::detail {
+
+  /// min integer encoded by 2 bytes
+  constexpr static size_t kMinUint16 = (1ul << 6u);
+  /// min integer encoded by 4 bytes
+  constexpr static size_t kMinUint32 = (1ul << 14u);
+  /// min integer encoded as multibyte
+  constexpr static size_t kMinBigInteger = (1ul << 30u);
+
+  /// Returns the compact encoded length for the given value.
+  size_t compactLen(std::unsigned_integral auto val) {
+    if (val < kMinUint16) return 1;
+    if (val < kMinUint32) return 2;
+    if (val < kMinBigInteger) return 4;
+    size_t counter = 1;
+    while ((val >>= 8)) ++counter;
+    return counter;
+  }
+
+  /**
+   * Encodes any integer type to compact-integer representation
+   * @tparam T integer type
+   * @tparam S output stream type
+   * @param value integer value
+   * @return byte array representation of value as compact-integer
+   */
+  template <typename T, typename S>
+    requires(std::integral<T> or std::is_same_v<T, CompactInteger>)
+  void encodeCompactInteger(T integer, S &s) {
+    boost::multiprecision::cpp_int value{integer};
+
+    // cannot encode negative numbers
+    // there is no description how to encode compact negative numbers
+    if (value < 0) {
+      raise(EncodeError::NEGATIVE_COMPACT_INTEGER);
+    }
+
+    if (value < kMinUint16) {
+      uint8_t v = (value.convert_to<uint8_t>() << 2u) | 0b00;
+      return encodeInteger(v, s);
+    }
+
+    else if (value < kMinUint32) {
+      // only values from [kMinUint16, kMinUint32) can be put here
+      uint16_t v = (value.convert_to<uint16_t>() << 2u) | 0b01;
+      return encodeInteger(v, s);
+    }
+
+    else if (value < kMinBigInteger) {
+      // only values from [kMinUint32, kMinBigInteger) can be put here
+      uint32_t v = (value.convert_to<uint32_t>() << 2u) | 0b10;
+      return encodeInteger(v, s);
+    }
+
+    // number of bytes required to represent value
+    size_t significant_bytes_n = msb(value) / 8 + 1;
+
+    if (significant_bytes_n > 67) {
+      raise(EncodeError::COMPACT_INTEGER_TOO_BIG);
+    }
+
+    // The upper 6 bits of the header are used to encode the number of bytes
+    // required to store the big integer. The value stored in these 6 bits
+    // ranges from 0 to 63 (2^6 - 1). However, the actual byte count starts
+    // from 4, so we subtract 4 from the byte count before encoding it.
+    // This makes the range of byte counts for storing big integers 4 to 67.
+    // To construct the final header, the upper 6 bits are shifted left by
+    // 2 positions (equivalent to multiplying by 4).
+    // The lower 2 bits (minor bits) store the encoding option, which in this
+    // case is 0b11 (decimal value 3). The final header is formed by adding 3
+    // to the result of the previous operations.
+    uint8_t header = ((significant_bytes_n - 4) << 2u) | 0b11;
+
+    s << header;
+
+    for (auto v = value; v != 0; v >>= 8) {
+      // push back the least significant byte
+      s << static_cast<uint8_t>(v & 0xff);
+    }
+  }
+
+  template <typename T, typename S>
+    requires std::is_same_v<T, CompactInteger>
+  T decodeCompactInteger(S &stream) {
+    auto first_byte = stream.nextByte();
+
+    const uint8_t flag = (first_byte)&0b00000011u;
+
+    size_t number = 0u;
+
+    switch (flag) {
+      case 0b00u: {
+        number = static_cast<size_t>(first_byte >> 2u);
+        break;
+      }
+
+      case 0b01u: {
+        auto second_byte = stream.nextByte();
+
+        number = (static_cast<size_t>((first_byte)&0b11111100u)
+                  + static_cast<size_t>(second_byte) * 256u)
+                 >> 2u;
+        if ((number >> 6) == 0) {
+          raise(DecodeError::REDUNDANT_COMPACT_ENCODING);
+        }
+        break;
+      }
+
+      case 0b10u: {
+        number = first_byte;
+        size_t multiplier = 256u;
+        if (!stream.hasMore(3u)) {
+          raise(DecodeError::NOT_ENOUGH_DATA);
+        }
+
+        for (auto i = 0u; i < 3u; ++i) {
+          // we assured that there are 3 more bytes,
+          // no need to make checks in a loop
+          number += (stream.nextByte()) * multiplier;
+          multiplier = multiplier << 8u;
+        }
+        number = number >> 2u;
+        if ((number >> 14) == 0) {
+          raise(DecodeError::REDUNDANT_COMPACT_ENCODING);
+        }
+        break;
+      }
+
+      case 0b11: {
+        auto bytes_count = ((first_byte) >> 2u) + 4u;
+        if (!stream.hasMore(bytes_count)) {
+          raise(DecodeError::NOT_ENOUGH_DATA);
+        }
+
+        CompactInteger multiplier{1u};
+        CompactInteger value = 0;
+        // we assured that there are m more bytes,
+        // no need to make checks in a loop
+        for (auto i = 0u; i < bytes_count; ++i) {
+          value += (stream.nextByte()) * multiplier;
+          multiplier *= 256u;
+        }
+        if (value.is_zero()) {
+          raise(DecodeError::REDUNDANT_COMPACT_ENCODING);
+        }
+        auto bits = msb(value) + 1;
+        if (bits <= 30 or (bits + 7) / 8 < bytes_count) {
+          raise(DecodeError::REDUNDANT_COMPACT_ENCODING);
+        }
+        return value;  // special case
+      }
+
+      default:
+        UNREACHABLE
+    }
+
+    return CompactInteger{number};
+  }
+
+  /**
+   * Decodes any integer type from compact-integer representation
+   * @tparam T integer type
+   * @tparam S input stream type
+   * @param value integer value
+   * @return value according compact-integer representation
+   */
+  template <typename T, typename S>
+    requires std::unsigned_integral<T>
+  T decodeCompactInteger(S &s) {
+    auto integer = decodeCompactInteger<CompactInteger>(s);
+    if (not integer.is_zero()
+        and msb(integer) >= std::numeric_limits<T>::digits) {
+      raise(DecodeError::DECODED_VALUE_OVERFLOWS_TARGET);
+    }
+    return static_cast<T>(integer);
+  }
+
+
+}  // namespace scale::detail

include/scale/detail/jam_compact_integer.hpp

@@ -0,0 +1,126 @@
+/**
+ * Copyright Quadrivium LLC
+ * All Rights Reserved
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#pragma once
+
+#include <array>
+#include <concepts>
+#include <cstddef>
+#include <cstdint>
+
+#include <scale/outcome/outcome_throw.hpp>
+#include <scale/scale_error.hpp>
+#include <scale/types.hpp>
+
+namespace scale::detail {
+
+  /**
+   * Encodes any integer type to jam-compact-integer representation
+   * @tparam T integer type
+   * @tparam S output stream type
+   * @param value integer value
+   * @return byte array representation of value as jam-compact-integer
+   */
+  template <typename T, typename S>
+    requires(std::unsigned_integral<T> or std::is_same_v<T, CompactInteger>)
+  void encodeJamCompactInteger(T integer, S &s) {
+    size_t value;
+
+    if constexpr (std::is_same_v<T, CompactInteger>) {
+      // cannot encode negative numbers
+      // there is no description how to encode compact negative numbers
+      if (integer < 0) {
+        raise(EncodeError::NEGATIVE_COMPACT_INTEGER);
+      }
+      if (integer.is_zero()) {
+        value = 0;
+      } else {
+        if (msb(integer) >= std::numeric_limits<size_t>::digits) {
+          raise(EncodeError::VALUE_TOO_BIG_FOR_COMPACT_REPRESENTATION);
+        }
+        value = integer.template convert_to<size_t>();
+      }
+    } else {
+      value = static_cast<size_t>(integer);
+    }
+
+    if (value < 0x80) {
+      s << static_cast<uint8_t>(value);
+      return;
+    }
+
+    std::array<uint8_t, sizeof(size_t) + 1> bytes;
+    uint8_t &prefix = bytes[0] = 0;
+    size_t len = 1;
+
+    for (decltype(value) i = value; i != 0; i >>= 8) {
+      // minimal value in prefix
+      if (i <= (static_cast<uint8_t>(~prefix) >> 1)) {
+        prefix |= i;
+        break;
+      }
+      prefix = (prefix >> 1) | 0x80;
+      bytes[len++] = static_cast<uint8_t>(i & 0xff);
+    }
+
+    for (auto byte : bytes) {
+      s << byte;
+      if (--len == 0) break;
+    }
+  }
+
+  /**
+   * Decodes any integer type from jam-compact-integer representation
+   * @tparam T integer type
+   * @tparam S input stream type
+   * @param value integer value
+   * @return value according jam-compact-integer representation
+   */
+  template <typename T>
+    requires std::unsigned_integral<T> or std::is_same_v<T, CompactInteger>
+  T decodeJamCompactInteger(auto &s) {
+    uint8_t byte;
+
+    s >> byte;
+
+    if (byte == 0) {
+      return 0;
+    }
+
+    uint8_t len_bits = byte;
+    uint8_t val_bits = byte;
+    uint8_t val_mask = 0xff;
+
+    size_t value = 0;
+
+    for (uint8_t i = 0; i < 8; ++i) {
+      val_mask >>= 1;
+      val_bits &= val_mask;
+
+      if ((len_bits & static_cast<uint8_t>(0x80)) == 0) {  // no more significant bytes
+        value |= static_cast<size_t>(val_bits) << (8 * i);
+        break;
+      }
+      len_bits <<= 1;
+
+      s >> byte;
+      value |= static_cast<size_t>(byte) << (8 * i);
+    }
+    if (val_bits == 0 and (byte & ~val_mask) == 0) {
+      raise(DecodeError::REDUNDANT_COMPACT_ENCODING);
+    }
+
+    if constexpr (not std::is_same_v<T, size_t>
+                  and not std::is_same_v<T, CompactInteger>) {
+      if (value > std::numeric_limits<T>::max()) {
+        raise(DecodeError::DECODED_VALUE_OVERFLOWS_TARGET);
+      }
+    }
+
+    return static_cast<T>(value);
+  }
+
+}  // namespace scale::detail