From f4624fd2c88128ed458dedd395f2cf62442a5c23 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Fri, 23 May 2025 08:36:43 +0300 Subject: [PATCH 01/14] feature: app state manager tests # Conflicts: # tests/unit/CMakeLists.txt --- src/app/impl/state_manager_impl.hpp | 2 +- tests/mock/app/state_manager_mock.hpp | 67 +++++++ tests/unit/CMakeLists.txt | 1 + tests/unit/app/CMakeLists.txt | 31 +++ tests/unit/app/state_manager_test.cpp | 266 ++++++++++++++++++++++++++ 5 files changed, 366 insertions(+), 1 deletion(-) create mode 100644 tests/mock/app/state_manager_mock.hpp create mode 100644 tests/unit/app/CMakeLists.txt create mode 100644 tests/unit/app/state_manager_test.cpp diff --git a/src/app/impl/state_manager_impl.hpp b/src/app/impl/state_manager_impl.hpp index 9f39388f..72425c19 100644 --- a/src/app/impl/state_manager_impl.hpp +++ b/src/app/impl/state_manager_impl.hpp @@ -25,7 +25,7 @@ namespace jam::log { namespace jam::app { - class StateManagerImpl final + class StateManagerImpl // left non-final on purpose to be accessible in tests : Singleton, public StateManager, public std::enable_shared_from_this { diff --git a/tests/mock/app/state_manager_mock.hpp b/tests/mock/app/state_manager_mock.hpp new file mode 100644 index 00000000..50613a2c --- /dev/null +++ b/tests/mock/app/state_manager_mock.hpp @@ -0,0 +1,67 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "app/state_manager.hpp" + +#include + +namespace jam::app { + + class StateManagerMock : public StateManager { + public: + MOCK_METHOD(void, atPrepare, (OnPrepare), ()); + void atPrepare(OnPrepare &&cb) override { + atPrepare(cb); + } + + MOCK_METHOD(void, atLaunch, (OnLaunch), ()); + void atLaunch(OnLaunch &&cb) override { + atLaunch(cb); + } + + MOCK_METHOD(void, atShutdown, (OnShutdown), ()); + void atShutdown(OnShutdown &&cb) override { + atShutdown(cb); + } + + MOCK_METHOD(void, run, (), (override)); + + MOCK_METHOD(void, shutdown, (), (override)); + + MOCK_METHOD(void, doPrepare, (), (override)); + + MOCK_METHOD(void, doLaunch, (), (override)); + + MOCK_METHOD(void, doShutdown, (), (override)); + + MOCK_METHOD(State, state, (), (const, override)); + }; + + /// `StartApp::start` calls all `prepare` and `start` callbacks + struct StartApp : StateManagerMock { + std::pair, std::vector> queue_; + + void atPrepare(OnPrepare &&cb) override { + queue_.first.emplace_back(cb); + } + + void atLaunch(OnLaunch &&cb) override { + queue_.second.emplace_back(cb); + } + + void start() { + auto queue = std::move(queue_); + for (auto &cb : queue.first) { + EXPECT_TRUE(cb()); + } + for (auto &cb : queue.second) { + EXPECT_TRUE(cb()); + } + } + }; +} // namespace kagome::application diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 93406f4b..64282f03 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -4,4 +4,5 @@ # SPDX-License-Identifier: Apache-2.0 # +add_subdirectory(app) add_subdirectory(storage) diff --git a/tests/unit/app/CMakeLists.txt b/tests/unit/app/CMakeLists.txt new file mode 100644 index 00000000..d42b5c27 --- /dev/null +++ b/tests/unit/app/CMakeLists.txt @@ -0,0 +1,31 @@ +# +# Copyright Quadrivium LLC +# All Rights Reserved +# SPDX-License-Identifier: Apache-2.0 +# + +#addtest(chain_spec_test +# chain_spec_test.cpp +# ) +#target_link_libraries(chain_spec_test +# chain_spec +# logger_for_tests +# ) + +addtest(app_state_manager_test + state_manager_test.cpp + ) +target_link_libraries(app_state_manager_test + app_state_manager + logger_for_tests + ) + +#addtest(app_config_test +# config_test.cpp +# ) +#target_link_libraries(app_config_test +# app_config +# filesystem +# logger_for_tests +# telemetry +# ) diff --git a/tests/unit/app/state_manager_test.cpp b/tests/unit/app/state_manager_test.cpp new file mode 100644 index 00000000..3f40e201 --- /dev/null +++ b/tests/unit/app/state_manager_test.cpp @@ -0,0 +1,266 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include + +#include +#include + +#include "app/impl/state_manager_impl.hpp" +#include "testutil/prepare_loggers.hpp" + +using jam::app::AppStateException; +using jam::app::StateManager; +using jam::app::StateManagerImpl; +using jam::log::LoggingSystem; +using OnPrepare = StateManager::OnPrepare; +using OnLaunch = StateManager::OnLaunch; +using OnShutdown = StateManager::OnShutdown; + +using testing::Return; +using testing::Sequence; + +class OnPrepareMock { + public: + MOCK_METHOD(bool, call, ()); + bool operator()() { + return call(); + } +}; +class OnLaunchMock { + public: + MOCK_METHOD(bool, call, ()); + bool operator()() { + return call(); + } +}; +class OnShutdownMock { + public: + MOCK_METHOD(void, call, ()); + void operator()() { + return call(); + } +}; + +class StateManagerTest : public testing::Test { + public: + class StateManagerHacked : public StateManagerImpl { + public: + using StateManagerImpl::doLaunch; + using StateManagerImpl::doPrepare; + using StateManagerImpl::doShutdown; + using StateManagerImpl::reset; + using StateManagerImpl::state; + using StateManagerImpl::StateManagerImpl; + }; + + void SetUp() override { + app_state_manager = + std::make_shared(testutil::prepareLoggers()); + app_state_manager->reset(); + prepare_cb = std::make_shared(); + launch_cb = std::make_shared(); + shutdown_cb = std::make_shared(); + } + void TearDown() override { + prepare_cb.reset(); + launch_cb.reset(); + shutdown_cb.reset(); + } + + std::shared_ptr app_state_manager; + std::shared_ptr prepare_cb; + std::shared_ptr launch_cb; + std::shared_ptr shutdown_cb; +}; + +/** + * @given new created StateManager + * @when switch stages in order + * @then state changes according to the order + */ +TEST_F(StateManagerTest, StateSequence_Normal) { + ASSERT_EQ(app_state_manager->state(), StateManager::State::Init); + ASSERT_NO_THROW(app_state_manager->doPrepare()); + ASSERT_EQ(app_state_manager->state(), StateManager::State::ReadyToStart); + ASSERT_NO_THROW(app_state_manager->doLaunch()); + ASSERT_EQ(app_state_manager->state(), StateManager::State::Works); + ASSERT_NO_THROW(app_state_manager->doShutdown()); + ASSERT_EQ(app_state_manager->state(), StateManager::State::ReadyToStop); +} + +/** + * @given StateManager in state 'ReadyToStart' (after stage 'prepare') + * @when trying to run stage 'prepare' again + * @then thrown exception, state wasn't change. Can to run stages 'launch' and + * 'shutdown' + */ +TEST_F(StateManagerTest, StateSequence_Abnormal_1) { + app_state_manager->doPrepare(); + EXPECT_THROW(app_state_manager->doPrepare(), AppStateException); + EXPECT_EQ(app_state_manager->state(), StateManager::State::ReadyToStart); + EXPECT_NO_THROW(app_state_manager->doLaunch()); + EXPECT_NO_THROW(app_state_manager->doShutdown()); +} + +/** + * @given StateManager in state 'Works' (after stage 'launch') + * @when trying to run stages 'prepare' and 'launch' again + * @then thrown exceptions, state wasn't change. Can to run stages 'launch' and + * 'shutdown' + */ +TEST_F(StateManagerTest, StateSequence_Abnormal_2) { + app_state_manager->doPrepare(); + app_state_manager->doLaunch(); + EXPECT_THROW(app_state_manager->doPrepare(), AppStateException); + EXPECT_THROW(app_state_manager->doLaunch(), AppStateException); + EXPECT_EQ(app_state_manager->state(), StateManager::State::Works); + EXPECT_NO_THROW(app_state_manager->doShutdown()); +} + +/** + * @given StateManager in state 'ReadyToStop' (after stage 'shutdown') + * @when trying to run any stages 'prepare' and 'launch' again + * @then thrown exceptions, except shutdown. State wasn't change. + */ +TEST_F(StateManagerTest, StateSequence_Abnormal_3) { + app_state_manager->doPrepare(); + app_state_manager->doLaunch(); + app_state_manager->doShutdown(); + EXPECT_THROW(app_state_manager->doPrepare(), AppStateException); + EXPECT_THROW(app_state_manager->doLaunch(), AppStateException); + EXPECT_THROW(app_state_manager->doShutdown(), AppStateException); + EXPECT_EQ(app_state_manager->state(), StateManager::State::ReadyToStop); +} + +/** + * @given new created StateManager + * @when add callbacks for each stages + * @then done without exceptions + */ +TEST_F(StateManagerTest, AddCallback_Initial) { + EXPECT_NO_THROW(app_state_manager->atPrepare([] {})); + EXPECT_NO_THROW(app_state_manager->atLaunch([] {})); + EXPECT_NO_THROW(app_state_manager->atShutdown([] {})); +} + +/** + * @given StateManager in state 'ReadyToStart' (after stage 'prepare') + * @when add callbacks for each stages + * @then thrown exception only for 'prepare' stage callback + */ +TEST_F(StateManagerTest, AddCallback_AfterPrepare) { + app_state_manager->doPrepare(); + EXPECT_THROW(app_state_manager->atPrepare([] {}), AppStateException); + EXPECT_NO_THROW(app_state_manager->atLaunch([] {})); + EXPECT_NO_THROW(app_state_manager->atShutdown([] {})); +} + +/** + * @given StateManager in state 'Works' (after stage 'launch') + * @when add callbacks for each stages + * @then done without exception only for 'shutdown' stage callback + */ +TEST_F(StateManagerTest, AddCallback_AfterLaunch) { + app_state_manager->doPrepare(); + app_state_manager->doLaunch(); + EXPECT_THROW(app_state_manager->atPrepare([] {}), AppStateException); + EXPECT_THROW(app_state_manager->atLaunch([] {}), AppStateException); + EXPECT_NO_THROW(app_state_manager->atShutdown([] {})); +} + +/** + * @given StateManager in state 'ReadyToStop' (after stage 'shutdown') + * @when add callbacks for each stages + * @then trown exceptions for each calls + */ +TEST_F(StateManagerTest, AddCallback_AfterShutdown) { + app_state_manager->doPrepare(); + app_state_manager->doLaunch(); + app_state_manager->doShutdown(); + EXPECT_THROW(app_state_manager->atPrepare([] {}), AppStateException); + EXPECT_THROW(app_state_manager->atLaunch([] {}), AppStateException); + EXPECT_THROW(app_state_manager->atShutdown([] {}), AppStateException); +} + +struct UnderControlObject { + UnderControlObject(OnPrepareMock &p, OnLaunchMock &l, OnShutdownMock &s) + : p(p), l(l), s(s) {} + + OnPrepareMock &p; + OnLaunchMock &l; + OnShutdownMock &s; + int tag = 0; + + bool prepare() { + tag = 1; + return p(); + } + + bool start() { + tag = 2; + return l(); + } + + void stop() { + tag = 3; + s(); + } +}; + +/** + * @given new created StateManager + * @when register callbacks by reg() method + * @then each callcack registered for appropriate state + */ +TEST_F(StateManagerTest, RegCallbacks) { + UnderControlObject x(*prepare_cb, *launch_cb, *shutdown_cb); + + app_state_manager->takeControl(x); + + EXPECT_CALL(*prepare_cb, call()).WillOnce(Return(true)); + EXPECT_CALL(*launch_cb, call()).WillOnce(Return(true)); + EXPECT_CALL(*shutdown_cb, call()).WillOnce(Return()); + + EXPECT_NO_THROW(app_state_manager->doPrepare()); + EXPECT_EQ(x.tag, 1); + EXPECT_NO_THROW(app_state_manager->doLaunch()); + EXPECT_EQ(x.tag, 2); + EXPECT_NO_THROW(app_state_manager->doShutdown()); + EXPECT_EQ(x.tag, 3); +} + +/** + * @given new created StateManager + * @when register callbacks by reg() method and run() StateManager + * @then each callcack executed according to the stages order + */ +TEST_F(StateManagerTest, Run_CallSequence) { + app_state_manager.reset(); // Enforce destruction of previous instance + app_state_manager = + std::make_shared(testutil::prepareLoggers()); + + UnderControlObject x(*prepare_cb, *launch_cb, *shutdown_cb); + app_state_manager->takeControl(x); + + Sequence seq; + EXPECT_CALL(*prepare_cb, call()).InSequence(seq).WillOnce(Return(true)); + EXPECT_CALL(*launch_cb, call()).InSequence(seq).WillOnce(Return(true)); + EXPECT_CALL(*shutdown_cb, call()).InSequence(seq).WillOnce(Return()); + + app_state_manager->atLaunch([] { + std::thread terminator([] { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + std::raise(SIGQUIT); + }); + terminator.join(); + return true; + }); + + std::thread main([&] { EXPECT_NO_THROW(app_state_manager->run()); }); + main.join(); +} From 19547cc6da9a2c69221674ff69f3baff6b8992f1 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Wed, 14 May 2025 11:48:07 +0300 Subject: [PATCH 02/14] feature: crypto Signed-off-by: Dmitriy Khaustov aka xDimon --- CMakeLists.txt | 2 + src/CMakeLists.txt | 10 +- src/crypto/CMakeLists.txt | 19 ++ src/crypto/blake2/CMakeLists.txt | 13 ++ src/crypto/blake2/blake2b.cpp | 208 +++++++++++++++++ src/crypto/blake2/blake2b.h | 65 ++++++ src/crypto/blake2/blake2s.cpp | 220 ++++++++++++++++++ src/crypto/blake2/blake2s.h | 86 +++++++ src/crypto/hash_types.hpp | 16 ++ src/crypto/hasher.hpp | 89 +++++++ src/crypto/hasher/hasher_impl.cpp | 68 ++++++ src/crypto/hasher/hasher_impl.hpp | 39 ++++ src/crypto/keccak/keccak.hpp | 24 ++ src/crypto/sha/CMakeLists.txt | 15 ++ src/crypto/sha/sha256.cpp | 26 +++ src/crypto/sha/sha256.hpp | 29 +++ src/crypto/twox/CMakeLists.txt | 13 ++ src/crypto/twox/twox.cpp | 64 ++++++ src/crypto/twox/twox.hpp | 21 ++ src/injector/CMakeLists.txt | 1 + src/injector/node_injector.cpp | 2 + src/jam_types/types.tmp.hpp | 7 +- src/scale/tie_hash.hpp | 44 ++++ src/third_party/CMakeLists.txt | 1 + src/third_party/keccak/CMakeLists.txt | 14 ++ src/third_party/keccak/keccak.c | 320 ++++++++++++++++++++++++++ src/third_party/keccak/keccak.h | 61 +++++ src/utils/tuple_hash.hpp | 35 +++ vcpkg.json | 4 +- 29 files changed, 1510 insertions(+), 6 deletions(-) create mode 100644 src/crypto/CMakeLists.txt create mode 100644 src/crypto/blake2/CMakeLists.txt create mode 100644 src/crypto/blake2/blake2b.cpp create mode 100644 src/crypto/blake2/blake2b.h create mode 100644 src/crypto/blake2/blake2s.cpp create mode 100644 src/crypto/blake2/blake2s.h create mode 100644 src/crypto/hash_types.hpp create mode 100644 src/crypto/hasher.hpp create mode 100644 src/crypto/hasher/hasher_impl.cpp create mode 100644 src/crypto/hasher/hasher_impl.hpp create mode 100644 src/crypto/keccak/keccak.hpp create mode 100644 src/crypto/sha/CMakeLists.txt create mode 100644 src/crypto/sha/sha256.cpp create mode 100644 src/crypto/sha/sha256.hpp create mode 100644 src/crypto/twox/CMakeLists.txt create mode 100644 src/crypto/twox/twox.cpp create mode 100644 src/crypto/twox/twox.hpp create mode 100644 src/scale/tie_hash.hpp create mode 100644 src/third_party/CMakeLists.txt create mode 100644 src/third_party/keccak/CMakeLists.txt create mode 100644 src/third_party/keccak/keccak.c create mode 100644 src/third_party/keccak/keccak.h create mode 100644 src/utils/tuple_hash.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 0f5198bd..b5c117b3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -75,6 +75,8 @@ find_package(soralog CONFIG REQUIRED) find_package(Boost.DI CONFIG REQUIRED) find_package(qtils CONFIG REQUIRED) find_package(prometheus-cpp CONFIG REQUIRED) +find_package(OpenSSL CONFIG REQUIRED) +find_package(xxHash CONFIG REQUIRED) find_package(RocksDB CONFIG REQUIRED) # TODO Temporarily commented out until gcc is updated (gcc-13 crashes because of this). diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 584dd6f2..f670cf58 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -4,16 +4,15 @@ # SPDX-License-Identifier: Apache-2.0 # -include_directories(${CMAKE_SOURCE_DIR}) -include_directories(${CMAKE_SOURCE_DIR}/src) -include_directories(${CMAKE_BINARY_DIR}/generated) - # Executables (should contain `main()` function) add_subdirectory(executable) # Application's thinks add_subdirectory(app) +# Cryptography thinks +add_subdirectory(crypto) + # Dependency injection add_subdirectory(injector) @@ -37,3 +36,6 @@ add_subdirectory(storage) # Utilities add_subdirectory(utils) + +# 3rd-party things +add_subdirectory(third_party) diff --git a/src/crypto/CMakeLists.txt b/src/crypto/CMakeLists.txt new file mode 100644 index 00000000..cdebbf38 --- /dev/null +++ b/src/crypto/CMakeLists.txt @@ -0,0 +1,19 @@ +# +# Copyright Quadrivium LLC +# All Rights Reserved +# SPDX-License-Identifier: Apache-2.0 +# + +add_subdirectory(blake2) +add_subdirectory(sha) +add_subdirectory(twox) + +add_library(hasher + hasher/hasher_impl.cpp +) +target_link_libraries(hasher + blake2 + twox + sha + keccak +) diff --git a/src/crypto/blake2/CMakeLists.txt b/src/crypto/blake2/CMakeLists.txt new file mode 100644 index 00000000..2633ffe0 --- /dev/null +++ b/src/crypto/blake2/CMakeLists.txt @@ -0,0 +1,13 @@ +# +# Copyright Quadrivium LLC +# All Rights Reserved +# SPDX-License-Identifier: Apache-2.0 +# + +add_library(blake2 + blake2s.cpp + blake2b.cpp +) +target_link_libraries(blake2 + qtils::qtils +) diff --git a/src/crypto/blake2/blake2b.cpp b/src/crypto/blake2/blake2b.cpp new file mode 100644 index 00000000..db19712a --- /dev/null +++ b/src/crypto/blake2/blake2b.cpp @@ -0,0 +1,208 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +// taken from here +// https://github.com/mjosaarinen/blake2_mjosref/blob/master/blake2b.c + +// blake2b.c +// A simple BLAKE2b Reference Implementation. + +#include "blake2b.h" + +namespace jam::crypto { + + // Cyclic right rotation. + +#ifndef ROTR64 +#define ROTR64(x, y) (((x) >> (y)) ^ ((x) << (64 - (y)))) +#endif + + // Little-endian byte access. + +#define B2B_GET64(p) \ + (((uint64_t)((uint8_t *)(p))[0]) ^ (((uint64_t)((uint8_t *)(p))[1]) << 8) \ + ^ (((uint64_t)((uint8_t *)(p))[2]) << 16) \ + ^ (((uint64_t)((uint8_t *)(p))[3]) << 24) \ + ^ (((uint64_t)((uint8_t *)(p))[4]) << 32) \ + ^ (((uint64_t)((uint8_t *)(p))[5]) << 40) \ + ^ (((uint64_t)((uint8_t *)(p))[6]) << 48) \ + ^ (((uint64_t)((uint8_t *)(p))[7]) << 56)) + + // G Mixing function. + +#define B2B_G(a, b, c, d, x, y) \ + { \ + v[a] = v[a] + v[b] + (x); \ + v[d] = ROTR64(v[d] ^ v[a], 32); \ + v[c] = v[c] + v[d]; \ + v[b] = ROTR64(v[b] ^ v[c], 24); \ + v[a] = v[a] + v[b] + (y); \ + v[d] = ROTR64(v[d] ^ v[a], 16); \ + v[c] = v[c] + v[d]; \ + v[b] = ROTR64(v[b] ^ v[c], 63); \ + } + + // Initialization Vector. + + static const uint64_t blake2b_iv[8] = {0x6A09E667F3BCC908, + 0xBB67AE8584CAA73B, + 0x3C6EF372FE94F82B, + 0xA54FF53A5F1D36F1, + 0x510E527FADE682D1, + 0x9B05688C2B3E6C1F, + 0x1F83D9ABFB41BD6B, + 0x5BE0CD19137E2179}; + + // Compression function. "last" flag indicates last block. + + static void blake2b_compress(blake2b_ctx *ctx, int last) { + const uint8_t sigma[12][16] = { + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3}, + {11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4}, + {7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8}, + {9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13}, + {2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9}, + {12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11}, + {13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10}, + {6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5}, + {10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3}}; + int i; + uint64_t v[16]; + uint64_t m[16]; + + for (i = 0; i < 8; i++) { // init work variables + v[i] = ctx->h[i]; + v[i + 8] = blake2b_iv[i]; + } + + v[12] ^= ctx->t[0]; // low 64 bits of offset + v[13] ^= ctx->t[1]; // high 64 bits + if (last) { // last block flag set ? + v[14] = ~v[14]; + } + + for (i = 0; i < 16; i++) { // get little-endian words + m[i] = B2B_GET64(&ctx->b[8 * i]); + } + + for (i = 0; i < 12; i++) { // twelve rounds + B2B_G(0, 4, 8, 12, m[sigma[i][0]], m[sigma[i][1]]); + B2B_G(1, 5, 9, 13, m[sigma[i][2]], m[sigma[i][3]]); + B2B_G(2, 6, 10, 14, m[sigma[i][4]], m[sigma[i][5]]); + B2B_G(3, 7, 11, 15, m[sigma[i][6]], m[sigma[i][7]]); + B2B_G(0, 5, 10, 15, m[sigma[i][8]], m[sigma[i][9]]); + B2B_G(1, 6, 11, 12, m[sigma[i][10]], m[sigma[i][11]]); + B2B_G(2, 7, 8, 13, m[sigma[i][12]], m[sigma[i][13]]); + B2B_G(3, 4, 9, 14, m[sigma[i][14]], m[sigma[i][15]]); + } + + for (i = 0; i < 8; ++i) { + ctx->h[i] ^= v[i] ^ v[i + 8]; + } + } + + // Initialize the hashing context "ctx" with optional key "key". + // 1 <= outlen <= 64 gives the digest size in bytes. + // Secret key (also <= 64 bytes) is optional (keylen = 0). + + int blake2b_init(blake2b_ctx *ctx, + size_t outlen, + const void *key, + size_t keylen) // (keylen=0: no key) + { + size_t i; + + if (outlen == 0 || outlen > 64 || keylen > 64) { + return -1; // illegal parameters + } + + for (i = 0; i < 8; i++) { // state, "param block" + ctx->h[i] = blake2b_iv[i]; + } + ctx->h[0] ^= 0x01010000 ^ (keylen << 8) ^ outlen; + + ctx->t[0] = 0; // input count low word + ctx->t[1] = 0; // input count high word + ctx->c = 0; // pointer within buffer + ctx->outlen = outlen; + + for (i = keylen; i < 128; i++) { // zero input block + ctx->b[i] = 0; + } + if (keylen > 0) { + blake2b_update(ctx, key, keylen); + ctx->c = 128; // at the end + } + + return 0; + } + + // Add "inlen" bytes from "in" into the hash. + + void blake2b_update(blake2b_ctx *ctx, + const void *in, + size_t inlen) // data bytes + { + size_t i; + + for (i = 0; i < inlen; i++) { + if (ctx->c == 128) { // buffer full ? + ctx->t[0] += ctx->c; // add counters + if (ctx->t[0] < ctx->c) { // carry overflow ? + ctx->t[1]++; // high word + } + blake2b_compress(ctx, 0); // compress (not last) + ctx->c = 0; // counter to zero + } + ctx->b[ctx->c++] = ((const uint8_t *)in)[i]; + } + } + + // Generate the message digest (size given in init). + // Result placed in "out". + + void blake2b_final(blake2b_ctx *ctx, void *out) { + size_t i; + + ctx->t[0] += ctx->c; // mark last block offset + if (ctx->t[0] < ctx->c) { // carry overflow + ctx->t[1]++; // high word + } + + while (ctx->c < 128) { // fill up with zeros + ctx->b[ctx->c++] = 0; + } + blake2b_compress(ctx, 1); // final block flag = 1 + + // little endian convert and store + for (i = 0; i < ctx->outlen; i++) { + ((uint8_t *)out)[i] = (ctx->h[i >> 3] >> (8 * (i & 7))) & 0xFF; + } + } + + // Convenience function for all-in-one computation. + + int blake2b(void *out, + size_t outlen, + const void *key, + size_t keylen, + const void *in, + size_t inlen) { + blake2b_ctx ctx; + + if (blake2b_init(&ctx, outlen, key, keylen)) { + return -1; + } + blake2b_update(&ctx, in, inlen); + blake2b_final(&ctx, out); + + return 0; + } + +} // namespace jam::crypto diff --git a/src/crypto/blake2/blake2b.h b/src/crypto/blake2/blake2b.h new file mode 100644 index 00000000..4f948f5f --- /dev/null +++ b/src/crypto/blake2/blake2b.h @@ -0,0 +1,65 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +// taken from here +// https://github.com/mjosaarinen/blake2_mjosref/blob/master/blake2b.h + +#ifndef CORE_BLAKE2B_H +#define CORE_BLAKE2B_H + +#include +#include + +#include +#include + +namespace jam::crypto { + + // state context + typedef struct { + uint8_t b[128]; // input buffer + uint64_t h[8]; // chained state + uint64_t t[2]; // total number of bytes + size_t c; // pointer for b[] + size_t outlen; // digest size + } blake2b_ctx; + + // Initialize the hashing context "ctx" with optional key "key". + // 1 <= outlen <= 64 gives the digest size in bytes. + // Secret key (also <= 64 bytes) is optional (keylen = 0). + int blake2b_init(blake2b_ctx *ctx, + size_t outlen, + const void *key, + size_t keylen); // secret key + + // Add "inlen" bytes from "in" into the hash. + void blake2b_update(blake2b_ctx *ctx, // context + const void *in, + size_t inlen); // data to be hashed + + // Generate the message digest (size given in init). + // Result placed in "out". + void blake2b_final(blake2b_ctx *ctx, void *out); + + // All-in-one convenience function. + int blake2b(void *out, + size_t outlen, // return buffer for digest + const void *key, + size_t keylen, // optional secret key + const void *in, + size_t inlen); // data to be hashed + + template + inline qtils::ByteArr blake2b(qtils::ByteView buf) { + qtils::ByteArr out; + BOOST_VERIFY(blake2b(out.data(), N, nullptr, 0, buf.data(), buf.size()) + == 0); + return out; + } + +} // namespace jam::crypto + +#endif diff --git a/src/crypto/blake2/blake2s.cpp b/src/crypto/blake2/blake2s.cpp new file mode 100644 index 00000000..d3f666fe --- /dev/null +++ b/src/crypto/blake2/blake2s.cpp @@ -0,0 +1,220 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +// taken from here +// https://github.com/mjosaarinen/blake2_mjosref/blob/master/blake2s.c + +// blake2s.c +// A simple blake2s Reference Implementation. + +#include "blake2s.h" + +#include + +namespace jam::crypto { + +#define _256_bits 32 + + // Cyclic right rotation. + +#ifndef ROTR32 +#define ROTR32(x, y) (((x) >> (y)) ^ ((x) << (32 - (y)))) +#endif + + // Little-endian byte access. + + // state context + typedef struct { + uint8_t b[64]; // input buffer + uint32_t h[8]; // chained state + uint32_t t[2]; // total number of bytes + size_t c; // pointer for b[] + size_t outlen; // digest size + } blake2s_ctx_full; + +#define B2S_GET32(p) \ + (((uint32_t)((uint8_t *)(p))[0]) ^ (((uint32_t)((uint8_t *)(p))[1]) << 8) \ + ^ (((uint32_t)((uint8_t *)(p))[2]) << 16) \ + ^ (((uint32_t)((uint8_t *)(p))[3]) << 24)) + + // Mixing function G. + +#define B2S_G(a, b, c, d, x, y) \ + { \ + v[a] = v[a] + v[b] + (x); \ + v[d] = ROTR32(v[d] ^ v[a], 16); \ + v[c] = v[c] + v[d]; \ + v[b] = ROTR32(v[b] ^ v[c], 12); \ + v[a] = v[a] + v[b] + (y); \ + v[d] = ROTR32(v[d] ^ v[a], 8); \ + v[c] = v[c] + v[d]; \ + v[b] = ROTR32(v[b] ^ v[c], 7); \ + } + + // Initialization Vector. + + static const uint32_t blake2s_iv[8] = {0x6A09E667, + 0xBB67AE85, + 0x3C6EF372, + 0xA54FF53A, + 0x510E527F, + 0x9B05688C, + 0x1F83D9AB, + 0x5BE0CD19}; + + // Compression function. "last" flag indicates last block. + + static void blake2s_compress(blake2s_ctx_full *ctx, int last) { + const uint8_t sigma[10][16] = { + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3}, + {11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4}, + {7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8}, + {9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13}, + {2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9}, + {12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11}, + {13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10}, + {6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5}, + {10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0}}; + int i; + uint32_t v[16]; + uint32_t m[16]; + + for (i = 0; i < 8; i++) { // init work variables + v[i] = ctx->h[i]; + v[i + 8] = blake2s_iv[i]; + } + + v[12] ^= ctx->t[0]; // low 32 bits of offset + v[13] ^= ctx->t[1]; // high 32 bits + if (last) { // last block flag set ? + v[14] = ~v[14]; + } + + for (i = 0; i < 16; i++) { // get little-endian words + m[i] = B2S_GET32(&ctx->b[4 * i]); + } + + for (i = 0; i < 10; i++) { // ten rounds + B2S_G(0, 4, 8, 12, m[sigma[i][0]], m[sigma[i][1]]); + B2S_G(1, 5, 9, 13, m[sigma[i][2]], m[sigma[i][3]]); + B2S_G(2, 6, 10, 14, m[sigma[i][4]], m[sigma[i][5]]); + B2S_G(3, 7, 11, 15, m[sigma[i][6]], m[sigma[i][7]]); + B2S_G(0, 5, 10, 15, m[sigma[i][8]], m[sigma[i][9]]); + B2S_G(1, 6, 11, 12, m[sigma[i][10]], m[sigma[i][11]]); + B2S_G(2, 7, 8, 13, m[sigma[i][12]], m[sigma[i][13]]); + B2S_G(3, 4, 9, 14, m[sigma[i][14]], m[sigma[i][15]]); + } + + for (i = 0; i < 8; ++i) { + ctx->h[i] ^= v[i] ^ v[i + 8]; + } + } + + // Add "inlen" bytes from "in" into the hash. + void blake2s_update(blake2s_ctx *ctx_opaque, const void *in, size_t inlen) { + blake2s_ctx_full *ctx = (blake2s_ctx_full *)ctx_opaque->opaque; + + size_t i; + + for (i = 0; i < inlen; i++) { + if (ctx->c == 64) { // buffer full ? + ctx->t[0] += ctx->c; // add counters + if (ctx->t[0] < ctx->c) { // carry overflow ? + ctx->t[1]++; // high word + } + blake2s_compress(ctx, 0); // compress (not last) + ctx->c = 0; // counter to zero + } + ctx->b[ctx->c++] = ((const uint8_t *)in)[i]; + } + } + + // Generate the message digest (size given in init). + // Result placed in "out". + + void blake2s_final(blake2s_ctx *ctx_opaque, void *out) { + blake2s_ctx_full *ctx = (blake2s_ctx_full *)ctx_opaque->opaque; + + size_t i; + + ctx->t[0] += ctx->c; // mark last block offset + if (ctx->t[0] < ctx->c) { // carry overflow + ctx->t[1]++; // high word + } + + while (ctx->c < 64) { // fill up with zeros + ctx->b[ctx->c++] = 0; + } + blake2s_compress(ctx, 1); // final block flag = 1 + + // little endian convert and store + for (i = 0; i < ctx->outlen; i++) { + ((uint8_t *)out)[i] = (uint8_t)((ctx->h[i >> 2] >> (8 * (i & 3))) & 0xFF); + } + } + + int blake2s_init(blake2s_ctx *ctx_opaque, + size_t outlen, + const void *key, + size_t keylen) { + blake2s_ctx_full *ctx = (blake2s_ctx_full *)ctx_opaque->opaque; + + size_t i; + + if (outlen == 0 || outlen > 32 || keylen > 32) { + return -1; // illegal parameters + } + + for (i = 0; i < 8; i++) { // state, "param block" + ctx->h[i] = blake2s_iv[i]; + } + + ctx->h[0] ^= 0x01010000 ^ (keylen << 8) ^ outlen; + + ctx->t[0] = 0; // input count low word + ctx->t[1] = 0; // input count high word + ctx->c = 0; // pointer within buffer + ctx->outlen = outlen; + + for (i = keylen; i < 64; i++) { // zero input block + ctx->b[i] = 0; + } + + if (keylen > 0) { + blake2s_update(ctx_opaque, key, keylen); + ctx->c = 64; // at the end + } + + return 0; + } + + int blake2s(void *out, + size_t outlen, + const void *key, + size_t keylen, + const void *in, + size_t inlen) { + blake2s_ctx ctx; + + if (blake2s_init(&ctx, outlen, key, keylen)) { + return -1; + } + + blake2s_update(&ctx, in, inlen); + blake2s_final(&ctx, out); + + return 0; + } + + void blake2s_256_init(blake2s_ctx *ctx_opaque) { + blake2s_init(ctx_opaque, _256_bits, NULL, 0); + } + + void blake2s_256(void *out, const void *in, size_t inlen) { + blake2s(out, _256_bits, NULL, 0, in, inlen); + } +} // namespace jam::crypto diff --git a/src/crypto/blake2/blake2s.h b/src/crypto/blake2/blake2s.h new file mode 100644 index 00000000..c99d8891 --- /dev/null +++ b/src/crypto/blake2/blake2s.h @@ -0,0 +1,86 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +// taken from here +// https://github.com/mjosaarinen/blake2_mjosref/blob/master/blake2s.h + +#ifndef CORE_BLAKE2S_HASH +#define CORE_BLAKE2S_HASH + +#include + +namespace jam::crypto { + + typedef struct { + unsigned char opaque[128]; + } blake2s_ctx; + + /** + * @brief Initialize hash context + * @param ctx context + */ + void blake2s_256_init(blake2s_ctx *ctx); + + /** + * @brief Update context with incoming bytes + * @param ctx context + * @param in {@param inlen} byte array + * @param inlen size of {@param in} + */ + void blake2s_update(blake2s_ctx *ctx, const void *in, size_t inlen); + + /** + * @brief Finalize hash calculation + * @param ctx context + * @param out 32-byte output + * @return + */ + void blake2s_final(blake2s_ctx *ctx, void *out); + + /** + * @brief One-shot convenience function to calculate blake2s_256 hash + * @param out 32-byte buffer + * @param in {@param inlen} bytes input buffer + * @param inlen size of the input buffer + * @return + */ + void blake2s_256(void *out, const void *in, size_t inlen); + + /** + * @brief Generic blake2s init function + * @param ctx context + * @param outlen 1..32 bytes of output buffer size + * @param key optional key + * @param keylen length of {@param key} in bytes. Pass 0 to indicate that key + * is not provided. + * @return -1 in case of wrong input arguments; 0 in case of success. + */ + int blake2s_init(blake2s_ctx *ctx, + size_t outlen, + const void *key, + size_t keylen); + + /** + * @brief All in one blake2s hashing function. + * @param out output buffer + * @param outlen size of {@param out} + * @param key optional key + * @param keylen size of {@param key}. Pass 0 to indicate that key is not + * provided. + * @param in data to be hashed + * @param inlen size of {@param in} + * @return -1 in case of wrong input arguments; 0 in case of success + */ + int blake2s(void *out, + size_t outlen, + const void *key, + size_t keylen, + const void *in, + size_t inlen); + +} // namespace jam::crypto + +#endif // CORE_BLAKE2S_HASH diff --git a/src/crypto/hash_types.hpp b/src/crypto/hash_types.hpp new file mode 100644 index 00000000..685f8a52 --- /dev/null +++ b/src/crypto/hash_types.hpp @@ -0,0 +1,16 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +namespace jam { + using Hash64 = qtils::ByteArr<8>; + using Hash128 = qtils::ByteArr<16>; + using Hash256 = qtils::ByteArr<32>; + using Hash512 = qtils::ByteArr<64>; +} // namespace jam diff --git a/src/crypto/hasher.hpp b/src/crypto/hasher.hpp new file mode 100644 index 00000000..9facb21f --- /dev/null +++ b/src/crypto/hasher.hpp @@ -0,0 +1,89 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include "crypto/hash_types.hpp" + +namespace jam::crypto { + + class Hasher { + public: + virtual ~Hasher() = default; + + /** + * @brief twox_64 calculates 8-byte twox hash + * @param data source data + * @return 64-bit hash value + */ + virtual Hash64 twox_64(qtils::ByteView data) const = 0; + + /** + * @brief blake2b_64 function calculates 8-byte blake2b hash + * @param data source value + * @return 64-bit hash value + */ + virtual Hash64 blake2b_64(qtils::ByteView data) const = 0; + + /** + * @brief twox_128 calculates 16-byte twox hash + * @param data source data + * @return 128-bit hash value + */ + virtual Hash128 twox_128(qtils::ByteView data) const = 0; + + /** + * @brief blake2b_128 function calculates 16-byte blake2b hash + * @param data source value + * @return 128-bit hash value + */ + virtual Hash128 blake2b_128(qtils::ByteView data) const = 0; + + /** + * @brief twox_256 calculates 32-byte twox hash + * @param data source data + * @return 256-bit hash value + */ + virtual Hash256 twox_256(qtils::ByteView data) const = 0; + + /** + * @brief blake2b_256 function calculates 32-byte blake2b hash + * @param data source value + * @return 256-bit hash value + */ + virtual Hash256 blake2b_256(qtils::ByteView data) const = 0; + + /** + * @brief blake2b_512 function calculates 64-byte blake2b hash + * @param data source value + * @return 512-bit hash value + */ + virtual Hash512 blake2b_512(qtils::ByteView data) const = 0; + + /** + * @brief keccak_256 function calculates 32-byte keccak hash + * @param data source value + * @return 256-bit hash value + */ + virtual Hash256 keccak_256(qtils::ByteView data) const = 0; + + /** + * @brief blake2s_256 function calculates 32-byte blake2s hash + * @param data source value + * @return 256-bit hash value + */ + virtual Hash256 blake2s_256(qtils::ByteView data) const = 0; + + /** + * @brief sha2_256 function calculates 32-byte sha2-256 hash + * @param data source value + * @return 256-bit hash value + */ + virtual Hash256 sha2_256(qtils::ByteView data) const = 0; + }; +} // namespace jam::crypto diff --git a/src/crypto/hasher/hasher_impl.cpp b/src/crypto/hasher/hasher_impl.cpp new file mode 100644 index 00000000..9e4ed2e2 --- /dev/null +++ b/src/crypto/hasher/hasher_impl.cpp @@ -0,0 +1,68 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "crypto/hasher/hasher_impl.hpp" + +#include + +#include "crypto/blake2/blake2b.h" +#include "crypto/blake2/blake2s.h" +#include "crypto/keccak/keccak.hpp" +#include "crypto/sha/sha256.hpp" +#include "crypto/twox/twox.hpp" + +namespace jam::crypto { + + Hash64 HasherImpl::twox_64(qtils::ByteView data) const { + return make_twox64(data); + } + + Hash64 HasherImpl::blake2b_64(qtils::ByteView data) const { + return blake2b<8>(data); + } + + Hash128 HasherImpl::twox_128(qtils::ByteView data) const { + return make_twox128(data); + } + + Hash128 HasherImpl::blake2b_128(qtils::ByteView data) const { + return blake2b<16>(data); + } + + Hash256 HasherImpl::twox_256(qtils::ByteView data) const { + return make_twox256(data); + } + + Hash256 HasherImpl::blake2b_256(qtils::ByteView data) const { + return blake2b<32>(data); + } + + Hash512 HasherImpl::blake2b_512(qtils::ByteView data) const { + return blake2b<64>(data); + } + + Hash256 HasherImpl::keccak_256(qtils::ByteView data) const { + Hash256 out; + sha3_HashBuffer(256, + SHA3_FLAGS::SHA3_FLAGS_KECCAK, + data.data(), + data.size(), + out.data(), + 32); + return out; + } + + Hash256 HasherImpl::blake2s_256(qtils::ByteView data) const { + Hash256 out; + blake2s(out.data(), 32, nullptr, 0, data.data(), data.size()); + return out; + } + + Hash256 HasherImpl::sha2_256(qtils::ByteView data) const { + return sha256(data); + } + +} // namespace jam::crypto diff --git a/src/crypto/hasher/hasher_impl.hpp b/src/crypto/hasher/hasher_impl.hpp new file mode 100644 index 00000000..977ebf99 --- /dev/null +++ b/src/crypto/hasher/hasher_impl.hpp @@ -0,0 +1,39 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "crypto/hash_types.hpp" +#include "crypto/hasher.hpp" + +namespace jam::crypto { + + class HasherImpl : public Hasher { + public: + ~HasherImpl() override = default; + + Hash64 twox_64(qtils::ByteView data) const override; + + Hash64 blake2b_64(qtils::ByteView data) const override; + + Hash128 twox_128(qtils::ByteView data) const override; + + Hash128 blake2b_128(qtils::ByteView data) const override; + + Hash256 twox_256(qtils::ByteView data) const override; + + Hash256 blake2b_256(qtils::ByteView data) const override; + + Hash256 keccak_256(qtils::ByteView data) const override; + + Hash256 blake2s_256(qtils::ByteView data) const override; + + Hash256 sha2_256(qtils::ByteView data) const override; + + Hash512 blake2b_512(qtils::ByteView data) const override; + }; + +} // namespace jam::crypto diff --git a/src/crypto/keccak/keccak.hpp b/src/crypto/keccak/keccak.hpp new file mode 100644 index 00000000..37ec247e --- /dev/null +++ b/src/crypto/keccak/keccak.hpp @@ -0,0 +1,24 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include "keccak/keccak.h" + +namespace jam::crypto { + inline Hash256 keccak(qtils::ByteView buf) { + Hash256 out; + sha3_HashBuffer(256, + SHA3_FLAGS::SHA3_FLAGS_KECCAK, + buf.data(), + buf.size(), + out.data(), + 32); + return out; + } +} // namespace jam::crypto diff --git a/src/crypto/sha/CMakeLists.txt b/src/crypto/sha/CMakeLists.txt new file mode 100644 index 00000000..ad55bc5a --- /dev/null +++ b/src/crypto/sha/CMakeLists.txt @@ -0,0 +1,15 @@ +# +# Copyright Quadrivium LLC +# All Rights Reserved +# SPDX-License-Identifier: Apache-2.0 +# + +add_library(sha + sha256.hpp + sha256.cpp +) +target_link_libraries(sha + PUBLIC OpenSSL::SSL + OpenSSL::Crypto + qtils::qtils +) diff --git a/src/crypto/sha/sha256.cpp b/src/crypto/sha/sha256.cpp new file mode 100644 index 00000000..abded1bd --- /dev/null +++ b/src/crypto/sha/sha256.cpp @@ -0,0 +1,26 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "crypto/sha/sha256.hpp" + +#include + +namespace jam::crypto { + Hash256 sha256(std::string_view input) { + return sha256( + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + {reinterpret_cast(input.data()), input.size()}); + } + + Hash256 sha256(qtils::ByteView input) { + Hash256 out; + SHA256_CTX ctx; + SHA256_Init(&ctx); + SHA256_Update(&ctx, input.data(), input.size()); + SHA256_Final(out.data(), &ctx); + return out; + } +} // namespace jam::crypto diff --git a/src/crypto/sha/sha256.hpp b/src/crypto/sha/sha256.hpp new file mode 100644 index 00000000..5077ce56 --- /dev/null +++ b/src/crypto/sha/sha256.hpp @@ -0,0 +1,29 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include "crypto/hash_types.hpp" + +namespace jam::crypto { + + /** + * Take a SHA-256 hash from string + * @param input to be hashed + * @return hashed bytes + */ + Hash256 sha256(std::string_view input); + + /** + * Take a SHA-256 hash from bytes + * @param input to be hashed + * @return hashed bytes + */ + Hash256 sha256(qtils::ByteView input); + +} // namespace jam::crypto diff --git a/src/crypto/twox/CMakeLists.txt b/src/crypto/twox/CMakeLists.txt new file mode 100644 index 00000000..9cdf301f --- /dev/null +++ b/src/crypto/twox/CMakeLists.txt @@ -0,0 +1,13 @@ +# +# Copyright Quadrivium LLC +# All Rights Reserved +# SPDX-License-Identifier: Apache-2.0 +# + +add_library(twox + twox.cpp +) +target_link_libraries(twox + xxHash::xxhash + qtils::qtils + ) diff --git a/src/crypto/twox/twox.cpp b/src/crypto/twox/twox.cpp new file mode 100644 index 00000000..e8d9938c --- /dev/null +++ b/src/crypto/twox/twox.cpp @@ -0,0 +1,64 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "crypto/twox/twox.hpp" + +#include + +namespace jam::crypto { + + void make_twox64(const uint8_t *in, uint32_t len, uint8_t *out) { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + auto *ptr = reinterpret_cast(out); + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) + ptr[0] = XXH64(in, len, 0); + } + + Hash64 make_twox64(qtils::ByteView buf) { + Hash64 hash{}; + make_twox64(buf.data(), buf.size(), hash.data()); + return hash; + } + + void make_twox128(const uint8_t *in, uint32_t len, uint8_t *out) { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + auto *ptr = reinterpret_cast(out); + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) + ptr[0] = XXH64(in, len, 0); + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) + ptr[1] = XXH64(in, len, 1); + } + + Hash128 make_twox128(qtils::ByteView buf) { + Hash128 hash{}; + make_twox128(buf.data(), buf.size(), hash.data()); + return hash; + } + + void make_twox256(const uint8_t *in, uint32_t len, uint8_t *out) { + // Ensure the buffer is aligned to the boundary required for uint64_t + // (required for happy UBSAN) + std::array aligned_out{}; + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + auto *ptr = reinterpret_cast(aligned_out.data()); + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) + ptr[0] = XXH64(in, len, 0); + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) + ptr[1] = XXH64(in, len, 1); + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) + ptr[2] = XXH64(in, len, 2); + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) + ptr[3] = XXH64(in, len, 3); + std::memcpy(out, aligned_out.data(), 4 * sizeof(uint64_t)); + } + + Hash256 make_twox256(qtils::ByteView buf) { + Hash256 hash{}; + make_twox256(buf.data(), buf.size(), hash.data()); + return hash; + } + +} // namespace jam::crypto diff --git a/src/crypto/twox/twox.hpp b/src/crypto/twox/twox.hpp new file mode 100644 index 00000000..aa33ade3 --- /dev/null +++ b/src/crypto/twox/twox.hpp @@ -0,0 +1,21 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include + +namespace jam::crypto { + + Hash64 make_twox64(qtils::ByteView buf); + + Hash128 make_twox128(qtils::ByteView buf); + + Hash256 make_twox256(qtils::ByteView buf); + +} // namespace jam::crypto diff --git a/src/injector/CMakeLists.txt b/src/injector/CMakeLists.txt index 7953afaa..fd213f28 100644 --- a/src/injector/CMakeLists.txt +++ b/src/injector/CMakeLists.txt @@ -16,6 +16,7 @@ target_link_libraries(node_injector application metrics clock + hasher se_async modules storage diff --git a/src/injector/node_injector.cpp b/src/injector/node_injector.cpp index c0945cb6..55a23001 100644 --- a/src/injector/node_injector.cpp +++ b/src/injector/node_injector.cpp @@ -26,6 +26,7 @@ #include "app/impl/state_manager_impl.hpp" #include "app/impl/watchdog.hpp" #include "clock/impl/clock_impl.hpp" +#include "crypto/hasher/hasher_impl.hpp" #include "injector/bind_by_lambda.hpp" #include "loaders/loader.hpp" #include "log/logger.hpp" @@ -78,6 +79,7 @@ namespace { //di::bind.to(), di::bind.to(), di::bind.to(), + di::bind.to(), // user-defined overrides... std::forward(args)...); diff --git a/src/jam_types/types.tmp.hpp b/src/jam_types/types.tmp.hpp index ad2a98db..1bbc6bc6 100644 --- a/src/jam_types/types.tmp.hpp +++ b/src/jam_types/types.tmp.hpp @@ -9,9 +9,9 @@ #include #include +#include namespace jam { - // stub types. must be refactored in future struct Stub {}; @@ -24,8 +24,13 @@ namespace jam { struct BlockIndex { BlockNumber number; BlockHash hash; + auto operator<=>(const BlockIndex &other) const = default; }; +} // namespace jam +SCALE_TIE_HASH_STD(jam::BlockIndex); +namespace jam { + using Block = test_vectors::Block; using BlockHeader = test_vectors::Header; diff --git a/src/scale/tie_hash.hpp b/src/scale/tie_hash.hpp new file mode 100644 index 00000000..7d78aee6 --- /dev/null +++ b/src/scale/tie_hash.hpp @@ -0,0 +1,44 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include "utils/tuple_hash.hpp" + +namespace scale { + using detail::decompose_and_apply; + + size_t tieHash(const auto &value) { + size_t seed = 0; + + auto append = [&](const T &element) { + boost::hash_combine(seed, std::hash>()(element)); + }; + + auto apply = [&](const auto &...elements) { // + (append(elements), ...); + }; + + decompose_and_apply(value, apply); + + return seed; + } + +} // namespace scale + +#define SCALE_TIE_HASH_BOOST(type) \ + friend auto hash_value(const type &v) { \ + return ::scale::tieHash(v); \ + } + +#define SCALE_TIE_HASH_STD(type) \ + template <> \ + struct std::hash { \ + size_t operator()(const type &v) const { \ + return ::scale::tieHash(v); \ + } \ + } diff --git a/src/third_party/CMakeLists.txt b/src/third_party/CMakeLists.txt new file mode 100644 index 00000000..293ff3ee --- /dev/null +++ b/src/third_party/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(keccak) diff --git a/src/third_party/keccak/CMakeLists.txt b/src/third_party/keccak/CMakeLists.txt new file mode 100644 index 00000000..3c452a59 --- /dev/null +++ b/src/third_party/keccak/CMakeLists.txt @@ -0,0 +1,14 @@ +# +# Copyright Quadrivium LLC +# All Rights Reserved +# SPDX-License-Identifier: Apache-2.0 +# + +add_library(keccak + keccak.c +) +set_source_files_properties(keccak.с PROPERTIES LANGUAGE C) +target_link_libraries(keccak + OpenSSL::Crypto + qtils::qtils +) diff --git a/src/third_party/keccak/keccak.c b/src/third_party/keccak/keccak.c new file mode 100644 index 00000000..2b303a1f --- /dev/null +++ b/src/third_party/keccak/keccak.c @@ -0,0 +1,320 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include + +#include "keccak.h" + +#define SHA3_ASSERT(x) +#if defined(_MSC_VER) +#define SHA3_TRACE(format, ...) +#define SHA3_TRACE_BUF(format, buf, l, ...) +#else +#define SHA3_TRACE(format, args...) +#define SHA3_TRACE_BUF(format, buf, l, args...) +#endif + +/* + * This flag is used to configure "pure" Keccak, as opposed to NIST SHA3. + */ +#define SHA3_USE_KECCAK_FLAG 0x80000000 +#define SHA3_CW(x) ((x) & (~SHA3_USE_KECCAK_FLAG)) + +#if defined(_MSC_VER) +#define SHA3_CONST(x) x +#else +#define SHA3_CONST(x) x##L +#endif + +#ifndef SHA3_ROTL64 +#define SHA3_ROTL64(x, y) \ + (((x) << (y)) | ((x) >> ((sizeof(uint64_t) * 8) - (y)))) +#endif + +static const uint64_t keccakf_rndc[24] = { + SHA3_CONST(0x0000000000000001UL), SHA3_CONST(0x0000000000008082UL), + SHA3_CONST(0x800000000000808aUL), SHA3_CONST(0x8000000080008000UL), + SHA3_CONST(0x000000000000808bUL), SHA3_CONST(0x0000000080000001UL), + SHA3_CONST(0x8000000080008081UL), SHA3_CONST(0x8000000000008009UL), + SHA3_CONST(0x000000000000008aUL), SHA3_CONST(0x0000000000000088UL), + SHA3_CONST(0x0000000080008009UL), SHA3_CONST(0x000000008000000aUL), + SHA3_CONST(0x000000008000808bUL), SHA3_CONST(0x800000000000008bUL), + SHA3_CONST(0x8000000000008089UL), SHA3_CONST(0x8000000000008003UL), + SHA3_CONST(0x8000000000008002UL), SHA3_CONST(0x8000000000000080UL), + SHA3_CONST(0x000000000000800aUL), SHA3_CONST(0x800000008000000aUL), + SHA3_CONST(0x8000000080008081UL), SHA3_CONST(0x8000000000008080UL), + SHA3_CONST(0x0000000080000001UL), SHA3_CONST(0x8000000080008008UL)}; + +static const unsigned keccakf_rotc[24] = {1, 3, 6, 10, 15, 21, 28, 36, + 45, 55, 2, 14, 27, 41, 56, 8, + 25, 43, 62, 18, 39, 61, 20, 44}; + +static const unsigned keccakf_piln[24] = {10, 7, 11, 17, 18, 3, 5, 16, + 8, 21, 24, 4, 15, 23, 19, 13, + 12, 2, 20, 14, 22, 9, 6, 1}; + +/* generally called after SHA3_KECCAK_SPONGE_WORDS-ctx->capacityWords words + * are XORed into the state s + */ +void keccakf(uint64_t s[25]) { + int i, j, round; // NOLINT + uint64_t t, bc[5]; // NOLINT +#define KECCAK_ROUNDS 24 + +#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ + for (i = 0; i < 25; i++) { + s[i] = LE_BE_SWAP64(s[i]); + } +#endif + + for (round = 0; round < KECCAK_ROUNDS; round++) { + /* Theta */ + for (i = 0; i < 5; i++) { + bc[i] = s[i] ^ s[i + 5] ^ s[i + 10] ^ s[i + 15] ^ s[i + 20]; + } + + for (i = 0; i < 5; i++) { + t = bc[(i + 4) % 5] ^ SHA3_ROTL64(bc[(i + 1) % 5], 1); + for (j = 0; j < 25; j += 5) { + s[j + i] ^= t; + } + } + + /* Rho Pi */ + t = s[1]; + for (i = 0; i < 24; i++) { + j = keccakf_piln[i]; // NOLINT + bc[0] = s[j]; + s[j] = SHA3_ROTL64(t, keccakf_rotc[i]); + t = bc[0]; + } + + /* Chi */ + for (j = 0; j < 25; j += 5) { + for (i = 0; i < 5; i++) { + bc[i] = s[j + i]; + } + for (i = 0; i < 5; i++) { + s[j + i] ^= (~bc[(i + 1) % 5]) & bc[(i + 2) % 5]; + } + } + + /* Iota */ + s[0] ^= keccakf_rndc[round]; + } + +#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ + for (i = 0; i < 25; i++) { + s[i] = LE_BE_SWAP64(s[i]); + } +#endif +} + +/* *************************** Public Inteface ************************ */ + +/* For Init or Reset call these: */ +sha3_return_t sha3_Init(void *priv, unsigned bitSize) { + sha3_context *ctx = (sha3_context *)priv; + if (bitSize != 256 && bitSize != 384 && bitSize != 512) { + return SHA3_RETURN_BAD_PARAMS; + } + memset(ctx, 0, sizeof(*ctx)); + ctx->capacityWords = 2 * bitSize / (8 * sizeof(uint64_t)); + return SHA3_RETURN_OK; +} + +void sha3_Init256(void *priv) { + sha3_Init(priv, 256); +} + +void sha3_Init384(void *priv) { + sha3_Init(priv, 384); +} + +void sha3_Init512(void *priv) { + sha3_Init(priv, 512); +} + +enum SHA3_FLAGS sha3_SetFlags(void *priv, enum SHA3_FLAGS flags) { + sha3_context *ctx = (sha3_context *)priv; + flags &= SHA3_FLAGS_KECCAK; + ctx->capacityWords |= (flags == SHA3_FLAGS_KECCAK ? SHA3_USE_KECCAK_FLAG : 0); + return flags; +} + +void sha3_Update(void *priv, const void *bufIn, size_t len) { + sha3_context *ctx = (sha3_context *)priv; + + /* 0...7 -- how much is needed to have a word */ + unsigned old_tail = (8 - ctx->byteIndex) & 7; + + size_t words; + unsigned tail; + size_t i; + + const uint8_t *buf = bufIn; + + SHA3_TRACE_BUF("called to update with:", buf, len); + + SHA3_ASSERT(ctx->byteIndex < 8); + SHA3_ASSERT(ctx->wordIndex < sizeof(ctx->s) / sizeof(ctx->s[0])); + + if (len < old_tail) { /* have no complete word or haven't started + * the word yet */ + SHA3_TRACE("because %d<%d, store it and return", + (unsigned)len, + (unsigned)old_tail); + /* endian-independent code follows: */ + while (len--) { + ctx->saved |= (uint64_t)(*(buf++)) << ((ctx->byteIndex++) * 8); + } + SHA3_ASSERT(ctx->byteIndex < 8); + return; + } + + if (old_tail) { /* will have one word to process */ + SHA3_TRACE("completing one word with %d bytes", (unsigned)old_tail); + /* endian-independent code follows: */ + len -= old_tail; + while (old_tail--) { + ctx->saved |= (uint64_t)(*(buf++)) << ((ctx->byteIndex++) * 8); + } + + /* now ready to add saved to the sponge */ + ctx->s[ctx->wordIndex] ^= ctx->saved; + SHA3_ASSERT(ctx->byteIndex == 8); + ctx->byteIndex = 0; + ctx->saved = 0; + if (++ctx->wordIndex + == (SHA3_KECCAK_SPONGE_WORDS - SHA3_CW(ctx->capacityWords))) { + keccakf(ctx->s); + ctx->wordIndex = 0; + } + } + + /* now work in full words directly from input */ + + SHA3_ASSERT(ctx->byteIndex == 0); + + words = len / sizeof(uint64_t); + tail = len - words * sizeof(uint64_t); + + SHA3_TRACE("have %d full words to process", (unsigned)words); + + for (i = 0; i < words; i++, buf += sizeof(uint64_t)) { + const uint64_t t = + (uint64_t)(buf[0]) | ((uint64_t)(buf[1]) << 8 * 1) + | ((uint64_t)(buf[2]) << 8 * 2) | ((uint64_t)(buf[3]) << 8 * 3) + | ((uint64_t)(buf[4]) << 8 * 4) | ((uint64_t)(buf[5]) << 8 * 5) + | ((uint64_t)(buf[6]) << 8 * 6) | ((uint64_t)(buf[7]) << 8 * 7); +#if defined(__x86_64__) || defined(__i386__) + SHA3_ASSERT(memcmp(&t, buf, 8) == 0); +#endif + ctx->s[ctx->wordIndex] ^= t; + if (++ctx->wordIndex + == (SHA3_KECCAK_SPONGE_WORDS - SHA3_CW(ctx->capacityWords))) { + keccakf(ctx->s); + ctx->wordIndex = 0; + } + } + + SHA3_TRACE("have %d bytes left to process, save them", (unsigned)tail); + + /* finally, save the partial word */ + SHA3_ASSERT(ctx->byteIndex == 0 && tail < 8); + while (tail--) { + SHA3_TRACE("Store byte %02x '%c'", *buf, *buf); + ctx->saved |= (uint64_t)(*(buf++)) << ((ctx->byteIndex++) * 8); + } + SHA3_ASSERT(ctx->byteIndex < 8); + SHA3_TRACE("Have saved=0x%016" PRIx64 " at the end", ctx->saved); +} + +/* This is simply the 'update' with the padding block. + * The padding block is 0x01 || 0x00* || 0x80. First 0x01 and last 0x80 + * bytes are always present, but they can be the same byte. + */ +const void *sha3_Finalize(void *priv) { + sha3_context *ctx = (sha3_context *)priv; + + SHA3_TRACE("called with %d bytes in the buffer", ctx->byteIndex); + + /* Append 2-bit suffix 01, per SHA-3 spec. Instead of 1 for padding we + * use 1<<2 below. The 0x02 below corresponds to the suffix 01. + * Overall, we feed 0, then 1, and finally 1 to start padding. Without + * M || 01, we would simply use 1 to start padding. */ + + uint64_t t; + + if (ctx->capacityWords & SHA3_USE_KECCAK_FLAG) { + /* Keccak version */ + t = (uint64_t)(((uint64_t)1) << (ctx->byteIndex * 8)); // NOLINT + } else { + /* SHA3 version */ + t = (uint64_t)(((uint64_t)(0x02 | (1 << 2))) // NOLINT + << ((ctx->byteIndex) * 8)); // NOLINT + } + + ctx->s[ctx->wordIndex] ^= ctx->saved ^ t; + + ctx->s[SHA3_KECCAK_SPONGE_WORDS - SHA3_CW(ctx->capacityWords) - 1] ^= + SHA3_CONST(0x8000000000000000UL); + keccakf(ctx->s); + + /* Return first bytes of the ctx->s. This conversion is not needed for + * little-endian platforms e.g. wrap with #if !defined(__BYTE_ORDER__) + * || !defined(__ORDER_LITTLE_ENDIAN__) || + * __BYTE_ORDER__!=__ORDER_LITTLE_ENDIAN__ + * ... the conversion below ... + * #endif */ + { + unsigned i; + for (i = 0; i < SHA3_KECCAK_SPONGE_WORDS; i++) { // NOLINT + const unsigned t1 = (uint32_t)ctx->s[i]; + const unsigned t2 = (uint32_t)((ctx->s[i] >> 16) >> 16); + ctx->sb[i * 8 + 0] = (uint8_t)(t1); + ctx->sb[i * 8 + 1] = (uint8_t)(t1 >> 8); + ctx->sb[i * 8 + 2] = (uint8_t)(t1 >> 16); + ctx->sb[i * 8 + 3] = (uint8_t)(t1 >> 24); + ctx->sb[i * 8 + 4] = (uint8_t)(t2); + ctx->sb[i * 8 + 5] = (uint8_t)(t2 >> 8); + ctx->sb[i * 8 + 6] = (uint8_t)(t2 >> 16); + ctx->sb[i * 8 + 7] = (uint8_t)(t2 >> 24); + } + } + + SHA3_TRACE_BUF("Hash: (first 32 bytes)", ctx->sb, 256 / 8); + + return (ctx->sb); +} + +sha3_return_t sha3_HashBuffer(unsigned bitSize, + enum SHA3_FLAGS flags, + const void *in, + unsigned inBytes, + void *out, + unsigned outBytes) { + sha3_return_t err; + sha3_context c; + + err = sha3_Init(&c, bitSize); + if (err != SHA3_RETURN_OK) { + return err; + } + if (sha3_SetFlags(&c, flags) != flags) { + return SHA3_RETURN_BAD_PARAMS; + } + sha3_Update(&c, in, inBytes); + const void *h = sha3_Finalize(&c); + + if (outBytes > bitSize / 8) { + outBytes = bitSize / 8; + } + memcpy(out, h, outBytes); + return SHA3_RETURN_OK; +} diff --git a/src/third_party/keccak/keccak.h b/src/third_party/keccak/keccak.h new file mode 100644 index 00000000..ef50cee3 --- /dev/null +++ b/src/third_party/keccak/keccak.h @@ -0,0 +1,61 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#if defined(__cplusplus) +extern "C" { +#endif + +/* 'Words' here refers to uint64_t */ +#define SHA3_KECCAK_SPONGE_WORDS \ + (((1600) / 8 /*bits to byte*/) / sizeof(uint64_t)) +typedef struct sha3_context_ { + uint64_t saved; /* the portion of the input message that we + * didn't consume yet */ + union { /* Keccak's state */ + uint64_t s[SHA3_KECCAK_SPONGE_WORDS]; + uint8_t sb[SHA3_KECCAK_SPONGE_WORDS * 8]; + }; + unsigned byteIndex; /* 0..7--the next byte after the set one + * (starts from 0; 0--none are buffered) */ + unsigned wordIndex; /* 0..24--the next word to integrate input + * (starts from 0) */ + unsigned capacityWords; /* the double size of the hash output in + * words (e.g. 16 for Keccak 512) */ +} sha3_context; + +enum SHA3_FLAGS { SHA3_FLAGS_NONE = 0, SHA3_FLAGS_KECCAK = 1 }; + +enum SHA3_RETURN { SHA3_RETURN_OK = 0, SHA3_RETURN_BAD_PARAMS = 1 }; +typedef enum SHA3_RETURN sha3_return_t; + +/* For Init or Reset call these: */ +sha3_return_t sha3_Init(void *priv, unsigned bitSize); +void keccakf(uint64_t s[25]); + +void sha3_Init256(void *priv); +void sha3_Init384(void *priv); +void sha3_Init512(void *priv); + +enum SHA3_FLAGS sha3_SetFlags(void *priv, enum SHA3_FLAGS); + +void sha3_Update(void *priv, const void *bufIn, size_t len); + +const void *sha3_Finalize(void *priv); + +/* Single-call hashing */ +sha3_return_t sha3_HashBuffer( + unsigned bitSize, /* 256, 384, 512 */ + enum SHA3_FLAGS flags, /* SHA3_FLAGS_NONE or SHA3_FLAGS_KECCAK */ + const void *in, + unsigned inBytes, + void *out, + unsigned outBytes); /* up to bitSize/8; truncation OK */ + +#if defined(__cplusplus) +} +#endif diff --git a/src/utils/tuple_hash.hpp b/src/utils/tuple_hash.hpp new file mode 100644 index 00000000..3d75f636 --- /dev/null +++ b/src/utils/tuple_hash.hpp @@ -0,0 +1,35 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include + +template +struct std::hash> { + private: + template + void hash_element_of_tuple(size_t &result, + const std::tuple &v) const { + auto &item = std::get(v); + boost::hash_combine(result, + std::hash>()(item)); + if constexpr (sizeof...(Args) > I + 1) { + hash_element_of_tuple(result, v); + } + } + + public: + size_t operator()(const std::tuple &value) const { + size_t result = 0; + if constexpr (sizeof...(Args) > 0) { + hash_element_of_tuple<0>(result, value); + } + return result; + } +}; diff --git a/vcpkg.json b/vcpkg.json index ee598f75..ca86b6d2 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -15,7 +15,9 @@ "prometheus-cpp", "ftxui", "rocksdb", - "boost-property-tree" + "boost-property-tree", + "openssl", + "xxhash" ], "features": { "test": { "description": "Test", "dependencies": ["gtest"]} From 9d1637c05c6a7912a6cb7d91889a3f6f5fb21faa Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Fri, 6 Jun 2025 03:00:23 +0300 Subject: [PATCH 03/14] feature: blockchain types Signed-off-by: Dmitriy Khaustov aka xDimon --- src/jam_types/block.hpp | 20 +++ src/jam_types/block_data.hpp | 21 +++ src/jam_types/block_header.hpp | 135 ++++++++++++++++++++ src/jam_types/extricsic.hpp | 32 +++++ src/jam_types/justification.hpp | 17 +++ src/jam_types/types.tmp.hpp | 52 +++++++- src/modules/shared/networking_types.tmp.hpp | 1 + src/scale/jam_scale.hpp | 4 +- src/utils/custom_equality.hpp | 17 +++ 9 files changed, 293 insertions(+), 6 deletions(-) create mode 100644 src/jam_types/block.hpp create mode 100644 src/jam_types/block_data.hpp create mode 100644 src/jam_types/block_header.hpp create mode 100644 src/jam_types/extricsic.hpp create mode 100644 src/jam_types/justification.hpp create mode 100644 src/utils/custom_equality.hpp diff --git a/src/jam_types/block.hpp b/src/jam_types/block.hpp new file mode 100644 index 00000000..c8d21da1 --- /dev/null +++ b/src/jam_types/block.hpp @@ -0,0 +1,20 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include + +namespace jam { + + struct Block { + BlockHeader header; + BlockBody extrinsic; + bool operator==(const Block &) const = default; + }; + +} // namespace jam diff --git a/src/jam_types/block_data.hpp b/src/jam_types/block_data.hpp new file mode 100644 index 00000000..a836cbf2 --- /dev/null +++ b/src/jam_types/block_data.hpp @@ -0,0 +1,21 @@ +/** +* Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +namespace jam { + + using test_vectors::Extrinsic; + + struct BlockData { + BlockHash hash; + std::optional header; + std::optional extrinsic; + }; + +} // namespace jam diff --git a/src/jam_types/block_header.hpp b/src/jam_types/block_header.hpp new file mode 100644 index 00000000..aa1f3adf --- /dev/null +++ b/src/jam_types/block_header.hpp @@ -0,0 +1,135 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include +#include + +namespace jam { + + using test_vectors::BandersnatchVrfSignature; + using test_vectors::EpochMark; + using test_vectors::HeaderHash; + using test_vectors::OffendersMark; + using test_vectors::OpaqueHash; + using test_vectors::StateRoot; + using test_vectors::TicketsMark; + using test_vectors::TimeSlot; + using test_vectors::ValidatorIndex; + + /** + * @struct BlockHeader represents header of a block + */ + struct BlockHeader { + /// Hp - parent block header hash + HeaderHash parent; + /// Hr - prior state root + StateRoot parent_state_root; + /// Hx - exctrinsic hash + OpaqueHash extrinsic_hash; + /// Ht - time-slot index + TimeSlot slot; + /// He - epoch marker + std::optional epoch_mark; + /// Hw - winning-tickets marker + std::optional tickets_mark; + /// Ho - offenders marker + OffendersMark offenders_mark; + /// Hi - Bandersnatch block author index + ValidatorIndex author_index; + /// Hv - the entropy-yielding vrf signature + BandersnatchVrfSignature entropy_source; + /// Hs - block seal + BandersnatchVrfSignature seal; + + /// Block hash if calculated + mutable std::optional hash_opt{}; + + CUSTOM_EQUALITY(BlockHeader, + parent, + parent_state_root, + extrinsic_hash, + slot, + epoch_mark, + tickets_mark, + offenders_mark, + author_index, + entropy_source, + seal); + SCALE_CUSTOM_DECOMPOSITION(BlockHeader, + parent, + parent_state_root, + extrinsic_hash, + slot, + epoch_mark, + tickets_mark, + offenders_mark, + author_index, + entropy_source, + seal); + + const BlockHash &hash() const { + BOOST_ASSERT_MSG(hash_opt.has_value(), + "Hash must be calculated and saved before that"); + return hash_opt.value(); + } + + void updateHash(const crypto::Hasher &hasher) const { + auto enc_res = encode(*this); + BOOST_ASSERT_MSG(enc_res.has_value(), + "Header should be encoded errorless"); + hash_opt.emplace(hasher.blake2b_256(enc_res.value())); + } + + BlockInfo index() const { + return {slot, hash()}; + } + }; + + struct UnsealedBlockHeader : private BlockHeader { + UnsealedBlockHeader() = delete; + + using BlockHeader::author_index; + using BlockHeader::entropy_source; + using BlockHeader::epoch_mark; + using BlockHeader::extrinsic_hash; + using BlockHeader::offenders_mark; + using BlockHeader::parent; + using BlockHeader::parent_state_root; + using BlockHeader::slot; + using BlockHeader::tickets_mark; + + CUSTOM_EQUALITY(UnsealedBlockHeader, + parent, + parent_state_root, + extrinsic_hash, + slot, + epoch_mark, + tickets_mark, + offenders_mark, + author_index, + entropy_source); + SCALE_CUSTOM_DECOMPOSITION(UnsealedBlockHeader, + parent, + parent_state_root, + extrinsic_hash, + slot, + epoch_mark, + tickets_mark, + offenders_mark, + author_index, + entropy_source); + }; + + inline void calculateBlockHash(const BlockHeader &header, + const crypto::Hasher &hasher) { + header.updateHash(hasher); + } + +} // namespace jam diff --git a/src/jam_types/extricsic.hpp b/src/jam_types/extricsic.hpp new file mode 100644 index 00000000..c22d4698 --- /dev/null +++ b/src/jam_types/extricsic.hpp @@ -0,0 +1,32 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +namespace jam { + + using test_vectors::TicketsExtrinsic; + using test_vectors::PreimagesExtrinsic; + using test_vectors::GuaranteesExtrinsic; + using test_vectors::AssurancesExtrinsic; + using test_vectors::DisputesExtrinsic; + + struct BlockBody { + TicketsExtrinsic tickets; + PreimagesExtrinsic preimages; + GuaranteesExtrinsic guarantees; + AssurancesExtrinsic assurances; + DisputesExtrinsic disputes; + bool operator==(const BlockBody &) const = default; + + operator test_vectors::Extrinsic() const { + return reinterpret_cast(*this); + } + }; + +} // namespace jam diff --git a/src/jam_types/justification.hpp b/src/jam_types/justification.hpp new file mode 100644 index 00000000..5a86a1ac --- /dev/null +++ b/src/jam_types/justification.hpp @@ -0,0 +1,17 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include + +namespace jam { + + using Justification = qtils::Tagged; + +} diff --git a/src/jam_types/types.tmp.hpp b/src/jam_types/types.tmp.hpp index 1bbc6bc6..97191d09 100644 --- a/src/jam_types/types.tmp.hpp +++ b/src/jam_types/types.tmp.hpp @@ -18,11 +18,12 @@ namespace jam { // blockchain types - using BlockNumber = uint32_t; using BlockHash = test_vectors::HeaderHash; + using test_vectors::TimeSlot; + struct BlockIndex { - BlockNumber number; + TimeSlot slot; BlockHash hash; auto operator<=>(const BlockIndex &other) const = default; }; @@ -31,8 +32,11 @@ namespace jam { SCALE_TIE_HASH_STD(jam::BlockIndex); namespace jam { - using Block = test_vectors::Block; - using BlockHeader = test_vectors::Header; + using BlockInfo = BlockIndex; + + using BlockNumber = test_vectors::TimeSlot; + + using BlockId = std::variant; // networking types @@ -86,5 +90,45 @@ struct fmt::formatter { } }; +template <> +struct fmt::formatter { + // Presentation format: 's' - short, 'l' - long. + char presentation = 's'; + + // Parses format specifications of the form ['s' | 'l']. + constexpr auto parse(format_parse_context &ctx) -> decltype(ctx.begin()) { + // Parse the presentation format and store it in the formatter: + auto it = ctx.begin(), end = ctx.end(); + if (it != end && (*it == 's' or *it == 'l')) { + presentation = *it++; + } + + // Check if reached the end of the range: + if (it != end && *it != '}') { + throw format_error("invalid format"); + } + + // Return an iterator past the end of the parsed range: + return it; + } + + // Formats the BlockInfo using the parsed format specification (presentation) + // stored in this formatter. + template + auto format(const jam::BlockInfo &block_info, FormatContext &ctx) const + -> decltype(ctx.out()) { + // ctx.out() is an output iterator to write to. + + if (presentation == 's') { + return fmt::format_to( + ctx.out(), "{:0x} @ {}", block_info.hash, block_info.slot); + } + + return fmt::format_to( + ctx.out(), "{:0xx} @ {}", block_info.hash, block_info.slot); + } +}; + + template struct fmt::formatter> : formatter {}; diff --git a/src/modules/shared/networking_types.tmp.hpp b/src/modules/shared/networking_types.tmp.hpp index 703713df..e18aad96 100644 --- a/src/modules/shared/networking_types.tmp.hpp +++ b/src/modules/shared/networking_types.tmp.hpp @@ -7,6 +7,7 @@ #pragma once #include "jam_types/types.tmp.hpp" +#include "jam_types/block.hpp" #include "utils/request_id.hpp" namespace jam::messages { diff --git a/src/scale/jam_scale.hpp b/src/scale/jam_scale.hpp index 39ea2134..b5a1d761 100644 --- a/src/scale/jam_scale.hpp +++ b/src/scale/jam_scale.hpp @@ -9,8 +9,8 @@ #include namespace jam { - using scale::decode; - using scale::encode; + using scale::impl::memory::decode; + using scale::impl::memory::encode; template [[nodiscard]] outcome::result> encode_with_config( diff --git a/src/utils/custom_equality.hpp b/src/utils/custom_equality.hpp new file mode 100644 index 00000000..8d29b2e8 --- /dev/null +++ b/src/utils/custom_equality.hpp @@ -0,0 +1,17 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#define CUSTOM_EQUALITY(Self, ...) \ + decltype(auto) _tie() const { \ + return std::tie(__VA_ARGS__); \ + } \ + bool operator==(const Self &other) const noexcept { \ + return _tie() == other._tie(); \ + } From 2f4e817f0e5cbf28cc6cbbbc4cbfd98aa0816891 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Fri, 6 Jun 2025 01:08:24 +0300 Subject: [PATCH 04/14] feature: genesis block header Signed-off-by: Dmitriy Khaustov aka xDimon --- src/blockchain/CMakeLists.txt | 0 src/blockchain/genesis_block_header.hpp | 18 ++++++++++ .../impl/genesis_block_header_impl.cpp | 35 +++++++++++++++++++ .../impl/genesis_block_header_impl.hpp | 32 +++++++++++++++++ src/injector/node_injector.cpp | 2 ++ 5 files changed, 87 insertions(+) create mode 100644 src/blockchain/CMakeLists.txt create mode 100644 src/blockchain/genesis_block_header.hpp create mode 100644 src/blockchain/impl/genesis_block_header_impl.cpp create mode 100644 src/blockchain/impl/genesis_block_header_impl.hpp diff --git a/src/blockchain/CMakeLists.txt b/src/blockchain/CMakeLists.txt new file mode 100644 index 00000000..e69de29b diff --git a/src/blockchain/genesis_block_header.hpp b/src/blockchain/genesis_block_header.hpp new file mode 100644 index 00000000..00c101d2 --- /dev/null +++ b/src/blockchain/genesis_block_header.hpp @@ -0,0 +1,18 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "jam_types/block_header.hpp" + +namespace jam::blockchain { + + class GenesisBlockHeader : public BlockHeader { + public: + using BlockHeader::BlockHeader; + }; + +} // namespace jam::blockchain diff --git a/src/blockchain/impl/genesis_block_header_impl.cpp b/src/blockchain/impl/genesis_block_header_impl.cpp new file mode 100644 index 00000000..188ac822 --- /dev/null +++ b/src/blockchain/impl/genesis_block_header_impl.cpp @@ -0,0 +1,35 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + + +#include "blockchain/impl/genesis_block_header_impl.hpp" + +#include "app/chain_spec.hpp" +#include "crypto/hasher.hpp" +#include "log/logger.hpp" +#include "scale/jam_scale.hpp" +#include "jam_types/config-tiny.hpp" + +namespace jam::blockchain { + + GenesisBlockHeaderImpl::GenesisBlockHeaderImpl( + const qtils::SharedRef &logsys, + const qtils::SharedRef &chain_spec, + const qtils::SharedRef &hasher) { + scale::impl::memory::DecoderFromSpan decoder(chain_spec->genesisHeader(), test_vectors::config::tiny); + try { + decode(static_cast(*this), decoder); + } catch (std::system_error &e) { + auto logger = logsys->getLogger("ChainSpec", "application"); + SL_CRITICAL(logger, + "Failed to decode genesis block header from chain spec: {}", + e.code()); + qtils::raise(e.code()); + } + updateHash(*hasher); + } + +} // namespace jam::blockchain diff --git a/src/blockchain/impl/genesis_block_header_impl.hpp b/src/blockchain/impl/genesis_block_header_impl.hpp new file mode 100644 index 00000000..451b12c8 --- /dev/null +++ b/src/blockchain/impl/genesis_block_header_impl.hpp @@ -0,0 +1,32 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include "blockchain/genesis_block_header.hpp" + +namespace jam::app { + class ChainSpec; +} +namespace jam::log { + class LoggingSystem; +} +namespace jam::crypto { + class Hasher; +} + +namespace jam::blockchain { + + class GenesisBlockHeaderImpl final : public GenesisBlockHeader { + public: + GenesisBlockHeaderImpl(const qtils::SharedRef &logsys, + const qtils::SharedRef &chain_spec, + const qtils::SharedRef &hasher); + }; + +} // namespace jam::blockchain diff --git a/src/injector/node_injector.cpp b/src/injector/node_injector.cpp index 55a23001..5e289d8a 100644 --- a/src/injector/node_injector.cpp +++ b/src/injector/node_injector.cpp @@ -25,6 +25,7 @@ #include "app/impl/chain_spec_impl.hpp" #include "app/impl/state_manager_impl.hpp" #include "app/impl/watchdog.hpp" +#include "blockchain/impl/genesis_block_header_impl.hpp" #include "clock/impl/clock_impl.hpp" #include "crypto/hasher/hasher_impl.hpp" #include "injector/bind_by_lambda.hpp" @@ -80,6 +81,7 @@ namespace { di::bind.to(), di::bind.to(), di::bind.to(), + di::bind.to(), // user-defined overrides... std::forward(args)...); From f997d11850d5a459f72a2aca3d4afa4d22d63e72 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Mon, 16 Jun 2025 15:53:04 +0300 Subject: [PATCH 05/14] feature: block storage Signed-off-by: Dmitriy Khaustov aka xDimon --- example/config.yaml | 4 +- src/CMakeLists.txt | 3 + src/blockchain/CMakeLists.txt | 18 + src/blockchain/block_storage.hpp | 173 +++++++++ src/blockchain/block_storage_error.hpp | 24 ++ src/blockchain/genesis_block_hash.cpp | 16 + src/blockchain/genesis_block_hash.hpp | 24 ++ src/blockchain/impl/block_storage_error.cpp | 26 ++ src/blockchain/impl/block_storage_impl.cpp | 367 ++++++++++++++++++ src/blockchain/impl/block_storage_impl.hpp | 101 +++++ .../impl/block_storage_initializer.cpp | 84 ++++ .../impl/block_storage_initializer.hpp | 40 ++ src/blockchain/impl/storage_util.cpp | 81 ++++ src/blockchain/impl/storage_util.hpp | 101 +++++ src/injector/CMakeLists.txt | 1 + src/injector/node_injector.cpp | 8 +- src/storage/predefined_keys.hpp | 19 + src/storage/rocksdb/rocksdb_spaces.cpp | 7 +- src/storage/spaces.hpp | 4 + tests/mock/blockchain/block_storage_mock.hpp | 117 ++++++ 20 files changed, 1212 insertions(+), 6 deletions(-) create mode 100644 src/blockchain/block_storage.hpp create mode 100644 src/blockchain/block_storage_error.hpp create mode 100644 src/blockchain/genesis_block_hash.cpp create mode 100644 src/blockchain/genesis_block_hash.hpp create mode 100644 src/blockchain/impl/block_storage_error.cpp create mode 100644 src/blockchain/impl/block_storage_impl.cpp create mode 100644 src/blockchain/impl/block_storage_impl.hpp create mode 100644 src/blockchain/impl/block_storage_initializer.cpp create mode 100644 src/blockchain/impl/block_storage_initializer.hpp create mode 100644 src/blockchain/impl/storage_util.cpp create mode 100644 src/blockchain/impl/storage_util.hpp create mode 100644 src/storage/predefined_keys.hpp create mode 100644 tests/mock/blockchain/block_storage_mock.hpp diff --git a/example/config.yaml b/example/config.yaml index 327ac307..390c3835 100644 --- a/example/config.yaml +++ b/example/config.yaml @@ -39,4 +39,6 @@ logging: - name: rpc - name: metrics - name: threads - - name: storage \ No newline at end of file + - name: storage + children: + - name: block_storage diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f670cf58..4b7d0632 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -10,6 +10,9 @@ add_subdirectory(executable) # Application's thinks add_subdirectory(app) +# Blockchain +add_subdirectory(blockchain) + # Cryptography thinks add_subdirectory(crypto) diff --git a/src/blockchain/CMakeLists.txt b/src/blockchain/CMakeLists.txt index e69de29b..7588bb63 100644 --- a/src/blockchain/CMakeLists.txt +++ b/src/blockchain/CMakeLists.txt @@ -0,0 +1,18 @@ +# +# Copyright Quadrivium LLC +# All Rights Reserved +# SPDX-License-Identifier: Apache-2.0 +# + +add_library(blockchain + impl/storage_util.cpp + impl/block_storage_impl.cpp + impl/block_storage_error.cpp + impl/block_storage_initializer.cpp +) +target_link_libraries(blockchain + Boost::boost +) +add_dependencies(blockchain + generate_common_types +) \ No newline at end of file diff --git a/src/blockchain/block_storage.hpp b/src/blockchain/block_storage.hpp new file mode 100644 index 00000000..d61acaaa --- /dev/null +++ b/src/blockchain/block_storage.hpp @@ -0,0 +1,173 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include "jam_types/block.hpp" +#include "jam_types/block_data.hpp" +#include "jam_types/justification.hpp" +#include "jam_types/types.tmp.hpp" + +namespace jam::blockchain { + + /** + * A wrapper for a storage of blocks + * Provides a convenient interface to work with it + */ + class BlockStorage { + public: + virtual ~BlockStorage() = default; + + /** + * Gets leaves of a block tree + * @returns hashes of block tree leaves + */ + [[nodiscard]] virtual outcome::result> + getBlockTreeLeaves() const = 0; + + /** + * Saves provided block tree leaves + * @returns result of saving + */ + virtual outcome::result setBlockTreeLeaves( + std::vector leaves) = 0; + + /** + * Get the last finalized block + * @return BlockInfo of the block + */ + [[nodiscard]] virtual outcome::result getLastFinalized() + const = 0; + + // -- hash -- + + /** + * Adds slot-to-hash record for the provided block index to block + * storage + * @returns success or failure + */ + virtual outcome::result assignHashToSlot( + const BlockIndex &block_index) = 0; + + /** + * Removes slot-to-hash record for provided block index from block storage + * @returns success or failure + */ + virtual outcome::result deassignHashToSlot( + const BlockIndex &block_index) = 0; + + /** + * Tries to get block hashes by slot + * @returns vector of hashes or error + */ + [[nodiscard]] virtual outcome::result> getBlockHash( + BlockNumber slot) const = 0; + + // -- headers -- + + /** + * Check if header existing by provided block {@param block_hash} + * @returns result or error + */ + [[nodiscard]] virtual outcome::result hasBlockHeader( + const BlockHash &block_hash) const = 0; + + /** + * Saves block header to block storage + * @returns hash of saved header or error + */ + virtual outcome::result putBlockHeader( + const BlockHeader &header) = 0; + + /** + * Tries to get a block header by hash + * @returns block header or error + */ + [[nodiscard]] virtual outcome::result getBlockHeader( + const BlockHash &block_hash) const = 0; + + /** + * Attempts to retrieve the block header for the given hash}. + * @param block_hash The hash of the block whose header is to be retrieved. + * @returns An optional containing the block header if found, std::nullopt + * if not found, or an error if the operation fails. + */ + [[nodiscard]] virtual outcome::result> + tryGetBlockHeader(const BlockHash &block_hash) const = 0; + + // -- body -- + + /** + * Saves provided body of block to block storage + * @returns result of saving + */ + virtual outcome::result putBlockBody(const BlockHash &block_hash, + const BlockBody &block_body) = 0; + + /** + * Tries to get block body + * @returns block body or error + */ + [[nodiscard]] virtual outcome::result> + getBlockBody(const BlockHash &block_hash) const = 0; + + /** + * Removes body of block with hash {@param block_hash} from block storage + * @returns result of saving + */ + virtual outcome::result removeBlockBody( + const BlockHash &block_hash) = 0; + + // -- justification -- + + /** + * Saves {@param justification} of block with hash {@param block_hash} to + * block storage + * @returns result of saving + */ + virtual outcome::result putJustification( + const Justification &justification, const BlockHash &block_hash) = 0; + + /** + * Tries to get justification of block finality by {@param block_hash} + * @returns justification or error + */ + virtual outcome::result> getJustification( + const BlockHash &block_hash) const = 0; + + /** + * Removes justification of block with hash {@param block_hash} from block + * storage + * @returns result of saving + */ + virtual outcome::result removeJustification( + const BlockHash &block_hash) = 0; + + // -- combined + + /** + * Saves block to block storage + * @returns hash of saved header or error + */ + virtual outcome::result putBlock(const Block &block) = 0; + + /** + * Tries to get block data + * @returns block data or error + */ + [[nodiscard]] virtual outcome::result> + getBlockData(const BlockHash &block_hash) const = 0; + + /** + * Removes all data of block by hash from block storage + * @returns nothing or error + */ + virtual outcome::result removeBlock(const BlockHash &block_hash) = 0; + }; + +} // namespace jam::blockchain diff --git a/src/blockchain/block_storage_error.hpp b/src/blockchain/block_storage_error.hpp new file mode 100644 index 00000000..7ce5ecf4 --- /dev/null +++ b/src/blockchain/block_storage_error.hpp @@ -0,0 +1,24 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +namespace jam::blockchain { + + enum class BlockStorageError : uint8_t { + BLOCK_EXISTS = 1, + HEADER_NOT_FOUND, + GENESIS_BLOCK_ALREADY_EXISTS, + GENESIS_BLOCK_NOT_FOUND, + FINALIZED_BLOCK_NOT_FOUND, + BLOCK_TREE_LEAVES_NOT_FOUND + }; + +} + +OUTCOME_HPP_DECLARE_ERROR(jam::blockchain, BlockStorageError); diff --git a/src/blockchain/genesis_block_hash.cpp b/src/blockchain/genesis_block_hash.cpp new file mode 100644 index 00000000..e8f7a2f3 --- /dev/null +++ b/src/blockchain/genesis_block_hash.cpp @@ -0,0 +1,16 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "blockchain/genesis_block_hash.hpp" + +#include "blockchain/block_tree.hpp" + +namespace jam::blockchain { + + GenesisBlockHash::GenesisBlockHash(std::shared_ptr block_tree) + : primitives::BlockHash(block_tree->getGenesisBlockHash()){}; + +} // namespace jam::blockchain diff --git a/src/blockchain/genesis_block_hash.hpp b/src/blockchain/genesis_block_hash.hpp new file mode 100644 index 00000000..a24685bf --- /dev/null +++ b/src/blockchain/genesis_block_hash.hpp @@ -0,0 +1,24 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "primitives/common.hpp" + +namespace jam::blockchain { + + class BlockTree; + + class GenesisBlockHash final : public primitives::BlockHash { + public: + GenesisBlockHash(std::shared_ptr block_tree); + }; + +} // namespace jam::blockchain + +template <> +struct fmt::formatter + : fmt::formatter {}; diff --git a/src/blockchain/impl/block_storage_error.cpp b/src/blockchain/impl/block_storage_error.cpp new file mode 100644 index 00000000..3479f447 --- /dev/null +++ b/src/blockchain/impl/block_storage_error.cpp @@ -0,0 +1,26 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "blockchain/block_storage_error.hpp" + +OUTCOME_CPP_DEFINE_CATEGORY(jam::blockchain, BlockStorageError, e) { + using E = BlockStorageError; + switch (e) { + case E::BLOCK_EXISTS: + return "Block already exists on the chain"; + case E::HEADER_NOT_FOUND: + return "Block header was not found"; + case E::GENESIS_BLOCK_ALREADY_EXISTS: + return "Genesis block already exists"; + case E::FINALIZED_BLOCK_NOT_FOUND: + return "Finalized block not found. Possibly storage is corrupted"; + case E::GENESIS_BLOCK_NOT_FOUND: + return "Genesis block not found"; + case E::BLOCK_TREE_LEAVES_NOT_FOUND: + return "Block tree leaves not found"; + } + return "Unknown error"; +} diff --git a/src/blockchain/impl/block_storage_impl.cpp b/src/blockchain/impl/block_storage_impl.cpp new file mode 100644 index 00000000..88c059d6 --- /dev/null +++ b/src/blockchain/impl/block_storage_impl.cpp @@ -0,0 +1,367 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "blockchain/impl/block_storage_impl.hpp" + +#include + +#include "blockchain/block_storage_error.hpp" +#include "blockchain/impl/storage_util.hpp" +#include "scale/jam_scale.hpp" +#include "storage/predefined_keys.hpp" + +namespace jam::blockchain { + + BlockStorageImpl::BlockStorageImpl( + qtils::SharedRef logsys, + qtils::SharedRef storage, + qtils::SharedRef hasher, + std::shared_ptr) + : logger_(logsys->getLogger("BlockStorage", "block_storage")), + storage_(std::move(storage)), + hasher_(std::move(hasher)) {} + + outcome::result> BlockStorageImpl::getBlockTreeLeaves() + const { + if (block_tree_leaves_.has_value()) { + return block_tree_leaves_.value(); + } + + auto default_space = storage_->getSpace(storage::Space::Default); + OUTCOME_TRY(leaves_opt, + default_space->tryGet(storage::kBlockTreeLeavesLookupKey)); + if (not leaves_opt.has_value()) { + return BlockStorageError::BLOCK_TREE_LEAVES_NOT_FOUND; + } + auto &encoded_leaves = leaves_opt.value(); + + OUTCOME_TRY(leaves, decode>(encoded_leaves)); + + block_tree_leaves_.emplace(std::move(leaves)); + + return block_tree_leaves_.value(); + } + + outcome::result BlockStorageImpl::setBlockTreeLeaves( + std::vector leaves) { + std::ranges::sort(leaves); + + if (block_tree_leaves_.has_value() + and block_tree_leaves_.value() == leaves) { + return outcome::success(); + } + + auto default_space = storage_->getSpace(storage::Space::Default); + OUTCOME_TRY(encoded_leaves, encode(leaves)); + OUTCOME_TRY(default_space->put(storage::kBlockTreeLeavesLookupKey, + qtils::ByteVec{std::move(encoded_leaves)})); + + block_tree_leaves_.emplace(std::move(leaves)); + + return outcome::success(); + } + + outcome::result BlockStorageImpl::getLastFinalized() const { + OUTCOME_TRY(leaves, getBlockTreeLeaves()); + auto current_hash = leaves[0]; + for (;;) { + // OUTCOME_TRY(j_opt, getJustification(current_hash)); + // if (j_opt.has_value()) { + // break; + // } // FIXME + OUTCOME_TRY(header, getBlockHeader(current_hash)); + if (header.slot == 0) { + SL_TRACE(logger_, + "Not found block with justification. " + "Genesis block will be used as last finalized ({})", + current_hash); + return {0, current_hash}; // genesis + } + current_hash = header.parent; + } + + OUTCOME_TRY(header, getBlockHeader(current_hash)); + auto found_block = BlockIndex{header.slot, current_hash}; + SL_TRACE(logger_, + "Justification is found in block {}. " + "This block will be used as last finalized", + found_block); + return found_block; + } + + outcome::result BlockStorageImpl::assignHashToSlot( + const BlockInfo &block_index) { + SL_DEBUG(logger_, "Add slot-to-hash for {}", block_index); + auto slot_to_hash_key = slotToHashLookupKey(block_index.slot); + auto storage = storage_->getSpace(storage::Space::LookupKey); + OUTCOME_TRY(hashes, getBlockHash(block_index.slot)); + if (not qtils::cxx23::ranges::contains(hashes, block_index.hash)) { + hashes.emplace_back(block_index.hash); + OUTCOME_TRY(storage->put(slot_to_hash_key, block_index.hash)); + } + return outcome::success(); + } + + outcome::result BlockStorageImpl::deassignHashToSlot( + const BlockIndex &block_index) { + SL_DEBUG(logger_, "Remove num-to-idx for {}", block_index); + auto slot_to_hash_key = slotToHashLookupKey(block_index.slot); + auto storage = storage_->getSpace(storage::Space::LookupKey); + OUTCOME_TRY(hashes, getBlockHash(block_index.slot)); + auto to_erase = std::ranges::remove(hashes, block_index.hash); + if (not to_erase.empty()) { + hashes.erase(to_erase.begin(), to_erase.end()); + if (hashes.empty()) { + OUTCOME_TRY(storage->remove(slot_to_hash_key)); + } else { + OUTCOME_TRY(storage->put(slot_to_hash_key, encode(hashes).value())); + } + } + return outcome::success(); + } + + outcome::result> BlockStorageImpl::getBlockHash( + TimeSlot slot) const { + auto storage = storage_->getSpace(storage::Space::LookupKey); + OUTCOME_TRY(data_opt, storage->tryGet(slotToHashLookupKey(slot))); + if (data_opt.has_value()) { + return decode>(data_opt.value()); + } + return {{}}; + } + + // outcome::result> + // BlockStorageImpl::getBlockHash(const BlockId &block_id) const + // { + // return visit_in_place( + // block_id, + // [&](const BlockNumber &block_number) + // -> outcome::result> { + // auto key_space = storage_->getSpace(storage::Space::kLookupKey); + // OUTCOME_TRY(data_opt, + // key_space->tryGet(slotToHashLookupKey(block_number))); + // if (data_opt.has_value()) { + // OUTCOME_TRY(block_hash, + // BlockHash::fromSpan(data_opt.value())); + // return block_hash; + // } + // return std::nullopt; + // }, + // [](const Hash256 &block_hash) { return block_hash; }); + // } + + outcome::result BlockStorageImpl::hasBlockHeader( + const BlockHash &block_hash) const { + return hasInSpace(*storage_, storage::Space::Header, block_hash); + } + + outcome::result BlockStorageImpl::putBlockHeader( + const BlockHeader &header) { + OUTCOME_TRY(encoded_header, encode(header)); + header.updateHash(*hasher_); + const auto &block_hash = header.hash(); + OUTCOME_TRY(putToSpace(*storage_, + storage::Space::Header, + block_hash, + std::move(encoded_header))); + return block_hash; + } + + outcome::result BlockStorageImpl::getBlockHeader( + const BlockHash &block_hash) const { + OUTCOME_TRY(header_opt, fetchBlockHeader(block_hash)); + if (header_opt.has_value()) { + return header_opt.value(); + } + return BlockStorageError::HEADER_NOT_FOUND; + } + + outcome::result> + BlockStorageImpl::tryGetBlockHeader(const BlockHash &block_hash) const { + return fetchBlockHeader(block_hash); + } + + outcome::result BlockStorageImpl::putBlockBody( + const BlockHash &block_hash, const BlockBody &block_body) { + OUTCOME_TRY(encoded_body, encode(block_body)); + return putToSpace(*storage_, + storage::Space::Extrinsic, + block_hash, + std::move(encoded_body)); + } + + outcome::result> BlockStorageImpl::getBlockBody( + const BlockHash &block_hash) const { + OUTCOME_TRY(encoded_block_body_opt, + getFromSpace(*storage_, storage::Space::Extrinsic, block_hash)); + if (encoded_block_body_opt.has_value()) { + OUTCOME_TRY(block_body, + decode(encoded_block_body_opt.value())); + return std::make_optional(std::move(block_body)); + } + return std::nullopt; + } + + outcome::result BlockStorageImpl::removeBlockBody( + const BlockHash &block_hash) { + auto space = storage_->getSpace(storage::Space::Extrinsic); + return space->remove(block_hash); + } + + outcome::result BlockStorageImpl::putJustification( + const Justification &justification, const BlockHash &hash) { + BOOST_ASSERT(not justification.empty()); + + OUTCOME_TRY(encoded_justification, encode(justification)); + OUTCOME_TRY(putToSpace(*storage_, + storage::Space::Justification, + hash, + std::move(encoded_justification))); + + return outcome::success(); + } + + outcome::result> + BlockStorageImpl::getJustification(const BlockHash &block_hash) const { + OUTCOME_TRY( + encoded_justification_opt, + getFromSpace(*storage_, storage::Space::Justification, block_hash)); + if (encoded_justification_opt.has_value()) { + OUTCOME_TRY(justification, + decode(encoded_justification_opt.value())); + return justification; + } + return std::nullopt; + } + + outcome::result BlockStorageImpl::removeJustification( + const BlockHash &block_hash) { + auto space = storage_->getSpace(storage::Space::Justification); + return space->remove(block_hash); + } + + outcome::result BlockStorageImpl::putBlock(const Block &block) { + // insert provided block's parts into the database + OUTCOME_TRY(block_hash, putBlockHeader(block.header)); + + OUTCOME_TRY(encoded_header, encode(block.header)); + OUTCOME_TRY(putToSpace(*storage_, + storage::Space::Header, + block_hash, + std::move(encoded_header))); + + OUTCOME_TRY(encoded_body, encode(block.extrinsic)); + OUTCOME_TRY(putToSpace(*storage_, + storage::Space::Extrinsic, + block_hash, + std::move(encoded_body))); + + logger_->info("Added block {} as child of {}", + BlockIndex{block.header.slot, block_hash}, + block.header.parent); + return block_hash; + } + + outcome::result> BlockStorageImpl::getBlockData( + const BlockHash &block_hash) const { + BlockData block_data{.hash = block_hash}; + + // Block header + OUTCOME_TRY(header, getBlockHeader(block_hash)); + block_data.header = std::move(header); + + // Block body + OUTCOME_TRY(body_opt, getBlockBody(block_hash)); + block_data.extrinsic = std::move(body_opt); + + // // Justification + // OUTCOME_TRY(justification_opt, getJustification(block_hash)); + // block_data.justification = std::move(justification_opt); + + return block_data; + } + + outcome::result BlockStorageImpl::removeBlock( + const BlockHash &block_hash) { + // Check if block still in storage + OUTCOME_TRY(header_opt, fetchBlockHeader(block_hash)); + if (not header_opt) { + return outcome::success(); + } + const auto &header = header_opt.value(); + + auto block_index = header.index(); + + SL_TRACE(logger_, "Removing block {}…", block_index); + + { // Remove slot-to-hash assigning + auto num_to_hash_key = slotToHashLookupKey(block_index.slot); + + auto key_space = storage_->getSpace(storage::Space::LookupKey); + OUTCOME_TRY(hash_opt, key_space->tryGet(num_to_hash_key.view())); + if (hash_opt == block_hash) { + if (auto res = key_space->remove(num_to_hash_key); res.has_error()) { + SL_ERROR(logger_, + "could not remove slot-to-hash of {} from the storage: {}", + block_index, + res.error()); + return res; + } + SL_DEBUG(logger_, "Removed slot-to-hash of {}", block_index); + } + } + + // TODO(xDimon): needed to clean up trie storage if block deleted + + // Remove the block body + if (auto res = removeBlockBody(block_index.hash); res.has_error()) { + SL_ERROR(logger_, + "could not remove body of block {} from the storage: {}", + block_index, + res.error()); + return res; + } + + // Remove justification for a block + if (auto res = removeJustification(block_index.hash); res.has_error()) { + SL_ERROR( + logger_, + "could not remove justification of block {} from the storage: {}", + block_index, + res.error()); + return res; + } + + { // Remove the block header + auto header_space = storage_->getSpace(storage::Space::Header); + if (auto res = header_space->remove(block_index.hash); res.has_error()) { + SL_ERROR(logger_, + "could not remove header of block {} from the storage: {}", + block_index, + res.error()); + return res; + } + } + + logger_->info("Removed block {}", block_index); + + return outcome::success(); + } + + outcome::result> + BlockStorageImpl::fetchBlockHeader(const BlockHash &block_hash) const { + OUTCOME_TRY(encoded_header_opt, + getFromSpace(*storage_, storage::Space::Header, block_hash)); + if (encoded_header_opt.has_value()) { + auto &encoded_header = encoded_header_opt.value(); + OUTCOME_TRY(header, decode(encoded_header)); + header.hash_opt.emplace(block_hash); + return std::make_optional(std::move(header)); + } + return std::nullopt; + } + +} // namespace jam::blockchain diff --git a/src/blockchain/impl/block_storage_impl.hpp b/src/blockchain/impl/block_storage_impl.hpp new file mode 100644 index 00000000..ac42de42 --- /dev/null +++ b/src/blockchain/impl/block_storage_impl.hpp @@ -0,0 +1,101 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "blockchain/block_storage.hpp" +#include "blockchain/impl/block_storage_initializer.hpp" +#include "log/logger.hpp" +#include "storage/spaced_storage.hpp" + +namespace jam::blockchain { + + class BlockStorageImpl : public BlockStorage, Singleton { + public: + friend class BlockStorageInitializer; + + BlockStorageImpl(qtils::SharedRef logsys, + qtils::SharedRef storage, + qtils::SharedRef hasher, + std::shared_ptr); + + ~BlockStorageImpl() override = default; + + outcome::result setBlockTreeLeaves( + std::vector leaves) override; + + outcome::result> getBlockTreeLeaves() const override; + + outcome::result getLastFinalized() const override; + + // -- hash -- + + outcome::result assignHashToSlot(const BlockIndex &block) override; + + outcome::result deassignHashToSlot( + const BlockIndex &block_index) override; + + outcome::result> getBlockHash( + BlockNumber slot) const override; + + // -- header -- + + outcome::result hasBlockHeader( + const BlockHash &block_hash) const override; + + outcome::result putBlockHeader( + const BlockHeader &header) override; + + outcome::result getBlockHeader( + const BlockHash &block_hash) const override; + + outcome::result> tryGetBlockHeader( + const BlockHash &block_hash) const override; + + // -- body -- + + outcome::result putBlockBody(const BlockHash &block_hash, + const BlockBody &block_body) override; + + outcome::result> getBlockBody( + const BlockHash &block_hash) const override; + + outcome::result removeBlockBody(const BlockHash &block_hash) override; + + // -- justification -- + + outcome::result putJustification( + const Justification &justification, + const BlockHash &block_hash) override; + + outcome::result> getJustification( + const BlockHash &block_hash) const override; + + outcome::result removeJustification( + const BlockHash &block_hash) override; + + // -- combined + + outcome::result putBlock(const Block &block) override; + + outcome::result> getBlockData( + const BlockHash &block_hash) const override; + + outcome::result removeBlock(const BlockHash &block_hash) override; + + private: + outcome::result> fetchBlockHeader( + const BlockHash &block_hash) const; + + log::Logger logger_; + + qtils::SharedRef storage_; + + std::shared_ptr hasher_; + + mutable std::optional> block_tree_leaves_; + }; +} // namespace jam::blockchain diff --git a/src/blockchain/impl/block_storage_initializer.cpp b/src/blockchain/impl/block_storage_initializer.cpp new file mode 100644 index 00000000..3fbd6304 --- /dev/null +++ b/src/blockchain/impl/block_storage_initializer.cpp @@ -0,0 +1,84 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + + +#include "blockchain/impl/block_storage_initializer.hpp" + +#include +#include +#include + +#include "blockchain/block_storage_error.hpp" +#include "blockchain/genesis_block_header.hpp" +#include "blockchain/impl/block_storage_impl.hpp" +#include "blockchain/impl/storage_util.hpp" +#include "jam_types/block.hpp" + +namespace jam::blockchain { + + BlockStorageInitializer::BlockStorageInitializer( + qtils::SharedRef logsys, + qtils::SharedRef storage, + qtils::SharedRef genesis_header, + qtils::SharedRef chain_spec, + qtils::SharedRef hasher) { + // temporary instance of block storage + BlockStorageImpl block_storage(std::move(logsys), storage, hasher, {}); + + auto genesis_block_hash = genesis_header->hash(); + + // Try to get genesis header from storage + auto genesis_block_existing_res = + block_storage.hasBlockHeader(genesis_block_hash); + if (genesis_block_existing_res.has_error()) { + block_storage.logger_->critical( + "Database error at check existing genesis block: {}", + genesis_block_existing_res.error()); + qtils::raise(genesis_block_existing_res.error()); + } + auto genesis_header_is_exist = genesis_block_existing_res.value(); + + if (not genesis_header_is_exist) { + // genesis block initialization + Block genesis_block{ + .header = *genesis_header, + }; + + auto res = block_storage.putBlock(genesis_block); + if (res.has_error()) { + block_storage.logger_->critical( + "Database error at store genesis block into: {}", res.error()); + qtils::raise(res.error()); + } + BOOST_ASSERT(genesis_block_hash == res.value()); + + auto assignment_res = + block_storage.assignHashToSlot(genesis_header->index()); + if (assignment_res.has_error()) { + block_storage.logger_->critical( + "Database error at assigning genesis block hash: {}", + assignment_res.error()); + qtils::raise(assignment_res.error()); + } + + auto sel_leaves_res = + block_storage.setBlockTreeLeaves({genesis_header->hash()}); + if (sel_leaves_res.has_error()) { + block_storage.logger_->critical( + "Database error at set genesis block as leaf: {}", + sel_leaves_res.error()); + qtils::raise(sel_leaves_res.error()); + } + + // TODO Save genesis state here + + block_storage.logger_->info("Genesis block {}, state {}", + genesis_block_hash, + genesis_header->parent_state_root); + } + } + +} // namespace jam::blockchain diff --git a/src/blockchain/impl/block_storage_initializer.hpp b/src/blockchain/impl/block_storage_initializer.hpp new file mode 100644 index 00000000..9d7e0fdf --- /dev/null +++ b/src/blockchain/impl/block_storage_initializer.hpp @@ -0,0 +1,40 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include + +namespace jam::log { + class LoggingSystem; +} +namespace jam::storage { + class SpacedStorage; +} +namespace jam::blockchain { + class GenesisBlockHeader; +} +namespace jam::app { + class ChainSpec; +} +namespace jam::crypto { + class Hasher; +} + +namespace jam::blockchain { + + class BlockStorageInitializer final : Singleton { + public: + BlockStorageInitializer(qtils::SharedRef logsys, + qtils::SharedRef storage, + qtils::SharedRef genesis_header, + qtils::SharedRef chain_spec, + qtils::SharedRef hasher); + }; + +} // namespace jam::blockchain diff --git a/src/blockchain/impl/storage_util.cpp b/src/blockchain/impl/storage_util.cpp new file mode 100644 index 00000000..eb61bc71 --- /dev/null +++ b/src/blockchain/impl/storage_util.cpp @@ -0,0 +1,81 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "blockchain/impl/storage_util.hpp" + +#include + +#include "storage/storage_error.hpp" + +using qtils::ByteVec; +// using jam::Hash256; +// using jam::primitives::BlockId; +// using jam::primitives::BlockNumber; +// using jam::storage::Space; + +namespace jam::blockchain { + + outcome::result> blockIdToBlockHash( + storage::SpacedStorage &storage, const BlockId &block_id) { + return visit_in_place( + block_id, + [&](const BlockNumber &block_number) + -> outcome::result> { + auto key_space = storage.getSpace(storage::Space::LookupKey); + return key_space->tryGet(slotToHashLookupKey(block_number)); + }, + [](const BlockHash &block_hash) { + return std::make_optional(ByteVec(block_hash)); + }); + } + + outcome::result> blockHashByNumber( + storage::SpacedStorage &storage, BlockNumber block_number) { + auto key_space = storage.getSpace(storage::Space::LookupKey); + OUTCOME_TRY(data_opt, key_space->tryGet(slotToHashLookupKey(block_number))); + if (data_opt.has_value()) { + OUTCOME_TRY(hash, BlockHash::fromSpan(data_opt.value())); + return hash; + } + return std::nullopt; + } + + outcome::result hasInSpace(storage::SpacedStorage &storage, + storage::Space space, + const BlockId &block_id) { + OUTCOME_TRY(key, blockIdToBlockHash(storage, block_id)); + if (not key.has_value()) { + return false; + } + + auto target_space = storage.getSpace(space); + return target_space->contains(key.value()); + } + + outcome::result putToSpace(storage::SpacedStorage &storage, + storage::Space space, + const BlockHash &block_hash, + qtils::ByteVecOrView &&value) { + auto target_space = storage.getSpace(space); + return target_space->put(block_hash, std::move(value)); + } + + outcome::result> getFromSpace( + storage::SpacedStorage &storage, + storage::Space space, + const BlockHash &block_hash) { + auto target_space = storage.getSpace(space); + return target_space->tryGet(block_hash); + } + + outcome::result removeFromSpace(storage::SpacedStorage &storage, + storage::Space space, + const BlockHash &block_hash) { + auto target_space = storage.getSpace(space); + return target_space->remove(block_hash); + } + +} // namespace jam::blockchain diff --git a/src/blockchain/impl/storage_util.hpp b/src/blockchain/impl/storage_util.hpp new file mode 100644 index 00000000..ad8d6588 --- /dev/null +++ b/src/blockchain/impl/storage_util.hpp @@ -0,0 +1,101 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include "jam_types/types.tmp.hpp" + +// #include "primitives/block_id.hpp" +#include "storage/spaced_storage.hpp" + +/** + * Storage schema overview + * + * A key-value approach is used for block storage. + * Different parts containing a block are stored in multiple storage spaces but + * have to be addressed with the same key. + * + * A key is the combination of block's number concatenated with its hash. Let's + * name it as NumHashKey. + * + * There is also an auxilary space named Space::kLookupKey where + * BlockId->NumHashKey mappings are stored. Effectively there are could be two + * types of mappings: either BlockNumber->NumHashKey or BlockHash->NumHashKey. + * Anyways, the resulting NumHashKey is good to be used for further manipulating + * with the Block in other storage spaces. + */ + +/** + * Auxiliary functions to simplify usage of persistant map based storage + * as a Blockchain storage + */ + +namespace jam::blockchain { + + /** + * Convert slot into a short lookup key (LE representation) + */ + inline qtils::ByteVec slotToHashLookupKey(TimeSlot slot) { + BOOST_STATIC_ASSERT(std::is_same_v); + return encode(slot).value(); + } + + /** + * Returns block hash by number if any + */ + outcome::result> blockHashByNumber( + storage::SpacedStorage &storage, BlockNumber block_number); + + /** + * Check if an entry is contained in the database + * @param storage - to get the entry from + * @param space - key space in the storage to which the entry belongs + * @param block_id - id of the block to get entry for + * @return true if the entry exists, false if does not, and error at fail + */ + outcome::result hasInSpace(storage::SpacedStorage &storage, + storage::Space space, + const BlockId &block_id); + + /** + * Put an entry to the key space \param space + * @param storage to put the entry to + * @param space keyspace for the entry value + * @param block_hash block hash that could be used to retrieve the value + * @param value data to be put to the storage + * @return storage error if any + */ + outcome::result putToSpace(storage::SpacedStorage &storage, + storage::Space space, + const BlockHash &block_hash, + qtils::ByteVecOrView &&value); + + /** + * Get an entry from the database + * @param storage - to get the entry from + * @param space - key space in the storage to which the entry belongs + * @param block_hash - hash of the block to get entry for + * @return error, or an encoded entry, if any, or std::nullopt, if none + */ + outcome::result> getFromSpace( + storage::SpacedStorage &storage, + storage::Space space, + const BlockHash &block_hash); + + /** + * Remove an entry from the key space \param space and corresponding lookup keys + * @param storage to put the entry to + * @param space keyspace for the entry value + * @param block_hash block hash that could be used to retrieve the value + * @return storage error if any + */ + outcome::result removeFromSpace(storage::SpacedStorage &storage, + storage::Space space, + const BlockHash &block_hash); + +} // namespace jam::blockchain diff --git a/src/injector/CMakeLists.txt b/src/injector/CMakeLists.txt index fd213f28..1d740867 100644 --- a/src/injector/CMakeLists.txt +++ b/src/injector/CMakeLists.txt @@ -20,4 +20,5 @@ target_link_libraries(node_injector se_async modules storage + blockchain ) diff --git a/src/injector/node_injector.cpp b/src/injector/node_injector.cpp index 5e289d8a..650fbf32 100644 --- a/src/injector/node_injector.cpp +++ b/src/injector/node_injector.cpp @@ -26,6 +26,7 @@ #include "app/impl/state_manager_impl.hpp" #include "app/impl/watchdog.hpp" #include "blockchain/impl/genesis_block_header_impl.hpp" +#include "blockchain/impl/block_storage_impl.hpp" #include "clock/impl/clock_impl.hpp" #include "crypto/hasher/hasher_impl.hpp" #include "injector/bind_by_lambda.hpp" @@ -59,13 +60,13 @@ namespace { Ts &&...args) { // clang-format off return di::make_injector( + di::bind.to(config), + di::bind.to(logsys), di::bind.to(), di::bind.to(), di::bind.to(), di::bind.to(), - di::bind. to(), - di::bind.to(config), - di::bind.to(logsys), + di::bind.to(), di::bind.to(), di::bind.to(), di::bind.to>(), @@ -82,6 +83,7 @@ namespace { di::bind.to(), di::bind.to(), di::bind.to(), + di::bind.to(), // user-defined overrides... std::forward(args)...); diff --git a/src/storage/predefined_keys.hpp b/src/storage/predefined_keys.hpp new file mode 100644 index 00000000..bd3325b4 --- /dev/null +++ b/src/storage/predefined_keys.hpp @@ -0,0 +1,19 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include + +namespace jam::storage { + + using qtils::literals::operator""_vec; + + inline const qtils::ByteVec kBlockTreeLeavesLookupKey = + ":jam:block_tree_leaves"_vec; + +} // namespace jam::storage diff --git a/src/storage/rocksdb/rocksdb_spaces.cpp b/src/storage/rocksdb/rocksdb_spaces.cpp index 337e13cd..c7b82478 100644 --- a/src/storage/rocksdb/rocksdb_spaces.cpp +++ b/src/storage/rocksdb/rocksdb_spaces.cpp @@ -16,9 +16,12 @@ namespace jam::storage { + // Names of non-default space static constexpr std::string_view kNamesArr[] = { - "" - // Add here names of non-default space + "lookup_key", + "header", + "extrinsic", + "justification", }; constexpr std::span kNames = kNamesArr; diff --git a/src/storage/spaces.hpp b/src/storage/spaces.hpp index dca4f5ec..98e96ac3 100644 --- a/src/storage/spaces.hpp +++ b/src/storage/spaces.hpp @@ -29,8 +29,12 @@ namespace jam::storage { */ enum class Space : uint8_t { Default = 0, ///< Default space used for general-purpose storage + LookupKey, ///< Space used for mapping lookup keys // application-defined spaces + Header, + Extrinsic, + Justification, // ... append here Total ///< Total number of defined spaces (must be last) diff --git a/tests/mock/blockchain/block_storage_mock.hpp b/tests/mock/blockchain/block_storage_mock.hpp new file mode 100644 index 00000000..c2921884 --- /dev/null +++ b/tests/mock/blockchain/block_storage_mock.hpp @@ -0,0 +1,117 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include "blockchain/block_storage.hpp" + +namespace jam::blockchain { + + class BlockStorageMock : public BlockStorage { + public: + MOCK_METHOD(outcome::result, + setBlockTreeLeaves, + (std::vector), + (override)); + + MOCK_METHOD(outcome::result>, + getBlockTreeLeaves, + (), + (const, override)); + + MOCK_METHOD(outcome::result, + getLastFinalized, + (), + (const, override)); + + MOCK_METHOD(outcome::result, + assignHashToSlot, + (const BlockInfo &), + (override)); + + MOCK_METHOD(outcome::result, + deassignHashToSlot, + (const BlockInfo &), + (override)); + + MOCK_METHOD(outcome::result>, + getBlockHash, + (TimeSlot), + (const, override)); + // MOCK_METHOD(outcome::result>, + // getBlockHash, + // (const BlockId &), + // (const, override)); + + MOCK_METHOD(outcome::result, + hasBlockHeader, + (const BlockHash &), + (const, override)); + + MOCK_METHOD(outcome::result, + putBlockHeader, + (const BlockHeader &), + (override)); + + MOCK_METHOD(outcome::result, + getBlockHeader, + (const BlockHash &), + (const, override)); + + MOCK_METHOD(outcome::result>, + tryGetBlockHeader, + (const BlockHash &), + (const, override)); + + MOCK_METHOD(outcome::result, + putBlockBody, + (const BlockHash &, const BlockBody &), + (override)); + + MOCK_METHOD(outcome::result>, + getBlockBody, + (const BlockHash &), + (const, override)); + + MOCK_METHOD(outcome::result, + removeBlockBody, + (const BlockHash &), + (override)); + + MOCK_METHOD(outcome::result, + putJustification, + (const Justification &, const BlockHash &), + (override)); + + MOCK_METHOD(outcome::result>, + getJustification, + (const BlockHash &), + (const, override)); + + MOCK_METHOD(outcome::result, + removeJustification, + (const BlockHash &), + (override)); + + MOCK_METHOD(outcome::result, + putBlock, + (const Block &), + (override)); + + MOCK_METHOD(outcome::result>, + getBlockData, + (const BlockHash &), + (const, override)); + + MOCK_METHOD(outcome::result, + removeBlock, + (const BlockHash &), + (override)); + }; + +} // namespace jam::blockchain From b2fbdc2081f31cd956e6f57363ebcadb93aa3b7e Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Tue, 17 Jun 2025 12:55:53 +0300 Subject: [PATCH 06/14] feature: chain spec mock --- tests/mock/app/chain_spec_mock.hpp | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 tests/mock/app/chain_spec_mock.hpp diff --git a/tests/mock/app/chain_spec_mock.hpp b/tests/mock/app/chain_spec_mock.hpp new file mode 100644 index 00000000..e1844fae --- /dev/null +++ b/tests/mock/app/chain_spec_mock.hpp @@ -0,0 +1,30 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include "app/chain_spec.hpp" + +namespace jam::app { + + class ChainSpecMock final : public ChainSpec { + public: + MOCK_METHOD(const std::string &, id, (), (const, override)); + + MOCK_METHOD(const std::vector &, + bootNodes, + (), + (const, override)); + + MOCK_METHOD(const qtils::ByteVec &, genesisHeader, (), (const, override)); + + using KVMap = std::map; + MOCK_METHOD(const KVMap &, genesisState, (), (const, override)); + }; + +} // namespace jam::app From 5365ab8f16ce05c9d8c1eb4e1a5ce9787ed10a53 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Thu, 22 May 2025 18:50:09 +0300 Subject: [PATCH 07/14] feature: block storage tests --- tests/mock/crypto/hasher_mock.hpp | 40 +++ tests/mock/storage/generic_storage_mock.hpp | 61 ++++ tests/mock/storage/spaced_storage_mock.hpp | 20 ++ tests/unit/CMakeLists.txt | 1 + tests/unit/blockchain/CMakeLists.txt | 14 + tests/unit/blockchain/block_storage_test.cpp | 275 +++++++++++++++++++ 6 files changed, 411 insertions(+) create mode 100644 tests/mock/crypto/hasher_mock.hpp create mode 100644 tests/mock/storage/generic_storage_mock.hpp create mode 100644 tests/mock/storage/spaced_storage_mock.hpp create mode 100644 tests/unit/blockchain/CMakeLists.txt create mode 100644 tests/unit/blockchain/block_storage_test.cpp diff --git a/tests/mock/crypto/hasher_mock.hpp b/tests/mock/crypto/hasher_mock.hpp new file mode 100644 index 00000000..1f0b6ce9 --- /dev/null +++ b/tests/mock/crypto/hasher_mock.hpp @@ -0,0 +1,40 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "crypto/hasher.hpp" +#include +#include + +namespace jam::crypto { + + class HasherMock : public Hasher { + public: + ~HasherMock() override = default; + + MOCK_METHOD(Hash64, twox_64, (qtils::ByteView), (const, override)); + + MOCK_METHOD(Hash64, blake2b_64, (qtils::ByteView), (const, override)); + + MOCK_METHOD(Hash128, blake2b_128, (qtils::ByteView), (const, override)); + + MOCK_METHOD(Hash128, twox_128, (qtils::ByteView), (const, override)); + + MOCK_METHOD(Hash256, twox_256, (qtils::ByteView), (const, override)); + + MOCK_METHOD(Hash256, blake2b_256, (qtils::ByteView), (const, override)); + + MOCK_METHOD(Hash256, blake2s_256, (qtils::ByteView), (const, override)); + + MOCK_METHOD(Hash256, keccak_256, (qtils::ByteView), (const, override)); + + MOCK_METHOD(Hash256, sha2_256, (qtils::ByteView), (const, override)); + + MOCK_METHOD(Hash512, blake2b_512, (qtils::ByteView), (const, override)); + }; + +} // namespace kagome::crypto diff --git a/tests/mock/storage/generic_storage_mock.hpp b/tests/mock/storage/generic_storage_mock.hpp new file mode 100644 index 00000000..87376c00 --- /dev/null +++ b/tests/mock/storage/generic_storage_mock.hpp @@ -0,0 +1,61 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include "storage/buffer_map_types.hpp" +#include "storage/face/generic_maps.hpp" + +namespace jam::storage::face { + + template + struct GenericStorageMock : public GenericStorage { + MOCK_METHOD0_T(batch, std::unique_ptr>()); + + MOCK_METHOD0_T(cursor, std::unique_ptr>()); + + MOCK_METHOD(outcome::result, getMock, (const View &), (const)); + + MOCK_METHOD(outcome::result>, + tryGetMock, + (const View &), + (const)); + + outcome::result> get(const View &key) const override { + OUTCOME_TRY(value, getMock(key)); + return value; + } + + outcome::result>> tryGet( + const View &key) const override { + OUTCOME_TRY(value, tryGetMock(key)); + if (value) { + return std::move(*value); + } + return std::nullopt; + } + + MOCK_CONST_METHOD1_T(contains, outcome::result(const View &)); + + MOCK_CONST_METHOD0_T(empty, bool()); + + MOCK_METHOD(outcome::result, put, (const View &, const V &)); + outcome::result put(const View &k, OwnedOrView &&v) override { + return put(k, v.mut()); + } + + MOCK_METHOD1_T(remove, outcome::result(const View &)); + + MOCK_CONST_METHOD0_T(size, size_t()); + }; + +} // namespace kagome::storage::face + +namespace jam::storage { + using BufferStorageMock = face::GenericStorageMock; +} // namespace kagome::storage diff --git a/tests/mock/storage/spaced_storage_mock.hpp b/tests/mock/storage/spaced_storage_mock.hpp new file mode 100644 index 00000000..afeedad2 --- /dev/null +++ b/tests/mock/storage/spaced_storage_mock.hpp @@ -0,0 +1,20 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include "storage/spaced_storage.hpp" + +namespace jam::storage { + + class SpacedStorageMock : public SpacedStorage { + public: + MOCK_METHOD(std::shared_ptr, getSpace, (Space), (override)); + }; + +} // namespace kagome::storage diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 64282f03..f239f557 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -5,4 +5,5 @@ # add_subdirectory(app) +add_subdirectory(blockchain) add_subdirectory(storage) diff --git a/tests/unit/blockchain/CMakeLists.txt b/tests/unit/blockchain/CMakeLists.txt new file mode 100644 index 00000000..ed52e570 --- /dev/null +++ b/tests/unit/blockchain/CMakeLists.txt @@ -0,0 +1,14 @@ +# +# Copyright Quadrivium LLC +# All Rights Reserved +# SPDX-License-Identifier: Apache-2.0 +# + +addtest(block_storage_test + block_storage_test.cpp + ) +target_link_libraries(block_storage_test + blockchain #??? + logger_for_tests + storage + ) \ No newline at end of file diff --git a/tests/unit/blockchain/block_storage_test.cpp b/tests/unit/blockchain/block_storage_test.cpp new file mode 100644 index 00000000..4319ddb2 --- /dev/null +++ b/tests/unit/blockchain/block_storage_test.cpp @@ -0,0 +1,275 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +#include +#include + +#include "blockchain/block_storage_error.hpp" +#include "blockchain/impl/block_storage_impl.hpp" +#include "blockchain/impl/block_storage_initializer.hpp" +#include "mock/app/chain_spec_mock.hpp" +#include "mock/crypto/hasher_mock.hpp" +#include "mock/storage/generic_storage_mock.hpp" +#include "mock/storage/spaced_storage_mock.hpp" +#include "scale/jam_scale.hpp" +#include "storage/storage_error.hpp" +// #include "testutil/literals.hpp" +#include +#include + +#include "qtils/error_throw.hpp" +#include "testutil/prepare_loggers.hpp" +#include "testutil/literals.hpp" + +using jam::Block; +using jam::BlockBody; +using jam::BlockData; +using jam::BlockHash; +using jam::BlockHeader; +using jam::BlockNumber; +using jam::encode; +using jam::app::ChainSpecMock; +using jam::blockchain::BlockStorageError; +using jam::blockchain::BlockStorageImpl; +using jam::blockchain::BlockStorageInitializer; +using jam::blockchain::GenesisBlockHeader; +using jam::crypto::HasherMock; +using jam::storage::BufferStorageMock; +using jam::storage::Space; +using jam::storage::SpacedStorageMock; +using qtils::ByteVec; +using qtils::ByteView; +using testing::_; +using testing::Ref; +using testing::Return; + +class BlockStorageTest : public testing::Test { + public: + static void SetUpTestCase() { + testutil::prepareLoggers(); + } + + void SetUp() override { + genesis_header->hash_opt = genesis_block_hash; + + hasher = std::make_shared(); + spaced_storage = std::make_shared(); + + std::set required_spaces = {Space::Default, + Space::Header, + Space::Justification, + Space::Extrinsic, + Space::LookupKey}; + + for (auto space : required_spaces) { + auto storage = std::make_shared(); + spaces[space] = storage; + EXPECT_CALL(*spaced_storage, getSpace(space)) + .WillRepeatedly(Return(storage)); + + EXPECT_CALL(*storage, put(_, _)) + .WillRepeatedly(Return(outcome::success())); + EXPECT_CALL(*storage, tryGetMock(_)).WillRepeatedly(Return(std::nullopt)); + } + } + + BlockHash genesis_block_hash{"genesis"_arr32}; + BlockHash regular_block_hash{"regular"_arr32}; + BlockHash unhappy_block_hash{"unhappy"_arr32}; + + qtils::SharedRef logsys = testutil::prepareLoggers(); + + qtils::SharedRef genesis_header = + std::make_shared(); + qtils::SharedRef chain_spec = + std::make_shared(); + qtils::SharedRef hasher = std::make_shared(); + qtils::SharedRef spaced_storage = + std::make_shared(); + std::map> spaces; + + qtils::SharedRef createWithGenesis() { + // calculate hash of genesis block at put block header + static auto encoded_header = ByteVec(encode(BlockHeader{}).value()); + ON_CALL(*hasher, blake2b_256(encoded_header.view())) + .WillByDefault(Return(genesis_block_hash)); + + auto new_block_storage = std::make_shared( + logsys, spaced_storage, hasher, nullptr); + + return new_block_storage; + } +}; + +/** + * @given a hasher instance, a genesis block, and an empty map storage + * @when initialising a block storage from it + * @then initialisation will successful + */ +TEST_F(BlockStorageTest, CreateWithGenesis) { + createWithGenesis(); +} + +/** + * @given a hasher instance and an empty map storage + * @when trying to initialise a block storage from it and storage throws an + * error + * @then storage will be initialized by genesis block + */ +TEST_F(BlockStorageTest, CreateWithEmptyStorage) { + auto empty_storage = std::make_shared(); + + // check if storage contained genesis block + EXPECT_CALL(*empty_storage, tryGetMock(_)) + .WillRepeatedly(Return(std::nullopt)); + + // put genesis block into storage + EXPECT_CALL(*empty_storage, put(_, _)) + .WillRepeatedly(Return(outcome::success())); + + ASSERT_NO_THROW(BlockStorageImpl x(logsys, spaced_storage, hasher, {})); +} + +/** + * @given a hasher instance, a genesis block, and an map storage containing the + * block + * @when initialising a block storage from it + * @then initialisation will fail because the genesis block is already at the + * underlying storage (which is actually supposed to be empty) + */ +TEST_F(BlockStorageTest, CreateWithExistingGenesis) { + // trying to get header of genesis block + EXPECT_CALL(*(spaces[Space::Header]), contains(ByteView{genesis_block_hash})) + .WillOnce(Return(outcome::success(true))); + + // Init underlying storage + ASSERT_NO_THROW(BlockStorageInitializer( + logsys, spaced_storage, genesis_header, chain_spec, hasher)); + + // Create block storage + ASSERT_NO_THROW(BlockStorageImpl(logsys, spaced_storage, hasher, {})); +} + +/** + * @given a hasher instance, a genesis block, and an map storage containing the + * block + * @when initialising a block storage from it and storage throws an error + * @then initialisation will fail + */ +TEST_F(BlockStorageTest, CreateWithStorageError) { + // trying to get header of genesis block + EXPECT_CALL(*(spaces[Space::Header]), contains(ByteView{genesis_block_hash})) + .WillOnce(Return(jam::storage::StorageError::IO_ERROR)); + + // Init underlying storage + EXPECT_THROW_OUTCOME( + BlockStorageInitializer( + logsys, spaced_storage, genesis_header, chain_spec, hasher), + jam::storage::StorageError::IO_ERROR); +} + +/** + * @given a block storage and a block that is not in storage yet + * @when putting a block in the storage + * @then block is successfully put + */ +TEST_F(BlockStorageTest, PutBlock) { + auto block_storage = createWithGenesis(); + + Block block; + block.header.slot = 1; + block.header.parent = genesis_block_hash; + + ASSERT_OUTCOME_SUCCESS(block_storage->putBlock(block)); +} + +/* + * @given a block storage and a block that is not in storage yet + * @when trying to get a block from the storage + * @then an error is returned + */ +TEST_F(BlockStorageTest, GetBlockNotFound) { + std::shared_ptr block_storage; + ASSERT_NO_THROW(block_storage = std::make_shared( + logsys, spaced_storage, hasher, nullptr)); + + EXPECT_OUTCOME_ERROR(get_res, + block_storage->getBlockHeader(genesis_block_hash), + BlockStorageError::HEADER_NOT_FOUND); +} + +/* + * @given a block storage and a block that is not in storage yet + * @when trying to get a block from the storage + * @then success value containing nullopt is returned + */ +TEST_F(BlockStorageTest, TryGetBlockNotFound) { + std::shared_ptr block_storage; + ASSERT_NO_THROW(block_storage = std::make_shared( + logsys, spaced_storage, hasher, nullptr)); + + ASSERT_OUTCOME_SUCCESS(try_get_res, + block_storage->tryGetBlockHeader(genesis_block_hash)); + ASSERT_FALSE(try_get_res.has_value()); +} + +/** + * @given a block storage and a block that is not in storage yet + * @when putting a block in the storage and underlying storage throws an + * error + * @then block is not put and error is returned + */ +TEST_F(BlockStorageTest, PutWithStorageError) { + auto block_storage = createWithGenesis(); + + Block block; + block.header.slot = 1; + block.header.parent = genesis_block_hash; + + auto encoded_header = ByteVec(encode(block.header).value()); + ON_CALL(*hasher, blake2b_256(encoded_header.view())) + .WillByDefault(Return(regular_block_hash)); + + ByteVec key{regular_block_hash}; + + EXPECT_CALL(*(spaces[Space::Extrinsic]), put(key.view(), _)) + .WillOnce(Return(jam::storage::StorageError::IO_ERROR)); + + ASSERT_OUTCOME_ERROR(block_storage->putBlock(block), + jam::storage::StorageError::IO_ERROR); +} + +/** + * @given a block storage + * @when removing a block from it + * @then block is successfully removed if no error occurs in the underlying + * storage, an error is returned otherwise + */ +TEST_F(BlockStorageTest, Remove) { + auto block_storage = createWithGenesis(); + + ByteView hash(genesis_block_hash); + + ByteVec encoded_header{encode(BlockHeader{}).value()}; + + EXPECT_CALL(*(spaces[Space::Header]), tryGetMock(hash)) + .WillOnce(Return(encoded_header)); + EXPECT_CALL(*(spaces[Space::Extrinsic]), remove(hash)) + .WillOnce(Return(outcome::success())); + EXPECT_CALL(*(spaces[Space::Header]), remove(hash)) + .WillOnce(Return(outcome::success())); + EXPECT_CALL(*(spaces[Space::Justification]), remove(hash)) + .WillOnce(Return(outcome::success())); + + ASSERT_OUTCOME_SUCCESS(block_storage->removeBlock(genesis_block_hash)); + + EXPECT_CALL(*(spaces[Space::Header]), tryGetMock(hash)) + .WillOnce(Return(std::nullopt)); + + ASSERT_OUTCOME_SUCCESS(block_storage->removeBlock(genesis_block_hash)); +} From 19bed71d029cf37fc3d0e276c16b1ab0812aadb5 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Mon, 16 Jun 2025 01:05:48 +0300 Subject: [PATCH 08/14] feature: unsafe methods of safe object --- src/se/impl/common.hpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/se/impl/common.hpp b/src/se/impl/common.hpp index 4dc0145a..c2aafb53 100644 --- a/src/se/impl/common.hpp +++ b/src/se/impl/common.hpp @@ -118,6 +118,14 @@ namespace jam::se::utils { return std::forward(f)(t_); } + T &unsafeGet() { + return t_; + } + + const T &unsafeGet() const { + return t_; + } + private: T t_; ///< The wrapped object mutable M cs_; ///< Mutex for synchronization From 7d8a8bd3a502999f15fec65514e4039c0b76f4c3 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Fri, 23 May 2025 01:57:25 +0300 Subject: [PATCH 09/14] feature: block tree --- example/config.yaml | 3 + src/blockchain/CMakeLists.txt | 8 +- src/blockchain/block_header_repository.hpp | 99 ++ src/blockchain/block_tree.hpp | 265 ++++ src/blockchain/block_tree_error.hpp | 45 + src/blockchain/impl/block_tree_error.cpp | 45 + src/blockchain/impl/block_tree_impl.cpp | 1210 +++++++++++++++++ src/blockchain/impl/block_tree_impl.hpp | 282 ++++ .../impl/block_tree_initializer.cpp | 270 ++++ .../impl/block_tree_initializer.hpp | 36 + src/blockchain/impl/cached_tree.cpp | 342 +++++ src/blockchain/impl/cached_tree.hpp | 119 ++ .../impl/justification_storage_policy.cpp | 35 + .../impl/justification_storage_policy.hpp | 35 + src/injector/node_injector.cpp | 4 + src/se/subscription_fwd.hpp | 13 +- tests/unit/blockchain/block_tree_test.cpp | 960 +++++++++++++ 17 files changed, 3768 insertions(+), 3 deletions(-) create mode 100644 src/blockchain/block_header_repository.hpp create mode 100644 src/blockchain/block_tree.hpp create mode 100644 src/blockchain/block_tree_error.hpp create mode 100644 src/blockchain/impl/block_tree_error.cpp create mode 100644 src/blockchain/impl/block_tree_impl.cpp create mode 100644 src/blockchain/impl/block_tree_impl.hpp create mode 100644 src/blockchain/impl/block_tree_initializer.cpp create mode 100644 src/blockchain/impl/block_tree_initializer.hpp create mode 100644 src/blockchain/impl/cached_tree.cpp create mode 100644 src/blockchain/impl/cached_tree.hpp create mode 100644 src/blockchain/impl/justification_storage_policy.cpp create mode 100644 src/blockchain/impl/justification_storage_policy.hpp create mode 100644 tests/unit/blockchain/block_tree_test.cpp diff --git a/example/config.yaml b/example/config.yaml index 390c3835..55b3dc69 100644 --- a/example/config.yaml +++ b/example/config.yaml @@ -42,3 +42,6 @@ logging: - name: storage children: - name: block_storage + - name: blockchain + children: + - name: block_tree diff --git a/src/blockchain/CMakeLists.txt b/src/blockchain/CMakeLists.txt index 7588bb63..a38bb6ad 100644 --- a/src/blockchain/CMakeLists.txt +++ b/src/blockchain/CMakeLists.txt @@ -6,9 +6,15 @@ add_library(blockchain impl/storage_util.cpp - impl/block_storage_impl.cpp impl/block_storage_error.cpp + impl/block_storage_impl.cpp impl/block_storage_initializer.cpp + impl/genesis_block_header_impl.cpp + impl/block_tree_error.cpp + impl/justification_storage_policy.cpp + impl/cached_tree.cpp + impl/block_tree_impl.cpp + impl/block_tree_initializer.cpp ) target_link_libraries(blockchain Boost::boost diff --git a/src/blockchain/block_header_repository.hpp b/src/blockchain/block_header_repository.hpp new file mode 100644 index 00000000..b5df7fb9 --- /dev/null +++ b/src/blockchain/block_header_repository.hpp @@ -0,0 +1,99 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include +// #include "common/visitor.hpp" +// #include +#include "jam_types/block_header.hpp" +// #include "primitives/block_id.hpp" + +namespace jam::blockchain { + + /** + * Status of a block + */ + enum class BlockStatus : uint8_t { + InChain, + Unknown, + }; + + /** + * An interface to a storage with block headers that provides several + * convenience methods, such as getting bloch number by its hash and vice + * versa or getting a block status + */ + class BlockHeaderRepository { + public: + virtual ~BlockHeaderRepository() = default; + + /** + * @return the number of the block with the provided {@param block_hash} + * in case one is in the storage or an error + */ + virtual outcome::result getNumberByHash( + const BlockHash &block_hash) const = 0; + + // /** + // * @param block_number - the number of a block, contained in a block + // header + // * @return the hash of the block with the provided number in case one is + // * in the storage or an error + // */ + // virtual outcome::result getHashByNumber( + // BlockNumber block_number) const = 0; + + /** + * @return block header with corresponding {@param block_hash} or an error + */ + [[nodiscard]] virtual outcome::result getBlockHeader( + const BlockHash &block_hash) const = 0; + + // /** + // * @return block header with corresponding {@param block_hash} or a none + // * optional if the corresponding block header is not in storage or a + // * storage error + // */ + // virtual outcome::result> + // tryGetBlockHeader(const BlockHash &block_hash) const = 0; + + // /** + // * @param id of a block which number is returned + // * @return block number or a none optional if the corresponding block + // * header is not in storage or a storage error + // */ + // outcome::result getNumberById( + // const BlockId &block_id) const { + // return visit_in_place( + // block_id, + // [](const BlockNumber &block_number) { + // return block_number; + // }, + // [this](const BlockHash &block_hash) { + // return getNumberByHash(block_hash); + // }); + // } + // + // /** + // * @param id of a block which hash is returned + // * @return block hash or a none optional if the corresponding block + // * header is not in storage or a storage error + // */ + // outcome::result getHashById( + // const BlockId &id) const { + // return visit_in_place( + // id, + // [this](const BlockNumber &n) { + // return getHashByNumber(n); + // }, + // [](const BlockHash &hash) { return hash; }); + // } + }; + +} // namespace jam::blockchain diff --git a/src/blockchain/block_tree.hpp b/src/blockchain/block_tree.hpp new file mode 100644 index 00000000..ffd82b1b --- /dev/null +++ b/src/blockchain/block_tree.hpp @@ -0,0 +1,265 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +// #include +// #include +// #include +// +#include "blockchain/block_header_repository.hpp" +// #include "consensus/timeline/types.hpp" +// #include +// #include "primitives/block.hpp" +// #include "primitives/block_id.hpp" +// #include "primitives/common.hpp" +// #include "primitives/justification.hpp" +// #include "primitives/version.hpp" + +#include "jam_types/block.hpp" +#include "jam_types/justification.hpp" +#include "jam_types/types.tmp.hpp" + +namespace jam::blockchain { + /** + * Storage for blocks, which has a form of tree; it serves two functions: + * - keep tracking of all finalized blocks (they are kept in the + * non-volatile storage) + * - work with blocks, which participate in the current round of block + * production (handling forks, pruning the blocks, resolving child-parent + * relations, etc) + */ + class BlockTree : public BlockHeaderRepository { + public: + virtual ~BlockTree() = default; + + /** + * @returns hash of genesis block + */ + [[nodiscard]] virtual const BlockHash &getGenesisBlockHash() const = 0; + + // /** + // * Get block hash by provided block number + // * @param block_number of the block header we are looking for + // * @return result containing block hash if it exists, error otherwise + // */ + // virtual outcome::result> + // getBlockHash( + // BlockNumber block_number) const = 0; + + /** + * Checks containing of block header by provided block id + * @param hash hash of the block header we are checking + * @return containing block header or does not, or error + */ + [[nodiscard]] virtual bool has(const BlockHash &hash) const = 0; + + /** + * Get a body (extrinsics) of the block (if present) + * @param block_hash hash of the block to get body for + * @return body, if the block exists in our storage, error in case it + does + * not exist in our storage, or actual error happens + */ + virtual outcome::result getBlockBody( + const BlockHash &block_hash) const = 0; + + // /** + // * Get a justification of the block (if present) + // * @param block_hash hash of the block to get justification for + // * @return body, if the block exists in our storage, error in case it + // does + // * not exist in our storage, or actual error happens + // */ + // virtual outcome::result getBlockJustification( + // const BlockHash &block_hash) const = 0; + + /** + * Adds header to the storage + * @param header that we are adding + * @return result with success if header's parent exists on storage and new + * header was added. Error otherwise + */ + virtual outcome::result addBlockHeader(const BlockHeader &header) = 0; + + /** + * Adds block body to the storage + * @param block_number that corresponds to the block which body we are + * adding + * @param block_hash that corresponds to the block which body we are + adding + * @param block_body that we are adding + * @return result with success if block body was inserted. Error + otherwise + */ + virtual outcome::result addBlockBody(const BlockHash &block_hash, + const BlockBody &block_body) = 0; + + /** + * Add an existent block to the tree + * @param block_hash is hash of the added block in the tree + * @param block_header is header of that block + * @return nothing or error; if error happens, no changes in the tree are + * made + */ + virtual outcome::result addExistingBlock( + const BlockHash &block_hash, const BlockHeader &block_header) = 0; + + // /** + // * Adjusts weight for the block as contained parachain data. + // * @param block_hash is hash of the weighted block + // */ + // virtual outcome::result markAsParachainDataBlock( + // const BlockHash &block_hash) = 0; + + /** + * The passed blocks will be marked as reverted, and their descendants will + * be marked as non-viable + * @param block_hashes is vector of reverted block hashes + */ + virtual outcome::result markAsRevertedBlocks( + const std::vector &block_hashes) = 0; + + /** + * Add a new block to the tree + * @param block to be stored and added to tree + * @return nothing or error; if error happens, no changes in the tree are + * made + * + * @note if block, which is specified in PARENT_HASH field of (\param block) + * is not in our local storage, corresponding error is returned. It is + * suggested that after getting that error, the caller would ask another + * peer for the parent block and try to insert it; this operation is to be + * repeated until a successful insertion happens + */ + virtual outcome::result addBlock(const Block &block) = 0; + + /** + * Remove leaf + * @param block_hash - hash of block to be deleted. The block must be leaf. + * @return nothing or error + */ + virtual outcome::result removeLeaf(const BlockHash &block_hash) = 0; + + /** + * Mark the block as finalized and store a finalization justification + * @param block to be finalized + * @param justification of the finalization + * @return nothing or error + */ + virtual outcome::result finalize( + const BlockHash &block, const Justification &justification) = 0; + + // enum class GetChainDirection : uint8_t { + // ASCEND, + // DESCEND, + // }; + + /** + * Get a chain of blocks from provided block to direction of the best block + * @param block, from which the chain is started + * @param maximum number of blocks to be retrieved + * @return chain or blocks or error + */ + virtual outcome::result> getBestChainFromBlock( + const BlockHash &block, uint64_t maximum) const = 0; + + /** + * Get a chain of blocks before provided block including its + * @param block, to which the chain is ended + * @param maximum number of blocks to be retrieved + * @return chain or blocks or error + */ + virtual outcome::result> getDescendingChainToBlock( + const BlockHash &block, uint64_t maximum) const = 0; + + // /** + // * Get a chain of blocks. + // * Implies `hasDirectChain(ancestor, descendant)`. + // * @param ancestor - block, which is closest to the genesis + // * @param descendant - block, which is farthest from the genesis + // * @return chain of blocks in ascending order or error + // */ + // virtual outcome::result> getChainByBlocks( + // const BlockHash &ancestor, + // const BlockHash &descendant) const = 0; + // + // /** + // * Check if one block is ancestor of second one (direct chain exists) + // * @param ancestor - block, which is closest to the genesis + // * @param descendant - block, which is farthest from the genesis + // * @return true if \param ancestor is ancestor of \param descendant + // */ + // virtual bool hasDirectChain( + // const BlockHash &ancestor, + // const BlockHash &descendant) const = 0; + // + // bool hasDirectChain(const BlockInfo &ancestor, + // const BlockInfo &descendant) const { + // return hasDirectChain(ancestor.hash, descendant.hash); + // } + // + // virtual bool isFinalized(const BlockInfo &block) const = 0; + + /** + * Get a best leaf of the tree + * @return best leaf + * + * @note best block is also a result of "SelectBestChain": if we are the + * leader, we connect a block, which we constructed, to that best block + */ + virtual BlockInfo bestBlock() const = 0; + + /** + * @brief Get the most recent block of the best (longest) chain among + * those that contain a block with \param target_hash + * @param target_hash is a hash of a block that the chosen chain must + * contain + * @param max_number is the max block number that the resulting block + (and + * the target one) may possess + */ + virtual outcome::result getBestContaining( + const BlockHash &target_hash) const = 0; + + /** + * Get all leaves of our tree + * @return collection of the leaves + */ + virtual std::vector getLeaves() const = 0; + // virtual std::vector getLeavesInfo() const = 0; + + /** + * Get children of the block with specified hash + * @param block to get children of + * @return collection of children hashes or error + */ + virtual outcome::result> getChildren( + const BlockHash &block) const = 0; + + /** + * Get the last finalized block + * @return hash of the block + */ + virtual BlockInfo getLastFinalized() const = 0; + + // /** + // * Warp synced to block. + // */ + // virtual void warp(const BlockInfo &block) = 0; + + /** + * Notify best and finalized block to subscriptions. + */ + virtual void notifyBestAndFinalized() = 0; + + /** + * Used when switching from fast-sync to full-sync. + */ + virtual void removeUnfinalized() = 0; + }; + +} // namespace jam::blockchain diff --git a/src/blockchain/block_tree_error.hpp b/src/blockchain/block_tree_error.hpp new file mode 100644 index 00000000..546ee0e4 --- /dev/null +++ b/src/blockchain/block_tree_error.hpp @@ -0,0 +1,45 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +namespace jam::blockchain { + + /** + * Errors of the block tree are here, so that other modules can use them, for + * example, to compare a received error with those + */ + enum class BlockTreeError : uint8_t { + NO_PARENT = 1, + BLOCK_EXISTS, + // target block number is past the given maximum number + TARGET_IS_PAST_MAX, + // block resides on a dead fork + BLOCK_ON_DEAD_END, + // block exists in chain but not found when following all leaves backwards + EXISTING_BLOCK_NOT_FOUND, + // non-finalized block is not found + NON_FINALIZED_BLOCK_NOT_FOUND, + // justification is not found in block storage + JUSTIFICATION_NOT_FOUND, + // block body is not found in block storage + BODY_NOT_FOUND, + // block header is not found in block storage + HEADER_NOT_FOUND, + // some block in the requested chain is missing + SOME_BLOCK_IN_CHAIN_NOT_FOUND, + // block is not a leaf + BLOCK_IS_NOT_LEAF, + BLOCK_NOT_EXISTS, + BLOCK_TREE_CORRUPTED, + WRONG_WORKFLOW, + }; + +} // namespace jam::blockchain + +OUTCOME_HPP_DECLARE_ERROR(jam::blockchain, BlockTreeError) diff --git a/src/blockchain/impl/block_tree_error.cpp b/src/blockchain/impl/block_tree_error.cpp new file mode 100644 index 00000000..a433e53d --- /dev/null +++ b/src/blockchain/impl/block_tree_error.cpp @@ -0,0 +1,45 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "blockchain/block_tree_error.hpp" + +OUTCOME_CPP_DEFINE_CATEGORY(jam::blockchain, BlockTreeError, e) { + using E = jam::blockchain::BlockTreeError; + switch (e) { + case E::NO_PARENT: + return "block, which should have been added, has no known parent"; + case E::BLOCK_EXISTS: + return "block, which should have been inserted, already exists in the " + "tree"; + case E::SOME_BLOCK_IN_CHAIN_NOT_FOUND: + return "one of the blocks for getting the chain was not found in the " + "local storage"; + case E::TARGET_IS_PAST_MAX: + return "target block number is past the given maximum number"; + case E::BLOCK_ON_DEAD_END: + return "block resides on a dead fork"; + case E::EXISTING_BLOCK_NOT_FOUND: + return "block exists in chain but not found when following all leaves " + "backwards"; + case E::NON_FINALIZED_BLOCK_NOT_FOUND: + return "a non-finalized block is not found"; + case E::JUSTIFICATION_NOT_FOUND: + return "the requested justification is not found in block storage"; + case E::HEADER_NOT_FOUND: + return "the requested block header is not found in block storage"; + case E::BODY_NOT_FOUND: + return "the requested block body is not found in block storage"; + case E::BLOCK_IS_NOT_LEAF: + return "the target block is not a leaf"; + case E::BLOCK_NOT_EXISTS: + return "target block doesn't exist"; + case E::BLOCK_TREE_CORRUPTED: + return "block tree corrupted"; + case E::WRONG_WORKFLOW: + return "trying to use block tree initializing data second time"; + } + return "unknown error"; +} diff --git a/src/blockchain/impl/block_tree_impl.cpp b/src/blockchain/impl/block_tree_impl.cpp new file mode 100644 index 00000000..d2ae868a --- /dev/null +++ b/src/blockchain/impl/block_tree_impl.cpp @@ -0,0 +1,1210 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "blockchain/impl/block_tree_impl.hpp" + +// #include +// #include +// #include +// #include + +#include + +#include "blockchain/block_tree_error.hpp" +#include "blockchain/impl/block_tree_initializer.hpp" +#include "blockchain/impl/cached_tree.hpp" +#include "blockchain/impl/justification_storage_policy.hpp" +#include "se/subscription.hpp" +#include "se/subscription_fwd.hpp" +// #include "blockchain/impl/storage_util.hpp" +// #include "common/main_thread_pool.hpp" +// #include "consensus/babe/impl/babe_digests_util.hpp" +// #include "consensus/babe/is_primary.hpp" +// #include "crypto/blake2/blake2b.h" +// #include "log/profiling_logger.hpp" +// #include "storage/database_error.hpp" +// #include "storage/trie_pruner/trie_pruner.hpp" +// #include "utils/pool_handler.hpp" +#include "tests/testutil/literals.hpp" + +// namespace { +// constexpr auto blockHeightMetricName = "jam_block_height"; +// constexpr auto knownChainLeavesMetricName = "jam_number_leaves"; +// } // namespace + +namespace jam::blockchain { + // using StorageError = jam::storage::StorageError; + + namespace { + // Check if block produced by ticket or fallback way + bool isPrimary(const BlockHeader &header) { + // Check the source of seal-keys (γs) + // - Z(γa) - by ticket + // - F(...) - fallback + + // FIXME calculate real result + return header.extrinsic_hash == "ticket"_arr32; + } + } // namespace + + + BlockTreeImpl::SafeBlockTreeData::SafeBlockTreeData(BlockTreeData data) + : block_tree_data_{std::move(data)} {} + + // outcome::result BlockTreeImpl::recover( + // const BlockId &target_block_id, + // std::shared_ptr storage, + // std::shared_ptr trie_storage, + // std::shared_ptr block_tree) { + // BOOST_ASSERT(storage != nullptr); + // BOOST_ASSERT(trie_storage != nullptr); + // + // log::Logger log = log::createLogger("BlockTree", "block_tree"); + // + // OUTCOME_TRY(block_tree_leaves, loadLeaves(storage, log)); + // + // BOOST_ASSERT_MSG(not block_tree_leaves.empty(), + // "Must be known or calculated at least one leaf"); + // + // auto target_block_hash_opt_res = storage->getBlockHash(target_block_id); + // if (target_block_hash_opt_res.has_failure()) { + // SL_CRITICAL(log, + // "Can't get header of target block: {}", + // target_block_hash_opt_res.error()); + // return BlockTreeError::HEADER_NOT_FOUND; + // } + // if (not target_block_hash_opt_res.value().has_value()) { + // SL_CRITICAL(log, "Can't get header of target block: header not found"); + // return BlockTreeError::HEADER_NOT_FOUND; + // } + // const auto &target_block_hash = + // target_block_hash_opt_res.value().value(); + // + // // Check if target block exists + // auto target_block_header_res = + // storage->getBlockHeader(target_block_hash); if + // (target_block_header_res.has_error()) { + // SL_CRITICAL(log, + // "Can't get header of target block: {}", + // target_block_header_res.error()); + // return target_block_header_res.as_failure(); + // } + // + // const auto &target_block_header = target_block_header_res.value(); + // const auto &state_root = target_block_header.state_root; + // + // // Check if target block has state + // if (auto res = trie_storage->getEphemeralBatchAt(state_root); + // res.has_error()) { + // SL_WARN(log, "Can't get state of target block: {}", res.error()); + // SL_CRITICAL( + // log, + // "You will need to use `--sync Fast' CLI arg the next time you + // start"); + // } + // + // for (auto it = block_tree_leaves.rbegin(); it != + // block_tree_leaves.rend(); + // it = block_tree_leaves.rbegin()) { + // auto block = *it; + // if (target_block_header.number >= block.number) { + // break; + // } + // + // auto header_res = storage->getBlockHeader(block.hash); + // if (header_res.has_error()) { + // SL_CRITICAL(log, + // "Can't get header of one of removing block: {}", + // header_res.error()); + // return header_res.as_failure(); + // } + // + // const auto &header = header_res.value(); + // block_tree_leaves.emplace(*header.parentInfo()); + // block_tree_leaves.erase(block); + // + // std::vector leaves; + // leaves.reserve(block_tree_leaves.size()); + // + // std::ranges::transform(block_tree_leaves, + // std::back_inserter(leaves), + // [](const auto it) { return it.hash; }); + // if (auto res = storage->setBlockTreeLeaves(leaves); res.has_error()) { + // SL_CRITICAL( + // log, "Can't save updated block tree leaves: {}", res.error()); + // return res.as_failure(); + // } + // + // if (auto res = block_tree->removeLeaf(block.hash); res.has_error()) { + // SL_CRITICAL(log, "Can't remove block {}: {}", block, res.error()); + // return res.as_failure(); + // } + // } + // + // return outcome::success(); + // } + + BlockTreeImpl::BlockTreeImpl( + qtils::SharedRef logsys, + qtils::SharedRef app_config, + qtils::SharedRef storage, + // const BlockInfo &finalized, + qtils::SharedRef hasher, + // primitives::events::ChainSubscriptionEnginePtr chain_events_engine, + // primitives::events::ExtrinsicSubscriptionEnginePtr + // extrinsic_events_engine, + // std::shared_ptr + // extrinsic_event_key_repo, + std::shared_ptr + justification_storage_policy, + // std::shared_ptr state_pruner, + // common::MainThreadPool &main_thread_pool, + std::shared_ptr se_manager, + qtils::SharedRef initializer) + : log_(logsys->getLogger("BlockTree", "block_tree")), + se_manager_(std::move(se_manager)), + block_tree_data_{{ + .storage_ = std::move(storage), + // .state_pruner_ = std::move(state_pruner), + .tree_ = std::make_unique( + std::get<0>(initializer->nonFinalizedSubTree())), + .hasher_ = std::move(hasher), + // .extrinsic_event_key_repo_ = + // std::move(extrinsic_event_key_repo), + .justification_storage_policy_ = + std::move(justification_storage_policy), + // .blocks_pruning_ = {app_config.blocksPruning(), + // finalized.number}, + }} // + // chain_events_engine_{std::move(chain_events_engine)}, + // main_pool_handler_{main_thread_pool.handlerStarted()}, + // extrinsic_events_engine_{std::move(extrinsic_events_engine)} + { + block_tree_data_.sharedAccess([&](const BlockTreeData &p) { + // // Register metrics + // metrics_registry_->registerGaugeFamily(blockHeightMetricName, + // "Block height info of the chain"); + // + // metric_best_block_height_ = metrics_registry_->registerGaugeMetric( + // blockHeightMetricName, {{"status", "best"}}); + // metric_best_block_height_->set(bestBlockNoLock(p).slot); + // + // metric_finalized_block_height_ = + // metrics_registry_->registerGaugeMetric( + // blockHeightMetricName, {{"status", "finalized"}}); + // + // metric_finalized_block_height_->set(getLastFinalizedNoLock(p).slot); + // + // metrics_registry_->registerGaugeFamily( + // knownChainLeavesMetricName, + // "Number of known chain leaves (aka forks)"); + // + // metric_known_chain_leaves_ = + // metrics_registry_->registerGaugeMetric(knownChainLeavesMetricName); + // metric_known_chain_leaves_->set(p.tree_->leafCount()); + // + // telemetry_->setGenesisBlockHash(getGenesisBlockHash()); + // + // if (p.blocks_pruning_.keep_) { + // SL_INFO(log_, + // "BlocksPruning: enabled with \"--blocks-pruning {}\"", + // *p.blocks_pruning_.keep_); + // } + }); + + // Add non-finalized block to the block tree + for (auto &item : std::get<1>(initializer->nonFinalizedSubTree())) { + const auto &block = item.first; + const auto header = std::move(item.second); + + auto res = BlockTreeImpl::addExistingBlock(block.hash, header); + if (res.has_error()) { + SL_WARN( + log_, "Failed to add existing block {}: {}", block, res.error()); + } + SL_TRACE(log_, + "Existing non-finalized block {} is added to block tree", + block); + } + + // OUTCOME_TRY(state_pruner->recoverState(*this)); + } + + const BlockHash &BlockTreeImpl::getGenesisBlockHash() const { + return block_tree_data_ + .sharedAccess([&](const BlockTreeData &p) + -> std::reference_wrapper { + if (p.genesis_block_hash_.has_value()) { + return p.genesis_block_hash_.value(); + } + + auto res = p.storage_->getBlockHash(0); + BOOST_ASSERT_MSG(res.has_value() and res.value().size() == 1, + "Block tree must contain exactly one genesis block"); + + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) + const_cast &>(p.genesis_block_hash_) + .emplace(res.value()[0]); + return p.genesis_block_hash_.value(); + }) + .get(); + } + + outcome::result BlockTreeImpl::addBlockHeader( + const BlockHeader &header) { + return block_tree_data_.exclusiveAccess( + [&](BlockTreeData &p) -> outcome::result { + auto parent_opt = p.tree_->find(header.parent); + if (not parent_opt.has_value()) { + return BlockTreeError::NO_PARENT; + } + const auto &parent = parent_opt.value(); + OUTCOME_TRY(p.storage_->putBlockHeader(header)); + + // update local meta with the new block + auto new_node = std::make_shared( + header.index(), parent, isPrimary(header)); + + auto reorg = p.tree_->add(new_node); + OUTCOME_TRY(reorgAndPrune(p, {std::move(reorg), {}})); + + auto header_ptr = std::make_shared(header); + se_manager_->notify(EventTypes::BlockAdded, header_ptr); + + SL_VERBOSE( + log_, "Block {} has been added into block tree", header.index()); + + return outcome::success(); + }); + } + + outcome::result BlockTreeImpl::addBlock(const Block &block) { + return block_tree_data_.exclusiveAccess( + [&](BlockTreeData &p) -> outcome::result { + // Check if we know parent of this block; if not, we cannot insert it + auto parent_opt = p.tree_->find(block.header.parent); + if (not parent_opt.has_value()) { + return BlockTreeError::NO_PARENT; + } + const auto &parent = parent_opt.value(); + + // Save block + OUTCOME_TRY(block_hash, p.storage_->putBlock(block)); + + // Update local meta with the block + auto new_node = std::make_shared( + block.header.index(), parent, isPrimary(block.header)); + + auto reorg = p.tree_->add(new_node); + OUTCOME_TRY(reorgAndPrune(p, {std::move(reorg), {}})); + + auto header_ptr = std::make_shared(block.header); + se_manager_->notify(EventTypes::BlockAdded, header_ptr); + + SL_DEBUG(log_, "Adding block {}", block_hash); + + // for (const auto &ext : block.extrinsic) { + // auto extrinsic_hash = p.hasher_->blake2b_256(ext.data); + // SL_DEBUG(log_, "Adding extrinsic with hash {}", extrinsic_hash); + // if (auto key = p.extrinsic_event_key_repo_->get(extrinsic_hash)) + // { + // main_pool_handler_->execute( + // [wself{weak_from_this()}, key{key.value()}, block_hash]() { + // if (auto self = wself.lock()) { + // self->extrinsic_events_engine_->notify( + // key, + // primitives::events::ExtrinsicLifecycleEvent::InBlock( + // key, block_hash)); + // } + // }); + // } + // } + + SL_VERBOSE(log_, + "Block {} has been added into block tree", + block.header.index()); + return outcome::success(); + }); + } + + void BlockTreeImpl::notifyChainEventsEngine(EventTypes event, + const BlockHeader &header) { + BOOST_ASSERT(header.hash_opt.has_value()); + auto header_ptr = std::make_shared(header); + se_manager_->notify(event, header_ptr); + } + + outcome::result BlockTreeImpl::removeLeaf(const BlockHash &block_hash) { + return block_tree_data_.exclusiveAccess( + [&](BlockTreeData &p) -> outcome::result { + auto finalized = getLastFinalizedNoLock(p); + if (block_hash == finalized.hash) { + OUTCOME_TRY(header, getBlockHeader(block_hash)); + // OUTCOME_TRY(p.storage_->removeJustification(finalized.hash)); + + OUTCOME_TRY(slot, getNumberByHash(header.parent)); + auto parent = BlockIndex(slot, header.parent); + + ReorgAndPrune changes{ + .reorg = Reorg{.common = parent, .revert = {finalized}}, + .prune = {finalized}, + }; + p.tree_ = std::make_unique(parent); + OUTCOME_TRY(reorgAndPrune(p, changes)); + return outcome::success(); + } + if (not p.tree_->isLeaf(block_hash)) { + return BlockTreeError::BLOCK_IS_NOT_LEAF; + } + auto changes = p.tree_->removeLeaf(block_hash); + OUTCOME_TRY(reorgAndPrune(p, changes)); + return outcome::success(); + }); + } + + // outcome::result BlockTreeImpl::markAsParachainDataBlock( + // const BlockHash &block_hash) { + // return block_tree_data_.exclusiveAccess( + // [&](BlockTreeData &p) -> outcome::result { + // SL_TRACE(log_, "Trying to adjust weight for block {}", block_hash); + // + // auto node = p.tree_->find(block_hash); + // if (node == nullptr) { + // SL_WARN(log_, "Block {} doesn't exists in block tree", + // block_hash); return BlockTreeError::BLOCK_NOT_EXISTS; + // } + // + // node->contains_approved_para_block = true; + // return outcome::success(); + // }); + // } + + outcome::result BlockTreeImpl::markAsRevertedBlocks( + const std::vector &block_hashes) { + return block_tree_data_.exclusiveAccess( + [&](BlockTreeData &p) -> outcome::result { + bool need_to_refresh_best = false; + auto best = bestBlockNoLock(p); + for (const auto &block_hash : block_hashes) { + auto node_opt = p.tree_->find(block_hash); + if (not node_opt.has_value()) { + SL_WARN( + log_, "Block {} doesn't exists in block tree", block_hash); + continue; + } + auto &node = node_opt.value(); + + if (not node->reverted) { + std::queue> to_revert; + to_revert.push(std::move(node)); + while (not to_revert.empty()) { + auto &reverting_tree_node = to_revert.front(); + + reverting_tree_node->reverted = true; + + if (reverting_tree_node->info == best) { + need_to_refresh_best = true; + } + + for (auto &child : reverting_tree_node->children) { + if (not child->reverted) { + to_revert.push(child); + } + } + + to_revert.pop(); + } + } + } + if (need_to_refresh_best) { + p.tree_->forceRefreshBest(); + } + return outcome::success(); + }); + } + + outcome::result BlockTreeImpl::addExistingBlockNoLock( + BlockTreeData &p, + const BlockHash &block_hash, + const BlockHeader &block_header) { + SL_TRACE(log_, + "Trying to add block {} into block tree", + BlockInfo(block_header.slot, block_hash)); + + auto node_opt = p.tree_->find(block_hash); + // Check if the tree doesn't have this block; if not, we skip that + if (node_opt.has_value()) { + SL_TRACE(log_, + "Block {} exists in block tree", + BlockInfo(block_header.slot, block_hash)); + return BlockTreeError::BLOCK_EXISTS; + } + + auto parent_opt = p.tree_->find(block_header.parent); + + // Check if we know parent of this block; if not, we cannot insert it + if (not parent_opt.has_value()) { + SL_TRACE(log_, + "Block {} parent of {} has not found in block tree. " + "Trying to restore missed branch", + block_header.parent, + BlockInfo(block_header.slot, block_hash)); + + // Trying to restore missed branch + std::stack> to_add; + + auto finalized = getLastFinalizedNoLock(p).slot; + + for (auto hash = block_header.parent;;) { + OUTCOME_TRY(header, p.storage_->getBlockHeader(hash)); + BlockInfo block_index(header.slot, hash); + SL_TRACE(log_, + "Block {} has found in storage and enqueued to add", + block_index); + + if (header.slot <= finalized) { + return BlockTreeError::BLOCK_ON_DEAD_END; + } + + auto parent_hash = header.parent; + to_add.emplace(hash, std::move(header)); + + if (p.tree_->find(parent_hash).has_value()) { + SL_TRACE(log_, + "Block {} parent of {} has found in block tree", + parent_hash, + block_index); + break; + } + + SL_TRACE(log_, + "Block {} has not found in block tree. " + "Trying to restore from storage", + parent_hash); + + hash = parent_hash; + } + + while (not to_add.empty()) { + const auto &[hash, header] = to_add.top(); + OUTCOME_TRY(addExistingBlockNoLock(p, hash, header)); + to_add.pop(); + } + + parent_opt = p.tree_->find(block_header.parent); + BOOST_ASSERT_MSG(parent_opt.has_value(), + "Parent must be restored at this moment"); + + SL_TRACE(log_, + "Trying to add block {} into block tree", + BlockInfo(block_header.slot, block_hash)); + } + auto &parent = parent_opt.value(); + + // Update local meta with the block + auto new_node = std::make_shared( + block_header.index(), parent, isPrimary(block_header)); + + auto reorg = p.tree_->add(new_node); + OUTCOME_TRY(reorgAndPrune(p, {std::move(reorg), {}})); + + SL_VERBOSE(log_, + "Block {} has been restored in block tree from storage", + block_header.index()); + + return outcome::success(); + } + + outcome::result BlockTreeImpl::addExistingBlock( + const BlockHash &block_hash, const BlockHeader &block_header) { + return block_tree_data_.exclusiveAccess( + [&](BlockTreeData &p) -> outcome::result { + return addExistingBlockNoLock(p, block_hash, block_header); + }); + } + + outcome::result BlockTreeImpl::addBlockBody(const BlockHash &block_hash, + const BlockBody &body) { + return block_tree_data_.exclusiveAccess( + [&](BlockTreeData &p) -> outcome::result { + return p.storage_->putBlockBody(block_hash, body); + }); + } + + outcome::result BlockTreeImpl::finalize( + const BlockHash &block_hash, const Justification &justification) { + return block_tree_data_.exclusiveAccess([&](BlockTreeData &p) + -> outcome::result { + auto last_finalized_block_info = getLastFinalizedNoLock(p); + if (block_hash == last_finalized_block_info.hash) { + return outcome::success(); + } + const auto node_opt = p.tree_->find(block_hash); + if (node_opt.has_value()) { + auto &node = node_opt.value(); + + SL_DEBUG(log_, "Finalizing block {}", node->info); + + OUTCOME_TRY(header, p.storage_->getBlockHeader(block_hash)); + + OUTCOME_TRY(p.storage_->putJustification(justification, block_hash)); + + std::vector retired_hashes; + for (auto parent = node->parent(); parent; parent = parent->parent()) { + retired_hashes.emplace_back(parent->info); + } + + auto changes = p.tree_->finalize(node); + OUTCOME_TRY(reorgAndPrune(p, changes)); + // OUTCOME_TRY(pruneTrie(p, node->info.slot)); + + auto finalized_header_ptr = std::make_shared(header); + se_manager_->notify(EventTypes::BlockFinalized, finalized_header_ptr); + + OUTCOME_TRY(body, p.storage_->getBlockBody(block_hash)); + if (body.has_value()) { + // for (auto &ext : body.value()) { + // auto extrinsic_hash = p.hasher_->blake2b_256(ext.data); + // if (auto key = p.extrinsic_event_key_repo_->get(extrinsic_hash)) + // { + // main_pool_handler_->execute([wself{weak_from_this()}, + // key{key.value()}, + // block_hash]() { + // if (auto self = wself.lock()) { + // self->extrinsic_events_engine_->notify( + // key, + // primitives::events::ExtrinsicLifecycleEvent::Finalized( + // key, block_hash)); + // } + // }); + // } + // } + } + + struct RemoveAfterFinalizationParams { + BlockInfo filanized; + std::vector removed; + }; + + auto data_ptr = std::make_shared( + header.index(), std::move(retired_hashes)); + se_manager_->notify(EventTypes::DeactivateAfterFinalization, data_ptr); + + log_->info("Finalized block {}", node->info); + // telemetry_->notifyBlockFinalized(node->info); + // telemetry_->pushBlockStats(); + // metric_finalized_block_height_->set(node->info.number); + + // we store justification for last finalized block only as long as it is + // last finalized (if it doesn't meet other justification storage rules, + // e.g. its number a multiple of 512) + OUTCOME_TRY(last_finalized_header, + p.storage_->getBlockHeader(last_finalized_block_info.hash)); + OUTCOME_TRY(shouldStoreLastFinalized, + p.justification_storage_policy_->shouldStoreFor( + last_finalized_header, getLastFinalizedNoLock(p).slot)); + if (!shouldStoreLastFinalized) { + OUTCOME_TRY( + justification_opt, + p.storage_->getJustification(last_finalized_block_info.hash)); + if (justification_opt.has_value()) { + SL_DEBUG(log_, + "Purge redundant justification for finalized block {}", + last_finalized_block_info); + OUTCOME_TRY(p.storage_->removeJustification( + last_finalized_block_info.hash)); + } + } + + // for (auto end = p.blocks_pruning_.max(node->info.slot); + // p.blocks_pruning_.next_ < end; + // ++p.blocks_pruning_.next_) { + // OUTCOME_TRY(hash, + // p.storage_->getBlockHash(p.blocks_pruning_.next_)); + // if (not hash) { + // continue; + // } + // SL_TRACE(log_, + // "BlocksPruning: remove body for block {}", + // p.blocks_pruning_.next_); + // OUTCOME_TRY(p.storage_->removeBlockBody(*hash)); + // } + } else { + OUTCOME_TRY(header, p.storage_->getBlockHeader(block_hash)); + const auto header_number = header.slot; + if (header_number >= last_finalized_block_info.slot) { + return BlockTreeError::NON_FINALIZED_BLOCK_NOT_FOUND; + } + + OUTCOME_TRY(hashes, p.storage_->getBlockHash(header_number)); + + if (not qtils::cxx23::ranges::contains(hashes, block_hash)) { + return BlockTreeError::BLOCK_ON_DEAD_END; + } + + if (not p.justification_storage_policy_ + ->shouldStoreFor(header, last_finalized_block_info.slot) + .value()) { + return outcome::success(); + } + OUTCOME_TRY(justification_opt, + p.storage_->getJustification(block_hash)); + if (justification_opt.has_value()) { + // block already has justification (in DB), fine + return outcome::success(); + } + OUTCOME_TRY(p.storage_->putJustification(justification, block_hash)); + } + return outcome::success(); + }); + } + + // outcome::result> + // BlockTreeImpl::getBlockHash(BlockNumber block_number) const { + // return block_tree_data_.sharedAccess( + // [&](const BlockTreeData &p) + // -> outcome::result> { + // OUTCOME_TRY(hash_opt, p.storage_->getBlockHash(block_number)); + // return hash_opt; + // }); + // } + + bool BlockTreeImpl::has(const BlockHash &hash) const { + return block_tree_data_.sharedAccess([&](const BlockTreeData &p) { + return p.tree_->find(hash) or p.storage_->hasBlockHeader(hash).value(); + }); + } + + outcome::result BlockTreeImpl::getBlockHeaderNoLock( + const BlockTreeData &p, const BlockHash &block_hash) const { + return p.storage_->getBlockHeader(block_hash); + } + + outcome::result BlockTreeImpl::getBlockHeader( + const BlockHash &block_hash) const { + return block_tree_data_.sharedAccess( + [&](const BlockTreeData &p) -> outcome::result { + return getBlockHeaderNoLock(p, block_hash); + }); + } + + // outcome::result> + // BlockTreeImpl::tryGetBlockHeader( + // const BlockHash &block_hash) const { + // return block_tree_data_.sharedAccess( + // [&](const BlockTreeData &p) + // -> outcome::result> { + // auto header = p.storage_->getBlockHeader(block_hash); + // if (header) { + // return header.value(); + // } + // const auto &header_error = header.error(); + // if (header_error == BlockTreeError::HEADER_NOT_FOUND) { + // return std::nullopt; + // } + // return header_error; + // }); + // } + + outcome::result BlockTreeImpl::getBlockBody( + const BlockHash &block_hash) const { + return block_tree_data_.sharedAccess( + [&](const BlockTreeData &p) -> outcome::result { + OUTCOME_TRY(body, p.storage_->getBlockBody(block_hash)); + if (body.has_value()) { + return body.value(); + } + return BlockTreeError::BODY_NOT_FOUND; + }); + } + + // outcome::result + // BlockTreeImpl::getBlockJustification( + // const BlockHash &block_hash) const { + // return block_tree_data_.sharedAccess( + // [&](const BlockTreeData &p) + // -> outcome::result { + // OUTCOME_TRY(justification, + // p.storage_->getJustification(block_hash)); if + // (justification.has_value()) { + // return justification.value(); + // } + // return BlockTreeError::JUSTIFICATION_NOT_FOUND; + // }); + // } + + outcome::result> BlockTreeImpl::getBestChainFromBlock( + const BlockHash &block, uint64_t maximum) const { + return block_tree_data_.sharedAccess( + [&](const BlockTreeData &p) -> outcome::result> { + auto block_header_res = p.storage_->getBlockHeader(block); + if (block_header_res.has_error()) { + log_->error("cannot retrieve block {}: {}", + block, + block_header_res.error()); + return BlockTreeError::HEADER_NOT_FOUND; + } + auto start_block_number = block_header_res.value().slot; + + if (maximum == 1) { + return std::vector{block}; + } + + auto current_depth = bestBlockNoLock(p).slot; + + if (start_block_number >= current_depth) { + return std::vector{block}; + } + + auto count = std::min( + current_depth - start_block_number + 1, maximum); + + BlockNumber finish_block_number = start_block_number + count - 1; + + auto finish_block_hash_res = + p.storage_->getBlockHash(finish_block_number); + if (finish_block_hash_res.has_error()) { + log_->error("cannot retrieve block with number {}: {}", + finish_block_number, + finish_block_hash_res.error()); + return BlockTreeError::HEADER_NOT_FOUND; + } + const auto &finish_block_hash = finish_block_hash_res.value()[0]; + + OUTCOME_TRY( + chain, + getDescendingChainToBlockNoLock(p, finish_block_hash, count)); + + if (chain.back() != block) { + return std::vector{block}; + } + std::ranges::reverse(chain); + return chain; + }); + } + + outcome::result> + BlockTreeImpl::getDescendingChainToBlockNoLock(const BlockTreeData &p, + const BlockHash &to_block, + uint64_t maximum) const { + std::vector chain; + + auto hash = to_block; + + // Try to retrieve from cached tree + if (auto node_opt = p.tree_->find(hash); node_opt.has_value()) { + auto node = node_opt.value(); + while (maximum > chain.size()) { + auto parent = node->parent(); + if (not parent) { + hash = node->info.hash; + break; + } + chain.emplace_back(node->info.hash); + node = parent; + } + } + + while (maximum > chain.size()) { + auto header_res = p.storage_->getBlockHeader(hash); + if (header_res.has_error()) { + if (chain.empty()) { + log_->error("Cannot retrieve block with hash {}: {}", + hash, + header_res.error()); + return header_res.error(); + } + break; + } + const auto &header = header_res.value(); + chain.emplace_back(hash); + + if (header.slot == 0) { + break; + } + hash = header.parent; + } + return chain; + } + + outcome::result> + BlockTreeImpl::getDescendingChainToBlock(const BlockHash &to_block, + uint64_t maximum) const { + return block_tree_data_.sharedAccess([&](const BlockTreeData &p) { + return getDescendingChainToBlockNoLock(p, to_block, maximum); + }); + } + + // BlockTreeImpl::outcome::result> + // BlockTreeImpl::getChainByBlocks( + // const BlockHash &ancestor, + // const BlockHash &descendant) const { + // return block_tree_data_.sharedAccess( + // [&](const BlockTreeData &p) -> + // BlockTreeImpl::outcome::result> { + // OUTCOME_TRY(from_header, p.storage_->getBlockHeader(ancestor)); + // auto from = from_header.number; + // OUTCOME_TRY(to_header, p.storage_->getBlockHeader(descendant)); + // auto to = to_header.number; + // if (to < from) { + // return BlockTreeError::TARGET_IS_PAST_MAX; + // } + // auto count = to - from + 1; + // OUTCOME_TRY(chain, + // getDescendingChainToBlockNoLock(p, descendant, count)); + // if (chain.size() != count) { + // return BlockTreeError::EXISTING_BLOCK_NOT_FOUND; + // } + // if (chain.back() != ancestor) { + // return BlockTreeError::BLOCK_ON_DEAD_END; + // } + // std::ranges::reverse(chain); + // return chain; + // }); + // } + // + // bool BlockTreeImpl::hasDirectChainNoLock( + // const BlockTreeData &p, + // const BlockHash &ancestor, + // const BlockHash &descendant) const { + // if (ancestor == descendant) { + // return true; + // } + // auto ancestor_node_ptr = p.tree_->find(ancestor); + // auto descendant_node_ptr = p.tree_->find(descendant); + // if (ancestor_node_ptr and descendant_node_ptr) { + // return canDescend(descendant_node_ptr, ancestor_node_ptr); + // } + // + // /* + // * check that ancestor is above descendant + // * optimization that prevents reading blockDB up the genesis + // * TODO (xDimon) it could be not right place for this check + // * or changing logic may make it obsolete + // * block numbers may be obtained somewhere else + // */ + // BlockNumber ancestor_depth = 0u; + // BlockNumber descendant_depth = 0u; + // if (ancestor_node_ptr) { + // ancestor_depth = ancestor_node_ptr->info.number; + // } else { + // auto header_res = p.storage_->getBlockHeader(ancestor); + // if (!header_res) { + // return false; + // } + // ancestor_depth = header_res.value().number; + // } + // if (descendant_node_ptr) { + // descendant_depth = descendant_node_ptr->info.number; + // } else { + // auto header_res = p.storage_->getBlockHeader(descendant); + // if (!header_res) { + // return false; + // } + // descendant_depth = header_res.value().number; + // } + // if (descendant_depth < ancestor_depth) { + // SL_DEBUG(log_, + // "Ancestor block is lower. {} in comparison with {}", + // BlockInfo(ancestor_depth, ancestor), + // BlockInfo(descendant_depth, descendant)); + // return false; + // } + // + // // Try to use optimal way, if ancestor and descendant in the finalized + // // chain + // auto finalized = [&](const BlockHash &hash, + // BlockNumber number) { + // return number <= getLastFinalizedNoLock(p).number + // and p.storage_->getBlockHash(number) + // == outcome::success( + // std::optional(hash)); + // }; + // if (descendant_node_ptr or finalized(descendant, descendant_depth)) { + // return finalized(ancestor, ancestor_depth); + // } + // + // auto current_hash = descendant; + // jam_PROFILE_START(search_finalized_chain) + // while (current_hash != ancestor) { + // auto current_header_res = p.storage_->getBlockHeader(current_hash); + // if (!current_header_res) { + // return false; + // } + // if (current_header_res.value().number <= ancestor_depth) { + // return false; + // } + // current_hash = current_header_res.value().parent_hash; + // } + // jam_PROFILE_END(search_finalized_chain) + // return true; + // } + // + // bool BlockTreeImpl::hasDirectChain( + // const BlockHash &ancestor, + // const BlockHash &descendant) const { + // return block_tree_data_.sharedAccess([&](const BlockTreeData &p) { + // return hasDirectChainNoLock(p, ancestor, descendant); + // }); + // } + // + // bool BlockTreeImpl::isFinalized(const BlockInfo &block) const { + // return block_tree_data_.sharedAccess([&](const BlockTreeData &p) { + // return block.number <= getLastFinalizedNoLock(p).number + // and p.storage_->getBlockHash(block.number) + // == outcome::success( + // std::optional(block.hash)); + // }); + // } + + BlockInfo BlockTreeImpl::bestBlockNoLock(const BlockTreeData &p) const { + return p.tree_->best(); + } + + BlockInfo BlockTreeImpl::bestBlock() const { + return block_tree_data_.sharedAccess( + [&](const BlockTreeData &p) { return bestBlockNoLock(p); }); + } + + outcome::result BlockTreeImpl::getBestContaining( + const BlockHash &target_hash) const { + return block_tree_data_.sharedAccess( + [&](const BlockTreeData &p) -> outcome::result { + if (getLastFinalizedNoLock(p).hash == target_hash) { + return bestBlockNoLock(p); + } + + auto target_node_opt = p.tree_->find(target_hash); + + // If a target has not found in the block tree (in memory), + // it means block finalized or discarded + if (not target_node_opt.has_value()) { + OUTCOME_TRY(target_header, p.storage_->getBlockHeader(target_hash)); + auto target_number = target_header.slot; + + OUTCOME_TRY(hashes, p.storage_->getBlockHash(target_number)); + + if (not qtils::cxx23::ranges::contains(hashes, target_hash)) { + return BlockTreeError::BLOCK_ON_DEAD_END; + } + + return bestBlockNoLock(p); + } + + return p.tree_->bestWith(target_node_opt.value()); + }); + } + + std::vector BlockTreeImpl::getLeavesNoLock( + const BlockTreeData &p) const { + return p.tree_->leafHashes(); + } + + std::vector BlockTreeImpl::getLeaves() const { + return block_tree_data_.sharedAccess( + [&](const BlockTreeData &p) { return getLeavesNoLock(p); }); + } + + // std::vector BlockTreeImpl::getLeavesInfo() const { + // return block_tree_data_.sharedAccess( + // [&](const BlockTreeData &p) { return p.tree_->leafInfo(); }); + // } + + outcome::result> BlockTreeImpl::getChildren( + const BlockHash &block) const { + return block_tree_data_.sharedAccess( + [&](const BlockTreeData &p) -> outcome::result> { + if (auto node_opt = p.tree_->find(block); node_opt.has_value()) { + const auto &node = node_opt.value(); + std::vector result; + result.reserve(node->children.size()); + for (const auto &child : node->children) { + result.push_back(child->info.hash); + } + return result; + } + OUTCOME_TRY(header, p.storage_->getBlockHeader(block)); + + // TODO slot of children can be greater then parent's by more then one + return p.storage_->getBlockHash(header.slot + 1); + }); + } + + BlockInfo BlockTreeImpl::getLastFinalizedNoLock( + const BlockTreeData &p) const { + return p.tree_->finalized(); + } + + BlockInfo BlockTreeImpl::getLastFinalized() const { + return block_tree_data_.sharedAccess( + [&](const BlockTreeData &p) { return getLastFinalizedNoLock(p); }); + } + + outcome::result BlockTreeImpl::reorgAndPrune( + BlockTreeData &p, const ReorgAndPrune &changes) { + OUTCOME_TRY(p.storage_->setBlockTreeLeaves(p.tree_->leafHashes())); + // metric_known_chain_leaves_->set(p.tree_->leafCount()); + if (changes.reorg) { + for (auto &block : changes.reorg->revert) { + OUTCOME_TRY(p.storage_->deassignHashToSlot(block)); + } + for (auto &block : changes.reorg->apply) { + OUTCOME_TRY(p.storage_->assignHashToSlot(block)); + } + // if (not changes.reorg->apply.empty()) { + // metric_best_block_height_->set(changes.reorg->apply.back().number); + // } else { + // metric_best_block_height_->set(changes.reorg->common.number); + // } + } + + // std::vector extrinsics; + + // remove from storage + for (const auto &block : changes.prune) { + OUTCOME_TRY(block_header, p.storage_->getBlockHeader(block.hash)); + OUTCOME_TRY(block_body_opt, p.storage_->getBlockBody(block.hash)); + if (block_body_opt.has_value()) { + // extrinsics.reserve(extrinsics.size() + + // block_body_opt.value().size()); for (auto &ext : + // block_body_opt.value()) { + // auto extrinsic_hash = p.hasher_->blake2b_256(ext.data); + // if (auto key = p.extrinsic_event_key_repo_->get(extrinsic_hash)) { + // main_pool_handler_->execute([wself{weak_from_this()}, + // key{key.value()}, + // block_hash{block.hash}]() { + // if (auto self = wself.lock()) { + // self->extrinsic_events_engine_->notify( + // key, + // primitives::events::ExtrinsicLifecycleEvent::Retracted( + // key, block_hash)); + // } + // }); + // } + // extrinsics.emplace_back(std::move(ext)); + // } + // p.state_pruner_->schedulePrune( + // block_header.state_root, + // block_header.index(), + // storage::trie_pruner::PruneReason::Discarded); + } + OUTCOME_TRY(p.storage_->removeBlock(block.hash)); + } + + return outcome::success(); + } + + // outcome::result BlockTreeImpl::pruneTrie( + // const BlockTreeData &block_tree_data, + // BlockNumber new_finalized) { + // // pruning is disabled + // if (!block_tree_data.state_pruner_->getPruningDepth().has_value()) { + // return outcome::success(); + // } + // auto last_pruned = block_tree_data.state_pruner_->getLastPrunedBlock(); + // + // BOOST_ASSERT(!last_pruned.has_value() + // || last_pruned.value().number + // <= getLastFinalizedNoLock(block_tree_data).number); + // auto next_pruned_number = last_pruned ? last_pruned->number + 1 : 0; + // + // OUTCOME_TRY(hash_opt, getBlockHash(next_pruned_number)); + // BOOST_ASSERT(hash_opt.has_value()); + // auto &hash = hash_opt.value(); + // auto pruning_depth = + // block_tree_data.state_pruner_->getPruningDepth().value_or(0); + // if (new_finalized < pruning_depth) { + // return outcome::success(); + // } + // + // auto last_to_prune = new_finalized - pruning_depth; + // for (auto n = next_pruned_number; n < last_to_prune; n++) { + // OUTCOME_TRY(next_hash_opt, getBlockHash(n + 1)); + // BOOST_ASSERT(next_hash_opt.has_value()); + // auto &next_hash = *next_hash_opt; + // OUTCOME_TRY(header, getBlockHeader(hash)); + // block_tree_data.state_pruner_->schedulePrune( + // header.state_root, + // header.index(), + // storage::trie_pruner::PruneReason::Finalized); + // hash = next_hash; + // } + // + // return outcome::success(); + // } + // + // void BlockTreeImpl::warp(const BlockInfo &block_info) { + // block_tree_data_.exclusiveAccess([&](BlockTreeData &p) { + // p.tree_ = std::make_unique(block_info); + // metric_known_chain_leaves_->set(1); + // metric_best_block_height_->set(block_info.number); + // telemetry_->notifyBlockFinalized(block_info); + // telemetry_->pushBlockStats(); + // metric_finalized_block_height_->set(block_info.number); + // }); + // } + + void BlockTreeImpl::notifyBestAndFinalized() { + auto best_info = bestBlock(); + auto best_header = getBlockHeader(best_info.hash).value(); + + auto best_header_ptr = std::make_shared(best_header); + se_manager_->notify(EventTypes::BlockAdded, best_header_ptr); + + auto finalized_info = getLastFinalized(); + auto finalized_header = getBlockHeader(finalized_info.hash).value(); + + auto finalized_header_ptr = std::make_shared(finalized_header); + se_manager_->notify(EventTypes::BlockFinalized, finalized_header_ptr); + } + + void BlockTreeImpl::removeUnfinalized() { + block_tree_data_.exclusiveAccess([&](BlockTreeData &p) { + auto changes = p.tree_->removeUnfinalized(); + if (auto r = reorgAndPrune(p, changes); r.has_error()) { + SL_WARN(log_, "removeUnfinalized error: {}", r.error()); + } + }); + } + + // BlockHeaderRepository methods + + outcome::result BlockTreeImpl::getNumberByHash( + const BlockHash &hash) const { + auto slot_opt = block_tree_data_.sharedAccess( + [&](const BlockTreeData &p) -> std::optional { + if (auto node = p.tree_->find(hash)) { + return node.value()->info.slot; + } + return std::nullopt; + }); + if (slot_opt.has_value()) { + return slot_opt.value(); + } + OUTCOME_TRY(header, getBlockHeader(hash)); + return header.slot; + } + + // outcome::result BlockTreeImpl::getHashByNumber( + // BlockNumber number) const { + // OUTCOME_TRY(block_hash_opt, getBlockHash(number)); + // if (block_hash_opt.has_value()) { + // return block_hash_opt.value(); + // } + // return BlockTreeError::HEADER_NOT_FOUND; + // } + // + // BlockTreeImpl::BlocksPruning::BlocksPruning(std::optional keep, + // BlockNumber + // finalized) + // : keep_{keep}, next_{max(finalized)} {} + // + // BlockNumber BlockTreeImpl::BlocksPruning::max( + // BlockNumber finalized) const { + // return keep_ and finalized > *keep_ ? finalized - *keep_ : 0; + // } + +} // namespace jam::blockchain diff --git a/src/blockchain/impl/block_tree_impl.hpp b/src/blockchain/impl/block_tree_impl.hpp new file mode 100644 index 00000000..67baabe1 --- /dev/null +++ b/src/blockchain/impl/block_tree_impl.hpp @@ -0,0 +1,282 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "blockchain/block_tree.hpp" + +// #include +// #include +// #include +// #include +// #include +// #include + +#include + +#include "app/configuration.hpp" +#include "blockchain/block_storage.hpp" +#include "blockchain/impl/block_tree_initializer.hpp" +// #include "blockchain/block_tree_error.hpp" +#include "blockchain/impl/cached_tree.hpp" +// #include "consensus/babe/types/babe_configuration.hpp" +// #include "consensus/timeline/types.hpp" +// #include "crypto/hasher.hpp" +#include + +#include "log/logger.hpp" +#include "se/impl/common.hpp" +#include "se/subscription_fwd.hpp" +// #include "metrics/metrics.hpp" +// #include "primitives/event_types.hpp" +// #include "storage/trie/trie_storage.hpp" +// #include "subscription/extrinsic_event_key_repository.hpp" +// #include "telemetry/service.hpp" +// #include "utils/safe_object.hpp" +#include "se/subscription.hpp" + +namespace jam::app { + class Configuration; +} + +// namespace jam { +// class PoolHandler; +// } // namespace jam +// +namespace jam::blockchain { + // struct ReorgAndPrune; + // class TreeNode; + class BlockTreeInitializer; +} // namespace jam::blockchain +// +// namespace jam::common { +// class MainThreadPool; +// } +// +// namespace jam::storage::trie_pruner { +// class TriePruner; +// } + +namespace jam::blockchain { + + class BlockTreeImpl : public BlockTree, + public std::enable_shared_from_this { + public: + // /// Recover block tree state at provided block + // static outcome::result recover( + // const BlockId &target_block_id, + // std::shared_ptr storage, + // std::shared_ptr trie_storage, + // std::shared_ptr block_tree); + + BlockTreeImpl( + qtils::SharedRef logsys, + qtils::SharedRef app_config, + qtils::SharedRef storage, + // // const BlockInfo &finalized, + qtils::SharedRef hasher, + // primitives::events::ChainSubscriptionEnginePtr chain_events_engine, + // primitives::events::ExtrinsicSubscriptionEnginePtr + // extrinsic_events_engine, + // std::shared_ptr + // extrinsic_event_key_repo, + std::shared_ptr + justification_storage_policy, + // std::shared_ptr state_pruner, + // common::MainThreadPool &main_thread_pool, + std::shared_ptr se_manager, + qtils::SharedRef initializer); + + ~BlockTreeImpl() override = default; + + const BlockHash &getGenesisBlockHash() const override; + + // outcome::result> getBlockHash( + // BlockNumber block_number) const override; + + bool has(const BlockHash &hash) const override; + + outcome::result getBlockHeader( + const BlockHash &block_hash) const override; + + // outcome::result> tryGetBlockHeader( + // const BlockHash &block_hash) const override; + + outcome::result getBlockBody( + const BlockHash &block_hash) const override; + + // outcome::result getBlockJustification( + // const BlockHash &block_hash) const override; + + outcome::result addBlockHeader(const BlockHeader &header) override; + + outcome::result addBlock(const Block &block) override; + + outcome::result removeLeaf(const BlockHash &block_hash) override; + + outcome::result addExistingBlock( + const BlockHash &block_hash, const BlockHeader &block_header) override; + + // outcome::result markAsParachainDataBlock( + // const BlockHash &block_hash) override; + + outcome::result markAsRevertedBlocks( + const std::vector &block_hashes) override; + + outcome::result addBlockBody(const BlockHash &block_hash, + const BlockBody &body) override; + + outcome::result finalize(const BlockHash &block_hash, + const Justification &justification) override; + + outcome::result> getBestChainFromBlock( + const BlockHash &block, uint64_t maximum) const override; + + outcome::result> getDescendingChainToBlock( + const BlockHash &block, uint64_t maximum) const override; + + // outcome::result> getChainByBlocks( + // const BlockHash &ancestor, + // const BlockHash &descendant) const override; + // + // bool hasDirectChain(const BlockHash &ancestor, + // const BlockHash &descendant) const override; + // + // bool isFinalized(const BlockInfo &block) const override; + + BlockInfo bestBlock() const override; + + outcome::result getBestContaining( + const BlockHash &target_hash) const override; + + std::vector getLeaves() const override; + // std::vector getLeavesInfo() const override; + + outcome::result> getChildren( + const BlockHash &block) const override; + + BlockInfo getLastFinalized() const override; + + // void warp(const BlockInfo &block_info) override; + + void notifyBestAndFinalized() override; + + void removeUnfinalized() override; + + // BlockHeaderRepository methods + + outcome::result getNumberByHash( + const BlockHash &block_hash) const override; + + // outcome::result getHashByNumber( + // BlockNumber block_number) const override; + + private: + // struct BlocksPruning { + // BlocksPruning(std::optional keep, + // BlockNumber finalized); + // + // BlockNumber max(BlockNumber finalized) const; + // + // std::optional keep_; + // BlockNumber next_; + // }; + + struct BlockTreeData { + qtils::SharedRef storage_; + // std::shared_ptr state_pruner_; + std::unique_ptr tree_; + qtils::SharedRef hasher_; + std::shared_ptr + justification_storage_policy_; + std::optional genesis_block_hash_; + // BlocksPruning blocks_pruning_; + }; + + outcome::result reorgAndPrune(BlockTreeData &p, + const ReorgAndPrune &changes); + + outcome::result getBlockHeaderNoLock( + const BlockTreeData &p, const BlockHash &block_hash) const; + + // outcome::result pruneTrie(const BlockTreeData &block_tree_data, + // BlockNumber new_finalized); + + BlockInfo getLastFinalizedNoLock(const BlockTreeData &p) const; + BlockInfo bestBlockNoLock(const BlockTreeData &p) const; + + // bool hasDirectChainNoLock(const BlockTreeData &p, + // const BlockHash &ancestor, + // const BlockHash &descendant); + std::vector getLeavesNoLock(const BlockTreeData &p) const; + + outcome::result> getDescendingChainToBlockNoLock( + const BlockTreeData &p, + const BlockHash &to_block, + uint64_t maximum) const; + + outcome::result addExistingBlockNoLock( + BlockTreeData &p, + const BlockHash &block_hash, + const BlockHeader &block_header); + + void notifyChainEventsEngine(EventTypes event, const BlockHeader &header); + + class SafeBlockTreeData { + public: + SafeBlockTreeData(BlockTreeData data); + + template + decltype(auto) exclusiveAccess(F &&f) { + // if this thread owns the mutex, it shall + // not be unlocked until this function exits + if (exclusive_owner_.load(std::memory_order_acquire) + == std::this_thread::get_id()) { + return f(block_tree_data_.unsafeGet()); + } + return block_tree_data_.exclusiveAccess([&f, + this](BlockTreeData &data) { + exclusive_owner_ = std::this_thread::get_id(); + qtils::FinalAction reset([&] { exclusive_owner_ = std::nullopt; }); + return f(data); + }); + } + + template + decltype(auto) sharedAccess(F &&f) const { + // if this thread owns the mutex, it shall + // not be unlocked until this function exits + if (exclusive_owner_.load(std::memory_order_acquire) + == std::this_thread::get_id()) { + return f(block_tree_data_.unsafeGet()); + } + return block_tree_data_.sharedAccess(std::forward(f)); + } + + private: + se::utils::SafeObject block_tree_data_; + std::atomic> exclusive_owner_; + }; + + log::Logger log_; + std::shared_ptr se_manager_; + + SafeBlockTreeData block_tree_data_; + + // primitives::events::ChainSubscriptionEnginePtr chain_events_engine_; + // std::shared_ptr main_pool_handler_; + // primitives::events::ExtrinsicSubscriptionEnginePtr + // extrinsic_events_engine_; + + // // Metrics + // metrics::RegistryPtr metrics_registry_ = metrics::createRegistry(); + // metrics::Gauge *metric_best_block_height_; + // metrics::Gauge *metric_finalized_block_height_; + // metrics::Gauge *metric_known_chain_leaves_; + // telemetry::Telemetry telemetry_ = telemetry::createTelemetryService(); + }; + +} // namespace jam::blockchain diff --git a/src/blockchain/impl/block_tree_initializer.cpp b/src/blockchain/impl/block_tree_initializer.cpp new file mode 100644 index 00000000..6d0dd5a8 --- /dev/null +++ b/src/blockchain/impl/block_tree_initializer.cpp @@ -0,0 +1,270 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + + +#include "blockchain/impl/block_tree_initializer.hpp" + +#include "blockchain/block_storage.hpp" +#include "blockchain/block_tree_error.hpp" +#include "log/logger.hpp" +#include "qtils/error_throw.hpp" + +namespace jam::blockchain { + + namespace { + /// Function-helper for loading (and repair if it needed) of leaves + outcome::result> loadLeaves( + const qtils::SharedRef &storage, const log::Logger &log) { + std::set block_tree_leaves; + { + OUTCOME_TRY(block_tree_unordered_leaves, storage->getBlockTreeLeaves()); + SL_TRACE(log, + "List of leaves has loaded: {} leaves", + block_tree_unordered_leaves.size()); + + for (auto &hash : block_tree_unordered_leaves) { + // get slot of block by hash + const auto header = storage->getBlockHeader(hash); + if (not header) { + if (header == outcome::failure(BlockTreeError::HEADER_NOT_FOUND)) { + SL_TRACE(log, "Leaf {} not found", hash); + continue; + } + SL_ERROR(log, "Leaf {} is corrupted: {}", hash, header.error()); + return header.as_failure(); + } + auto slot = header.value().slot; + SL_TRACE(log, "Leaf {} found", BlockInfo(slot, hash)); + block_tree_leaves.emplace(slot, hash); + } + } + + if (block_tree_leaves.empty()) { + SL_WARN(log, "No one leaf was found. Trying to repair"); + + TimeSlot slot = 0; + auto lower = std::numeric_limits::min(); + auto upper = std::numeric_limits::max(); + + for (;;) { + slot = lower + (upper - lower) / 2 + 1; + + auto hashes_opt_res = storage->getBlockHash(slot); + if (hashes_opt_res.has_failure()) { + SL_CRITICAL(log, + "Search best block has failed: {}", + hashes_opt_res.error()); + return BlockTreeError::HEADER_NOT_FOUND; + } + const auto &hashes = hashes_opt_res.value(); + + if (not hashes.empty()) { + SL_TRACE(log, "bisect {} -> found", slot); + lower = slot; + } else { + SL_TRACE(log, "bisect {} -> not found", slot); + upper = slot - 1; + } + if (lower == upper) { + slot = lower; + break; + } + } + + OUTCOME_TRY(hashes, storage->getBlockHash(slot)); + for (auto &hash : hashes) { + block_tree_leaves.emplace(slot, hash); + } + + if (auto res = storage->setBlockTreeLeaves(hashes); res.has_error()) { + SL_CRITICAL( + log, "Can't save recovered block tree leaves: {}", res.error()); + return res.as_failure(); + } + } + + return block_tree_leaves; + } + } // namespace + + BlockTreeInitializer::BlockTreeInitializer( + qtils::SharedRef logsys, + qtils::SharedRef storage) { + auto logger = logsys->getLogger("BlockTree", "block_tree"); + + // Get the last finalized block + auto last_finalized_block_info_res = storage->getLastFinalized(); + if (last_finalized_block_info_res.has_error()) { + SL_CRITICAL(logger, + "Failed to get last finalized block info: {}", + last_finalized_block_info_res.error()); + qtils::raise(last_finalized_block_info_res.error()); + } + auto &last_finalized_block_info = last_finalized_block_info_res.value(); + + // Get its header + auto finalized_block_header_res = + storage->getBlockHeader(last_finalized_block_info.hash); + if (finalized_block_header_res.has_error()) { + SL_CRITICAL(logger, + "Failed to get last finalized block header: {}", + finalized_block_header_res.error()); + qtils::raise(finalized_block_header_res.error()); + } + auto &finalized_block_header = finalized_block_header_res.value(); + + // // call chain_events_engine->notify to init babe_config_repo preventive + // chain_events_engine->notify( + // events::ChainEventType::kFinalizedHeads, + // finalized_block_header); + + // // Ensure if last_finalized_block_info has the necessary justifications + // OUTCOME_TRY(storage->getJustification(last_finalized_block_info.hash)); + + // Load (or recalculate) leaves + auto block_tree_leaves_res = loadLeaves(storage, logger); + if (block_tree_leaves_res.has_error()) { + SL_CRITICAL(logger, + "Failed to load block tree leaves: {}", + block_tree_leaves_res.error()); + qtils::raise(block_tree_leaves_res.error()); + } + + // Ensure if a list of leaves is empty + auto &block_tree_leaves = block_tree_leaves_res.value(); + if (block_tree_leaves.empty()) { + SL_CRITICAL(logger, "Not found any block tree leaves"); + qtils::raise(BlockTreeError::BLOCK_TREE_CORRUPTED); + } + + // Last known block + auto last_known_block = *block_tree_leaves.rbegin(); + SL_INFO(logger, + "Last known block: {}, Last finalized: {}", + last_known_block, + last_finalized_block_info); + + // Load non-finalized block from block storage + std::map collected; + + { + std::unordered_set observed; + std::unordered_set dead; + // Iterate leaves + for (auto &leaf : block_tree_leaves) { + std::unordered_set subchain; + // Iterate subchain from leaf to finalized or early observer + for (auto block = leaf;;) { + // Met last finalized + if (block.hash == last_finalized_block_info.hash) { + break; + } + + // Met early observed block + if (observed.contains(block.hash)) { + break; + } + + // Met known dead block + if (dead.contains(block)) { + dead.insert(subchain.begin(), subchain.end()); + break; + } + + // Check if non-pruned fork has detected + if (block.slot == last_finalized_block_info.slot) { + dead.insert(subchain.begin(), subchain.end()); + + auto main = last_finalized_block_info; + auto fork = block; + + // Collect as the dead all blocks that differ from the finalized + // chain + for (;;) { + dead.emplace(fork); + + auto f_res = storage->getBlockHeader(fork.hash); + if (f_res.has_error()) { + break; + } + const auto &fork_header = f_res.value(); + + auto m_res = storage->getBlockHeader(main.hash); + if (m_res.has_error()) { + break; + } + const auto &main_header = m_res.value(); + + BOOST_ASSERT(fork_header.slot == main_header.slot); + if (fork_header.parent == main_header.parent) { + break; + } + + fork = {fork_header.slot, fork_header.hash()}; + main = {main_header.slot, main_header.hash()}; + } + + break; + } + + subchain.emplace(block); + + auto header_res = storage->getBlockHeader(block.hash); + if (header_res.has_error()) { + SL_CRITICAL( + logger, + "Can't get header of existing non-finalized block {}: {}", + block, + header_res.error()); + qtils::raise(BlockTreeError::BLOCK_TREE_CORRUPTED); + } + + observed.emplace(block.hash); + + auto &header = header_res.value(); + if (header.slot < last_finalized_block_info.slot) { + SL_WARN(logger, + "Detected a leaf {} lower than the last finalized block {}", + block, + last_finalized_block_info); + break; + } + + auto [it, ok] = collected.emplace(block, std::move(header)); + + block = {it->second.slot, it->second.hash()}; + } + } + + if (not dead.empty()) { + SL_WARN(logger, + "Found {} orphan blocks; " + "these block will be removed for consistency", + dead.size()); + for (auto &block : dead) { + collected.erase(block); + std::ignore = storage->removeBlock(block.hash); + } + } + } + + // Prepare and create a block tree basing last finalized block + SL_DEBUG(logger, "Last finalized block {}", last_finalized_block_info); + + last_finalized_ = last_finalized_block_info; + non_finalized_ = std::move(collected); + } + + std::tuple> + BlockTreeInitializer::nonFinalizedSubTree() { + // if (used_.test_and_set()) { + // qtils::raise(BlockTreeError::WRONG_WORKFLOW); + // } + // return std::make_tuple(last_finalized_, std::move(non_finalized_)); + return std::make_tuple(last_finalized_, non_finalized_); + } + +} // namespace jam::blockchain diff --git a/src/blockchain/impl/block_tree_initializer.hpp b/src/blockchain/impl/block_tree_initializer.hpp new file mode 100644 index 00000000..28b9b013 --- /dev/null +++ b/src/blockchain/impl/block_tree_initializer.hpp @@ -0,0 +1,36 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include + +namespace jam::log { + class LoggingSystem; +} +namespace jam::blockchain { + class BlockStorage; +} + +namespace jam::blockchain { + + class BlockTreeInitializer final : Singleton { + public: + BlockTreeInitializer(qtils::SharedRef logsys, + qtils::SharedRef storage); + + std::tuple> + nonFinalizedSubTree(); + + private: + std::atomic_flag used_; + BlockInfo last_finalized_; + std::map non_finalized_; + }; + +} // namespace jam::blockchain diff --git a/src/blockchain/impl/cached_tree.cpp b/src/blockchain/impl/cached_tree.cpp new file mode 100644 index 00000000..9ca6dcad --- /dev/null +++ b/src/blockchain/impl/cached_tree.cpp @@ -0,0 +1,342 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "blockchain/impl/cached_tree.hpp" + +#include + +#include + +namespace jam::blockchain { + bool Reorg::empty() const { + return revert.empty() and apply.empty(); + } + + TreeNode::TreeNode(const BlockIndex &info) : info{info} {} + + TreeNode::TreeNode(const BlockIndex &info, + const qtils::SharedRef &parent, + bool primary) + : info{info}, + weak_parent{std::shared_ptr(parent)}, + fallback(not primary), + depth(parent->depth + 1), + reverted{parent->reverted} {} + + std::shared_ptr TreeNode::parent() const { + return weak_parent.lock(); + } + + void TreeNode::detach() { + return weak_parent.reset(); + } + + BlockWeight TreeNode::weight() const { + return {by_ticket_weight, by_fallback_weight, info.slot}; + } + + Reorg reorg(qtils::SharedRef from, qtils::SharedRef to) { + Reorg reorg; + while (from != to) { + if (from->info.slot > to->info.slot) { + reorg.revert.emplace_back(from->info); + from = from->parent(); + [[unlikely]] if (not from) { throw std::bad_weak_ptr{}; } + + } else { + reorg.apply.emplace_back(to->info); + to = to->parent(); + [[unlikely]] if (not from) { throw std::bad_weak_ptr{}; } + } + } + reorg.common = to->info; + std::ranges::reverse(reorg.apply); + return reorg; + } + + template + bool descend(qtils::SharedRef from, + const qtils::SharedRef &to, + const F &f) { + while (from != to) { + if (from->info.slot <= to->info.slot) { + return false; + } + f(from); + from = from->parent(); + [[unlikely]] if (not from) { throw std::bad_weak_ptr{}; } + } + return true; + } + + bool canDescend(qtils::SharedRef from, + const qtils::SharedRef &to) { + return descend(from, to, [](const qtils::SharedRef &) {}); + } + + bool CachedTree::chooseBest(qtils::SharedRef node) { + if (node->reverted) { + return false; + } + BOOST_ASSERT(not best_->reverted); + if (node->weight() > best_->weight()) { + best_ = node; + return true; + } + return false; + } + + struct Cmp { + bool operator()(const qtils::SharedRef &lhs, + const qtils::SharedRef &rhs) const { + BOOST_ASSERT(lhs and rhs); + return rhs->info.slot < lhs->info.slot; + } + }; + + void CachedTree::forceRefreshBest() { + std::set, Cmp> candidates; + for (auto &leaf : leaves_) { + auto node = find(leaf.first); + BOOST_ASSERT(node.has_value()); + candidates.emplace(std::move(node.value())); + } + + best_ = root_; + while (not candidates.empty()) { + auto node = candidates.extract(candidates.begin()); + BOOST_ASSERT(not node.empty()); + + auto &tree_node = node.value(); + if (tree_node->reverted) { + if (auto parent = tree_node->parent()) { + candidates.emplace(std::move(parent)); + } + continue; + } + + if (best_->weight() < tree_node->weight()) { + best_ = tree_node; + } + } + } + + CachedTree::CachedTree(const BlockIndex &root) + : root_{std::make_shared(root)}, + best_{root_}, + nodes_{{root.hash, root_}} { + leaves_.emplace(root.hash, root.slot); + } + + BlockIndex CachedTree::finalized() const { + return root_->info; + } + + BlockIndex CachedTree::best() const { + return best_->info; + } + + size_t CachedTree::leafCount() const { + return leaves_.size(); + } + + std::vector CachedTree::leafInfo() const { + std::vector output; + output.reserve(leaves_.size()); + std::ranges::transform( + leaves_, std::back_inserter(output), [](const auto &v) { + return BlockIndex(v.second, v.first); + }); + return output; + } + + std::vector CachedTree::leafHashes() const { + std::vector output; + output.reserve(leaves_.size()); + std::ranges::transform(leaves_, + std::back_inserter(output), + [](const auto &v) { return v.first; }); + return output; + } + + bool CachedTree::isLeaf(const BlockHash &hash) const { + return leaves_.find(hash) != leaves_.end(); + } + + BlockIndex CachedTree::bestWith( + const qtils::SharedRef &required) const { + std::set, Cmp> candidates; + for (const auto &leaf : leaves_ | std::views::keys) { + auto node_opt = find(leaf); + BOOST_ASSERT(node_opt.has_value()); + candidates.emplace(std::move(node_opt.value())); + } + auto best = required; + while (not candidates.empty()) { + auto _node = candidates.extract(candidates.begin()); + auto &node = _node.value(); + if (node->info.slot <= required->info.slot) { + continue; + } + if (node->reverted) { + if (auto parent = node->parent()) { + candidates.emplace(std::move(parent)); + } + continue; + } + if (node->weight() > best->weight()) { + if (canDescend(node, required)) { + best = node; + } + } + } + return best->info; + } + + std::optional> CachedTree::find( + const BlockHash &hash) const { + if (auto it = nodes_.find(hash); it != nodes_.end()) { + return it->second; + } + return std::nullopt; + } + + std::optional CachedTree::add( + const qtils::SharedRef &new_node) { + if (nodes_.find(new_node->info.hash) != nodes_.end()) { + return std::nullopt; + } + BOOST_ASSERT(new_node->children.empty()); + auto parent = new_node->parent(); + [[unlikely]] if (not parent) { throw std::bad_weak_ptr{}; } + BOOST_ASSERT( + std::find(parent->children.begin(), parent->children.end(), new_node) + == parent->children.end()); + + new_node->depth = parent->depth + 1; + new_node->by_ticket_weight = + parent->by_ticket_weight + (new_node->fallback ? 0 : 1); + new_node->by_fallback_weight = + parent->by_fallback_weight + (new_node->fallback ? 1 : 0); + + parent->children.emplace_back(new_node); + nodes_.emplace(new_node->info.hash, new_node); + leaves_.erase(parent->info.hash); + leaves_.emplace(new_node->info.hash, new_node->info.slot); + if (not new_node->reverted and new_node->weight() > best_->weight()) { + auto old_best = best_; + best_ = new_node; + return reorg(old_best, best_); + } + return std::nullopt; + } + + ReorgAndPrune CachedTree::finalize( + const qtils::SharedRef &new_finalized) { + BOOST_ASSERT(new_finalized->info.slot >= root_->info.slot); + if (new_finalized == root_) { + return {}; + } + BOOST_ASSERT(new_finalized->parent()); + ReorgAndPrune changes; + if (not canDescend(best_, new_finalized)) { + changes.reorg = reorg(best_, new_finalized); + } + std::deque> queue; + for (std::shared_ptr finalized_child = new_finalized, + parent = finalized_child->parent(); + parent; + finalized_child = parent, parent = parent->parent()) { + for (auto &child : parent->children) { + if (child == finalized_child) { + continue; + } + queue.emplace_back(child); + } + parent->children.clear(); + nodes_.erase(parent->info.hash); + } + while (not queue.empty()) { + auto parent = std::move(queue.front()); + queue.pop_front(); + changes.prune.emplace_back(parent->info); + for (auto &child : parent->children) { + queue.emplace_back(child); + } + if (parent->children.empty()) { + leaves_.erase(parent->info.hash); + } + parent->children.clear(); + nodes_.erase(parent->info.hash); + } + std::ranges::reverse(changes.prune); + root_ = new_finalized; + root_->detach(); + if (changes.reorg) { + forceRefreshBest(); + size_t offset = changes.reorg->apply.size(); + [[maybe_unused]] auto ok = descend( + best_, new_finalized, [&](const std::shared_ptr node) { + changes.reorg->apply.emplace_back(node->info); + }); + BOOST_ASSERT(ok); + std::reverse( + // NOLINTNEXTLINE(cppcoreguidelines-narrowing-conversions) + changes.reorg->apply.begin() + offset, + changes.reorg->apply.end()); + } + return changes; + } + + ReorgAndPrune CachedTree::removeLeaf(const BlockHash &hash) { + ReorgAndPrune changes; + auto node_it = nodes_.find(hash); + BOOST_ASSERT(node_it != nodes_.end()); + auto &node = node_it->second; + BOOST_ASSERT(node); + auto leaf_it = leaves_.find(hash); + BOOST_ASSERT(leaf_it != leaves_.end()); + BOOST_ASSERT(node->children.empty()); + auto parent = node->parent(); + [[unlikely]] if (not parent) { throw std::bad_weak_ptr{}; } + auto child_it = + std::find(parent->children.begin(), parent->children.end(), node); + BOOST_ASSERT(child_it != parent->children.end()); + changes.prune.emplace_back(node->info); + parent->children.erase(child_it); + if (parent->children.empty()) { + leaves_.emplace(parent->info.hash, parent->info.slot); + } + leaves_.erase(leaf_it); + if (node == best_) { + auto old_best = node; + forceRefreshBest(); + changes.reorg = reorg(old_best, best_); + } + nodes_.erase(node_it); + return changes; + } + + ReorgAndPrune CachedTree::removeUnfinalized() { + ReorgAndPrune changes; + if (best_ != root_) { + changes.reorg = reorg(best_, root_); + } + std::deque> queue{root_}; + while (not queue.empty()) { + auto parent = std::move(queue.front()); + queue.pop_front(); + for (auto &child : parent->children) { + changes.prune.emplace_back(child->info); + queue.emplace_back(child); + } + parent->children.clear(); + } + std::ranges::reverse(changes.prune); + *this = CachedTree{root_->info}; + return changes; + } +} // namespace jam::blockchain diff --git a/src/blockchain/impl/cached_tree.hpp b/src/blockchain/impl/cached_tree.hpp new file mode 100644 index 00000000..9a789a73 --- /dev/null +++ b/src/blockchain/impl/cached_tree.hpp @@ -0,0 +1,119 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include "jam_types/types.tmp.hpp" + +namespace jam::blockchain { + using BlockWeight = std::tuple; + + /** + * Used to update hashes of best chain by number. + */ + struct Reorg { + BlockIndex common; + std::vector revert; + std::vector apply; + + bool empty() const; + }; + + /** + * Used to enqueue blocks for removal. + * Children are removed before parent. + */ + struct ReorgAndPrune { + std::optional reorg; + std::vector prune; + }; + + /** + * In-memory light representation of the tree, used for efficiency and usage + * convenience - we would only ask the database for some info, when directly + * requested + */ + class TreeNode { + public: + TreeNode(const BlockIndex &info); + TreeNode(const BlockIndex &info, + const qtils::SharedRef &parent, + bool primary); + + [[nodiscard]] std::shared_ptr parent() const; + void detach(); + [[nodiscard]] BlockWeight weight() const; + + const BlockIndex info; + + private: + std::weak_ptr weak_parent; + + public: + const bool fallback = false; + + uint32_t by_ticket_weight = fallback ? 0 : 1; + uint32_t by_fallback_weight = fallback ? 1 : 0; + uint32_t depth = 0; + bool contains_approved_para_block = false; + bool reverted = false; // TODO Looks like actually unused + + std::vector> children{}; + }; + + Reorg reorg(qtils::SharedRef from, qtils::SharedRef to); + + bool canDescend(qtils::SharedRef from, + const qtils::SharedRef &to); + + /** + * Non-finalized part of the block tree + */ + class CachedTree { + public: + explicit CachedTree(const BlockIndex &root); + + [[nodiscard]] BlockIndex finalized() const; + [[nodiscard]] BlockIndex best() const; + [[nodiscard]] size_t leafCount() const; + [[nodiscard]] std::vector leafHashes() const; + [[nodiscard]] std::vector leafInfo() const; + [[nodiscard]] bool isLeaf(const BlockHash &hash) const; + [[nodiscard]] BlockIndex bestWith( + const qtils::SharedRef &required) const; + [[nodiscard]] std::optional> find( + const BlockHash &hash) const; + std::optional add(const qtils::SharedRef &new_node); + ReorgAndPrune finalize(const qtils::SharedRef &new_finalized); + + /** + * Can't remove finalized root. + */ + ReorgAndPrune removeLeaf(const BlockHash &hash); + + /** + * Used when switching from fast-sync to full-sync. + */ + ReorgAndPrune removeUnfinalized(); + + /// Force find and update the actual best block + void forceRefreshBest(); + + private: + /** + * Compare node weight with best and replace if heavier. + * @return true if heavier and replaced. + */ + bool chooseBest(qtils::SharedRef node); + + qtils::SharedRef root_; + qtils::SharedRef best_; + std::unordered_map> nodes_; + std::unordered_map leaves_; + }; +} // namespace jam::blockchain diff --git a/src/blockchain/impl/justification_storage_policy.cpp b/src/blockchain/impl/justification_storage_policy.cpp new file mode 100644 index 00000000..7a9a0b46 --- /dev/null +++ b/src/blockchain/impl/justification_storage_policy.cpp @@ -0,0 +1,35 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "blockchain/impl/justification_storage_policy.hpp" + +#include + +#include "blockchain/block_tree.hpp" + +namespace jam::blockchain { + + outcome::result JustificationStoragePolicyImpl::shouldStoreFor( + const BlockHeader &block_header, + BlockNumber last_finalized_number) const { + if (block_header.slot == 0) { + return true; + } + + BOOST_ASSERT_MSG(last_finalized_number >= block_header.slot, + "Target block must be finalized"); + + // TODO This is case of change authority and we need to save justification + // if (consensus::grandpa::HasAuthoritySetChange{block_header}) { + // return true; + // } + if (block_header.slot % 512 == 0) { // TODO slot or calculate depths? + return true; + } + return false; + } + +} // namespace kagome::blockchain diff --git a/src/blockchain/impl/justification_storage_policy.hpp b/src/blockchain/impl/justification_storage_policy.hpp new file mode 100644 index 00000000..aa37f5aa --- /dev/null +++ b/src/blockchain/impl/justification_storage_policy.hpp @@ -0,0 +1,35 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "jam_types/block_header.hpp" +#include "qtils/outcome.hpp" + +namespace jam::blockchain { + class BlockTree; +} + +namespace jam::blockchain { + + class JustificationStoragePolicy { + public: + virtual ~JustificationStoragePolicy() = default; + + [[nodiscard]] virtual outcome::result shouldStoreFor( + const BlockHeader &block, + BlockNumber last_finalized_number) const = 0; + }; + + class JustificationStoragePolicyImpl final + : public JustificationStoragePolicy { + public: + [[nodiscard]] outcome::result shouldStoreFor( + const BlockHeader &block, + BlockNumber last_finalized_number) const override; + }; + +} // namespace kagome::blockchain diff --git a/src/injector/node_injector.cpp b/src/injector/node_injector.cpp index 650fbf32..c47a047d 100644 --- a/src/injector/node_injector.cpp +++ b/src/injector/node_injector.cpp @@ -27,6 +27,8 @@ #include "app/impl/watchdog.hpp" #include "blockchain/impl/genesis_block_header_impl.hpp" #include "blockchain/impl/block_storage_impl.hpp" +#include "blockchain/impl/block_tree_impl.hpp" +#include "blockchain/impl/justification_storage_policy.hpp" #include "clock/impl/clock_impl.hpp" #include "crypto/hasher/hasher_impl.hpp" #include "injector/bind_by_lambda.hpp" @@ -84,6 +86,8 @@ namespace { di::bind.to(), di::bind.to(), di::bind.to(), + di::bind.to(), + di::bind.to(), // user-defined overrides... std::forward(args)...); diff --git a/src/se/subscription_fwd.hpp b/src/se/subscription_fwd.hpp index 5752b241..a997aaf6 100644 --- a/src/se/subscription_fwd.hpp +++ b/src/se/subscription_fwd.hpp @@ -48,11 +48,20 @@ namespace jam { PeerConnected, /// Peer disconnected PeerDisconnected, - /// Data of block is requested + /// Data of a block is requested BlockRequest, - /// Data of block is respond + /// Data of a block is respond BlockResponse, + // -- BlockTree + + // A new block was added + BlockAdded, + // BlockFinalized + BlockFinalized, + // Some side chain was deactivated after finalization + DeactivateAfterFinalization, + // -- Synchronizer /// Synchronizer module is loaded diff --git a/tests/unit/blockchain/block_tree_test.cpp b/tests/unit/blockchain/block_tree_test.cpp new file mode 100644 index 00000000..daad1203 --- /dev/null +++ b/tests/unit/blockchain/block_tree_test.cpp @@ -0,0 +1,960 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "blockchain/impl/block_tree_impl.hpp" + +#include +#include + +#include + +#include "blockchain/block_tree_error.hpp" +#include "blockchain/impl/cached_tree.hpp" +#include "common/main_thread_pool.hpp" +#include "consensus/babe/types/babe_block_header.hpp" +#include "consensus/babe/types/seal.hpp" +#include "crypto/hasher/hasher_impl.hpp" +#include "mock/core/application/app_configuration_mock.hpp" +#include "mock/core/application/app_state_manager_mock.hpp" +#include "mock/core/blockchain/block_storage_mock.hpp" +#include "mock/core/blockchain/justification_storage_policy.hpp" +#include "mock/core/consensus/babe/babe_config_repository_mock.hpp" +#include "mock/core/storage/trie_pruner/trie_pruner_mock.hpp" +#include "mock/core/transaction_pool/transaction_pool_mock.hpp" +#include "network/impl/extrinsic_observer_impl.hpp" +#include "scale/kagome_scale.hpp" +#include "testutil/literals.hpp" +#include "testutil/prepare_loggers.hpp" + +using namespace kagome; +using application::AppConfigurationMock; +using application::AppStateManagerMock; +using blockchain::BlockStorageMock; +using blockchain::BlockTreeError; +using blockchain::BlockTreeImpl; +using blockchain::JustificationStoragePolicyMock; +using blockchain::TreeNode; +using common::Buffer; +using common::MainThreadPool; +using consensus::SlotNumber; +using consensus::babe::BabeBlockHeader; +using consensus::babe::SlotType; +using crypto::HasherImpl; +using ::kagome::scale::encode; +using network::ExtrinsicObserverImpl; +using primitives::Block; +using primitives::BlockBody; +using primitives::BlockHash; +using primitives::BlockHeader; +using primitives::BlockId; +using primitives::BlockInfo; +using primitives::BlockNumber; +using primitives::calculateBlockHash; +using primitives::Consensus; +using primitives::Digest; +using primitives::Justification; +using primitives::PreRuntime; +using storage::trie_pruner::TriePrunerMock; +using transaction_pool::TransactionPoolMock; +using BabeSeal = consensus::babe::Seal; +using Seal = primitives::Seal; + +using testing::_; +using testing::Invoke; +using testing::Return; +using testing::ReturnRef; +using testing::StrictMock; + +namespace kagome::primitives { + void PrintTo(const BlockHeader &header, std::ostream *os) { + *os << "BlockHeader {\n" + << "\tnumber: " << header.number << ",\n" + << "\tparent: " << header.parent_hash << ",\n" + << "\tstate_root: " << header.state_root << ",\n" + << "\text_root: " << header.extrinsics_root << "\n}"; + } +} // namespace kagome::primitives + +struct BlockTreeTest : public testing::Test { + static void SetUpTestCase() { + testutil::prepareLoggers(); + } + + void SetUp() override { + EXPECT_CALL(*storage_, getBlockTreeLeaves()) + .WillOnce(Return(std::vector{kFinalizedBlockInfo.hash})); + + EXPECT_CALL(*storage_, setBlockTreeLeaves(_)) + .WillRepeatedly(Return(outcome::success())); + + for (BlockNumber i = 1; i < 100; ++i) { + EXPECT_CALL(*storage_, getBlockHash(i)) + .WillRepeatedly(Return(kFirstBlockInfo.hash)); + } + + EXPECT_CALL(*storage_, hasBlockHeader(kFirstBlockInfo.hash)) + .WillRepeatedly(Return(true)); + + EXPECT_CALL(*storage_, getBlockHeader(kFirstBlockInfo.hash)) + .WillRepeatedly(Return(first_block_header_)); + + EXPECT_CALL(*storage_, getBlockHeader(kFinalizedBlockInfo.hash)) + .WillRepeatedly(Return(finalized_block_header_)); + + EXPECT_CALL(*storage_, getJustification(kFinalizedBlockInfo.hash)) + .WillRepeatedly(Return(outcome::success(Justification{}))); + + EXPECT_CALL(*storage_, getLastFinalized()) + .WillOnce(Return(outcome::success(kFinalizedBlockInfo))); + + EXPECT_CALL(*storage_, removeBlock(_)) + .WillRepeatedly(Invoke([&](const auto &hash) { + delNumToHash(hash); + return outcome::success(); + })); + + EXPECT_CALL(*storage_, getBlockHash(testing::Matcher(_))) + .WillRepeatedly(Invoke( + [&](BlockNumber n) + -> outcome::result> { + auto it = num_to_hash_.find(n); + if (it == num_to_hash_.end()) { + return BlockTreeError::HEADER_NOT_FOUND; + } + return it->second; + })); + + EXPECT_CALL(*storage_, + getBlockHeader({finalized_block_header_.parent_hash})) + .WillRepeatedly(Return(BlockTreeError::HEADER_NOT_FOUND)); + + EXPECT_CALL(*storage_, getBlockHeader(kFinalizedBlockInfo.hash)) + .WillRepeatedly(Return(finalized_block_header_)); + + EXPECT_CALL(*storage_, assignNumberToHash(_)) + .WillRepeatedly( + Invoke([&](const BlockInfo &b) -> outcome::result { + putNumToHash(b); + return outcome::success(); + })); + + EXPECT_CALL(*storage_, deassignNumberToHash(_)) + .WillRepeatedly(Invoke([&](const auto &number) { + delNumToHash(number); + return outcome::success(); + })); + + ON_CALL(*state_pruner_, recoverState(_)) + .WillByDefault(Return(outcome::success())); + + ON_CALL(*state_pruner_, schedulePrune(_, _, _)).WillByDefault(Return()); + + putNumToHash(kGenesisBlockInfo); + putNumToHash(kFinalizedBlockInfo); + + auto chain_events_engine = + std::make_shared(); + auto ext_events_engine = + std::make_shared(); + + auto extrinsic_event_key_repo = + std::make_shared(); + + block_tree_ = BlockTreeImpl::create(*app_config_, + storage_, + hasher_, + chain_events_engine, + ext_events_engine, + extrinsic_event_key_repo, + justification_storage_policy_, + state_pruner_, + *main_thread_pool_) + .value(); + } + + void TearDown() override { + watchdog_->stop(); + } + + /** + * Add a block with some data, which is a child of the top-most block + * @return block, which was added, along with its hash + */ + BlockHash addBlock(const Block &block) { + auto encoded_block = encode(block).value(); + auto hash = hasher_->blake2b_256(encoded_block); + BlockInfo block_info(block.header.number, hash); + const_cast(block.header).hash_opt.emplace(hash); + + EXPECT_CALL(*storage_, putBlock(block)) + .WillRepeatedly(Invoke([&](const auto &block) { + putNumToHash(block_info); + return hash; + })); + + // for reorganizing + EXPECT_CALL(*storage_, getBlockHeader(hash)) + .WillRepeatedly(Return(block.header)); + + EXPECT_TRUE(block_tree_->addBlock(block)); + + return hash; + } + + /** + * Creates block and add it to block tree + * @param parent - hash of parent block + * @param number - number of new block + * @param state - hash of state root + * @return hash newly created block + * @note To create different block with same number and parent, use different + * hash ot state root + */ + std::tuple addHeaderToRepositoryAndGet( + const BlockHash &parent, + BlockNumber number, + storage::trie::RootHash state, + SlotType slot_type) { + BlockHeader header; + header.parent_hash = parent; + header.number = number; + header.state_root = state; + header.digest = make_digest(number, slot_type); + calculateBlockHash(header, *hasher_); + + auto hash = addBlock(Block{header, {}}); + + // hash for header repo and number for block storage just because tests + // currently require so + EXPECT_CALL(*storage_, getBlockHeader(hash)).WillRepeatedly(Return(header)); + + return {hash, header}; + } + + uint32_t state_nonce_ = 0; + BlockHash addHeaderToRepository(const BlockHash &parent, BlockNumber number) { + Hash256 state; + memcpy(state.data(), &state_nonce_, sizeof(state_nonce_)); + ++state_nonce_; + return std::get<0>(addHeaderToRepositoryAndGet( + parent, number, state, SlotType::SecondaryPlain)); + } + + BlockHash addHeaderToRepository(const BlockHash &parent, + BlockNumber number, + SlotType slot_type) { + return std::get<0>( + addHeaderToRepositoryAndGet(parent, number, {}, slot_type)); + } + + BlockHash addHeaderToRepository(const BlockHash &parent, + BlockNumber number, + storage::trie::RootHash state) { + return std::get<0>(addHeaderToRepositoryAndGet( + parent, number, state, SlotType::SecondaryPlain)); + } + + std::shared_ptr app_config_ = + std::make_shared(); + + std::shared_ptr storage_ = + std::make_shared(); + + std::shared_ptr pool_ = + std::make_shared(); + + std::shared_ptr extrinsic_observer_ = + std::make_shared(pool_); + + std::shared_ptr hasher_ = std::make_shared(); + + std::shared_ptr + justification_storage_policy_ = + std::make_shared>(); + + std::shared_ptr state_pruner_ = + std::make_shared(); + + std::shared_ptr app_state_manager_ = + std::make_shared(); + + std::shared_ptr watchdog_ = + std::make_shared(std::chrono::milliseconds(1)); + + std::shared_ptr main_thread_pool_ = + std::make_shared( + watchdog_, std::make_shared()); + + std::shared_ptr block_tree_; + + const BlockId kLastFinalizedBlockId = kFinalizedBlockInfo.hash; + + static Digest make_digest(SlotNumber slot, + SlotType slot_type = SlotType::SecondaryPlain) { + Digest digest; + + BabeBlockHeader babe_header{ + .slot_assignment_type = slot_type, + .authority_index = 0, + .slot_number = slot, + }; + Buffer encoded_header{encode(babe_header).value()}; + digest.emplace_back( + PreRuntime{{primitives::kBabeEngineId, encoded_header}}); + + BabeSeal seal{}; + Buffer encoded_seal{encode(seal).value()}; + digest.emplace_back(Seal{{primitives::kBabeEngineId, encoded_seal}}); + + return digest; + } + + const BlockInfo kGenesisBlockInfo{ + 0ul, BlockHash::fromString("genesis_block___________________").value()}; + + BlockHeader first_block_header_{ + 1, // number + kGenesisBlockInfo.hash, // parent + {}, // state root + {}, // extrinsics root + make_digest(1), // digests + kGenesisBlockInfo.hash // hash + }; + + const BlockInfo kFirstBlockInfo{ + 1ul, BlockHash::fromString("first_block_____________________").value()}; + + const BlockInfo kFinalizedBlockInfo{ + 42ull, BlockHash::fromString("finalized_block_________________").value()}; + + BlockHeader finalized_block_header_{ + kFinalizedBlockInfo.number, // number + // parent + BlockHash::fromString("parent_of_finalized_____________").value(), + {}, // state root + {}, // extrinsics root + make_digest(kFinalizedBlockInfo.number), // digests + kFinalizedBlockInfo.hash // hash + }; + + BlockBody finalized_block_body_{{Buffer{0x22, 0x44}}, {Buffer{0x55, 0x66}}}; + + std::map num_to_hash_; + + void putNumToHash(const BlockInfo &b) { + num_to_hash_.emplace(b.number, b.hash); + } + void delNumToHash(BlockHash hash) { + auto it = std::ranges::find_if( + num_to_hash_.begin(), num_to_hash_.end(), [&](const auto &it) { + return it.second == hash; + }); + if (it == num_to_hash_.end()) { + return; + } + num_to_hash_.erase(it); + } + void delNumToHash(BlockNumber number) { + num_to_hash_.erase(number); + } + + BlockHeader makeBlockHeader(BlockNumber number, + BlockHash parent, + Digest digest) { + BlockHeader header{number, std::move(parent), {}, {}, std::move(digest)}; + calculateBlockHash(header, *hasher_); + return header; + } +}; + +/** + * @given block tree with at least one block inside + * @when requesting body of that block + * @then body is returned + */ +TEST_F(BlockTreeTest, GetBody) { + // GIVEN + // WHEN + EXPECT_CALL(*storage_, getBlockBody(kFinalizedBlockInfo.hash)) + .WillOnce(Return(finalized_block_body_)); + + // THEN + ASSERT_OUTCOME_SUCCESS(body, + block_tree_->getBlockBody(kFinalizedBlockInfo.hash)); + ASSERT_EQ(body, finalized_block_body_); +} + +/** + * @given block tree with at least one block inside + * @when adding a new block, which is a child of that block + * @then block is added + */ +TEST_F(BlockTreeTest, AddBlock) { + // GIVEN + auto &&[deepest_block_number, deepest_block_hash] = block_tree_->bestBlock(); + ASSERT_EQ(deepest_block_hash, kFinalizedBlockInfo.hash); + + auto leaves = block_tree_->getLeaves(); + ASSERT_EQ(leaves.size(), 1); + ASSERT_EQ(leaves[0], kFinalizedBlockInfo.hash); + + auto children_res = block_tree_->getChildren(kFinalizedBlockInfo.hash); + ASSERT_TRUE(children_res); + ASSERT_TRUE(children_res.value().empty()); + + // WHEN + BlockHeader header = makeBlockHeader( + kFinalizedBlockInfo.number + 1, kFinalizedBlockInfo.hash, {PreRuntime{}}); + BlockBody body{{Buffer{0x55, 0x55}}}; + Block new_block{header, body}; + auto hash = addBlock(new_block); + + // THEN + auto new_deepest_block = block_tree_->bestBlock(); + ASSERT_EQ(new_deepest_block.hash, hash); + + leaves = block_tree_->getLeaves(); + ASSERT_EQ(leaves.size(), 1); + ASSERT_EQ(leaves[0], hash); + + children_res = block_tree_->getChildren(hash); + ASSERT_TRUE(children_res); + ASSERT_TRUE(children_res.value().empty()); +} + +/** + * @given block tree with at least one block inside + * @when adding a new block, which is not a child of any block inside + * @then corresponding error is returned + */ +TEST_F(BlockTreeTest, AddBlockNoParent) { + // GIVEN + BlockHeader header = makeBlockHeader(123, {}, {PreRuntime{}}); + BlockBody body{{Buffer{0x55, 0x55}}}; + Block new_block{header, body}; + + // WHEN-THEN + ASSERT_OUTCOME_ERROR(block_tree_->addBlock(new_block), + BlockTreeError::NO_PARENT); +} + +/** + * @given block tree with at least two blocks inside + * @when finalizing a non-finalized block + * @then finalization completes successfully + */ +TEST_F(BlockTreeTest, Finalize) { + // GIVEN + auto &&last_finalized_hash = block_tree_->getLastFinalized().hash; + ASSERT_EQ(last_finalized_hash, kFinalizedBlockInfo.hash); + + BlockHeader header = makeBlockHeader( + kFinalizedBlockInfo.number + 1, kFinalizedBlockInfo.hash, {PreRuntime{}}); + BlockBody body{{Buffer{0x55, 0x55}}}; + Block new_block{header, body}; + auto hash = addBlock(new_block); + + Justification justification{{0x45, 0xF4}}; + auto encoded_justification = encode(justification).value(); + EXPECT_CALL(*storage_, getJustification(kFinalizedBlockInfo.hash)) + .WillRepeatedly(Return(outcome::success(justification))); + EXPECT_CALL(*storage_, getJustification(hash)) + .WillRepeatedly(Return(outcome::failure(boost::system::error_code{}))); + EXPECT_CALL(*storage_, putJustification(justification, hash)) + .WillRepeatedly(Return(outcome::success())); + EXPECT_CALL(*storage_, removeJustification(kFinalizedBlockInfo.hash)) + .WillRepeatedly(Return(outcome::success())); + EXPECT_CALL(*storage_, getBlockHeader(hash)) + .WillRepeatedly(Return(outcome::success(header))); + EXPECT_CALL(*storage_, getBlockBody(hash)) + .WillRepeatedly(Return(outcome::success(body))); + EXPECT_CALL(*justification_storage_policy_, + shouldStoreFor(finalized_block_header_, _)) + .WillOnce(Return(outcome::success(false))); + + // WHEN + ASSERT_OUTCOME_SUCCESS(block_tree_->finalize(hash, justification)); + + // THEN + ASSERT_EQ(block_tree_->getLastFinalized().hash, hash); +} + +/** + * @given block tree with following topology (finalized blocks marked with an + * asterisk): + * + * +---B1---C1 + * / + * ---A*---B + * + * @when finalizing non-finalized block B1 + * @then finalization completes successfully: block B pruned, block C1 persists, + * metadata valid + */ +TEST_F(BlockTreeTest, FinalizeWithPruning) { + // GIVEN + auto &&A_finalized_hash = block_tree_->getLastFinalized().hash; + ASSERT_EQ(A_finalized_hash, kFinalizedBlockInfo.hash); + + BlockHeader B_header = makeBlockHeader( + kFinalizedBlockInfo.number + 1, A_finalized_hash, {PreRuntime{}}); + BlockBody B_body{{Buffer{0x55, 0x55}}}; + Block B_block{B_header, B_body}; + auto B_hash = addBlock(B_block); + + BlockHeader B1_header = makeBlockHeader( + kFinalizedBlockInfo.number + 1, A_finalized_hash, {PreRuntime{}}); + BlockBody B1_body{{Buffer{0x55, 0x56}}}; + Block B1_block{B1_header, B1_body}; + auto B1_hash = addBlock(B1_block); + + BlockHeader C1_header = + makeBlockHeader(kFinalizedBlockInfo.number + 2, B1_hash, {PreRuntime{}}); + BlockBody C1_body{{Buffer{0x55, 0x57}}}; + Block C1_block{C1_header, C1_body}; + auto C1_hash = addBlock(C1_block); + + Justification justification{{0x45, 0xF4}}; + auto encoded_justification = encode(justification).value(); + EXPECT_CALL(*storage_, getJustification(B1_hash)) + .WillRepeatedly(Return(outcome::failure(boost::system::error_code{}))); + EXPECT_CALL(*storage_, putJustification(justification, B1_hash)) + .WillRepeatedly(Return(outcome::success())); + EXPECT_CALL(*storage_, getBlockHeader(B1_hash)) + .WillRepeatedly(Return(outcome::success(B1_header))); + EXPECT_CALL(*storage_, getBlockBody(B1_hash)) + .WillRepeatedly(Return(outcome::success(B1_body))); + EXPECT_CALL(*storage_, getBlockBody(B_hash)) + .WillRepeatedly(Return(outcome::success(B1_body))); + EXPECT_CALL(*pool_, submitExtrinsic(_, _)) + .WillRepeatedly( + Return(outcome::success(hasher_->blake2b_256(Buffer{0xaa, 0xbb})))); + EXPECT_CALL(*storage_, removeJustification(kFinalizedBlockInfo.hash)) + .WillRepeatedly(Return(outcome::success())); + EXPECT_CALL(*justification_storage_policy_, + shouldStoreFor(finalized_block_header_, _)) + .WillOnce(Return(outcome::success(false))); + + // WHEN + ASSERT_TRUE(block_tree_->finalize(B1_hash, justification)); + + // THEN + ASSERT_EQ(block_tree_->getLastFinalized().hash, B1_hash); + ASSERT_EQ(block_tree_->getLeaves().size(), 1); + ASSERT_EQ(block_tree_->bestBlock().hash, C1_hash); +} + +/** + * @given block tree with following topology (finalized blocks marked with an + * asterisk): + * + * +---B1---C1 + * / + * ---A*---B + * + * @when finalizing non-finalized block B + * @then finalization completes successfully: blocks B1, C1 pruned, metadata + * valid + */ +TEST_F(BlockTreeTest, FinalizeWithPruningDeepestLeaf) { + // GIVEN + auto &&A_finalized_hash = block_tree_->getLastFinalized().hash; + ASSERT_EQ(A_finalized_hash, kFinalizedBlockInfo.hash); + + BlockHeader B_header = makeBlockHeader( + kFinalizedBlockInfo.number + 1, A_finalized_hash, {PreRuntime{}}); + BlockBody B_body{{Buffer{0x55, 0x55}}}; + Block B_block{B_header, B_body}; + auto B_hash = addBlock(B_block); + + BlockHeader B1_header = makeBlockHeader( + kFinalizedBlockInfo.number + 1, A_finalized_hash, {PreRuntime{}}); + BlockBody B1_body{{Buffer{0x55, 0x56}}}; + Block B1_block{B1_header, B1_body}; + auto B1_hash = addBlock(B1_block); + + BlockHeader C1_header = + makeBlockHeader(kFinalizedBlockInfo.number + 2, B1_hash, {PreRuntime{}}); + BlockBody C1_body{{Buffer{0x55, 0x57}}}; + Block C1_block{C1_header, C1_body}; + auto C1_hash = addBlock(C1_block); + + Justification justification{{0x45, 0xF4}}; + auto encoded_justification = encode(justification).value(); + EXPECT_CALL(*storage_, putJustification(justification, B_hash)) + .WillRepeatedly(Return(outcome::success())); + EXPECT_CALL(*storage_, getBlockHeader(B_hash)) + .WillRepeatedly(Return(outcome::success(B_header))); + EXPECT_CALL(*storage_, getBlockBody(B_hash)) + .WillRepeatedly(Return(outcome::success(B_body))); + EXPECT_CALL(*storage_, getBlockBody(B1_hash)) + .WillRepeatedly(Return(outcome::success(B1_body))); + EXPECT_CALL(*storage_, getBlockBody(C1_hash)) + .WillRepeatedly(Return(outcome::success(C1_body))); + EXPECT_CALL(*pool_, submitExtrinsic(_, _)) + .WillRepeatedly( + Return(outcome::success(hasher_->blake2b_256(Buffer{0xaa, 0xbb})))); + EXPECT_CALL(*storage_, removeJustification(kFinalizedBlockInfo.hash)) + .WillRepeatedly(Return(outcome::success())); + EXPECT_CALL(*justification_storage_policy_, + shouldStoreFor(finalized_block_header_, _)) + .WillOnce(Return(outcome::success(false))); + + // WHEN + ASSERT_TRUE(block_tree_->finalize(B_hash, justification)); + + // THEN + ASSERT_EQ(block_tree_->getLastFinalized().hash, B_hash); + ASSERT_EQ(block_tree_->getLeaves().size(), 1); + ASSERT_EQ(block_tree_->bestBlock().hash, B_hash); +} + +std::shared_ptr makeFullTree(size_t depth, size_t branching_factor) { + auto make_subtree = [branching_factor](std::shared_ptr parent, + BlockNumber current_depth, + BlockNumber max_depth, + std::string name, + auto &make_subtree) { + BlockHash hash{}; + std::copy_n(name.begin(), name.size(), hash.begin()); + auto node = std::make_shared( + BlockInfo{hash, current_depth}, parent, false); + if (current_depth + 1 == max_depth) { + return node; + } + for (size_t i = 0; i < branching_factor; i++) { + auto child = make_subtree(node, + current_depth + 1, + max_depth, + name + "_" + std::to_string(i), + make_subtree); + node->children.push_back(child); + } + return node; + }; + return make_subtree( + std::shared_ptr{nullptr}, 0, depth, "block0", make_subtree); +} + +struct NodeProcessor { + MOCK_METHOD(void, foo, (const TreeNode &), (const)); +}; + +/** + * @given block tree with at least three blocks inside + * @when asking for chain from the given block to top + * @then expected chain is returned + */ +TEST_F(BlockTreeTest, GetChainByBlockAscending) { + // GIVEN + BlockHeader header = makeBlockHeader( + kFinalizedBlockInfo.number + 1, kFinalizedBlockInfo.hash, {PreRuntime{}}); + BlockBody body{{Buffer{0x55, 0x55}}}; + Block new_block{header, body}; + auto hash1 = addBlock(new_block); + + header = + makeBlockHeader(kFinalizedBlockInfo.number + 2, hash1, {Consensus{}}); + body = BlockBody{{Buffer{0x55, 0x55}}}; + new_block = Block{header, body}; + auto hash2 = addBlock(new_block); + + std::vector expected_chain{kFinalizedBlockInfo.hash, hash1, hash2}; + + // WHEN + ASSERT_OUTCOME_SUCCESS( + chain, block_tree_->getBestChainFromBlock(kFinalizedBlockInfo.hash, 5)); + + // THEN + ASSERT_EQ(chain, expected_chain); +} + +/** + * @given block tree with at least three blocks inside + * @when asking for chain from the given block to bottom + * @then expected chain is returned + */ +TEST_F(BlockTreeTest, GetChainByBlockDescending) { + // GIVEN + BlockHeader header = makeBlockHeader( + kFinalizedBlockInfo.number + 1, kFinalizedBlockInfo.hash, {PreRuntime{}}); + BlockBody body{{Buffer{0x55, 0x55}}}; + Block new_block{header, body}; + auto hash1 = addBlock(new_block); + + header = makeBlockHeader(header.number + 1, hash1, {Consensus{}}); + body = BlockBody{{Buffer{0x55, 0x55}}}; + new_block = Block{header, body}; + auto hash2 = addBlock(new_block); + + EXPECT_CALL(*storage_, getBlockHeader({kFinalizedBlockInfo.hash})) + .WillOnce(Return(BlockTreeError::HEADER_NOT_FOUND)); + + std::vector expected_chain{hash2, hash1}; + + // WHEN + ASSERT_OUTCOME_SUCCESS(chain, + block_tree_->getDescendingChainToBlock(hash2, 5)); + + // THEN + ASSERT_EQ(chain, expected_chain); +} + +/** + * @given a block tree with one block in it + * @when trying to obtain the best chain that contais a block, which is + * present in the storage, but is not connected to the base block in the tree + * @then BLOCK_NOT_FOUND error is returned + */ +TEST_F(BlockTreeTest, GetBestChain_DiscardedBlock) { + BlockInfo target = kFirstBlockInfo; + BlockInfo other(kFirstBlockInfo.number, "OtherBlock#1"_hash256); + EXPECT_CALL(*storage_, getBlockHash(target.number)) + .WillRepeatedly(Return(other.hash)); + + ASSERT_OUTCOME_ERROR(block_tree_->getBestContaining(target.hash), + BlockTreeError::BLOCK_ON_DEAD_END); +} + +/** + * @given a block tree with a chain with two blocks + * @when trying to obtain the best chain with the second block + * @then the second block hash is returned + */ +TEST_F(BlockTreeTest, GetBestChain_ShortChain) { + auto target_hash = addHeaderToRepository(kFinalizedBlockInfo.hash, 1337); + + ASSERT_OUTCOME_SUCCESS(best_info, + block_tree_->getBestContaining(target_hash)); + ASSERT_EQ(best_info.hash, target_hash); +} + +/** + * @given a block tree with two branches-chains + * @when trying to obtain the best chain containing the root of the split on + two + * chains + * @then the longest chain with is returned + */ +TEST_F(BlockTreeTest, GetBestChain_TwoChains) { + /* + 42 43 44 45 46 47 + + LF - T - A - B - C1 + \ + C2 - D2 + */ + + auto T_hash = addHeaderToRepository(kFinalizedBlockInfo.hash, 43); + auto A_hash = addHeaderToRepository(T_hash, 44); + auto B_hash = addHeaderToRepository(A_hash, 45); + + [[maybe_unused]] // + auto C1_hash = addHeaderToRepository(B_hash, 46); + + auto C2_hash = addHeaderToRepository(B_hash, 46); + auto D2_hash = addHeaderToRepository(C2_hash, 47); + + ASSERT_OUTCOME_SUCCESS(best_info, block_tree_->getBestContaining(T_hash)); + ASSERT_EQ(best_info.hash, D2_hash); + + // test grandpa best chain selection when target block is not on best chain + // https://github.com/paritytech/polkadot-sdk/pull/5153 + // https://github.com/paritytech/polkadot-sdk/blob/776e95748901b50ff2833a7d27ea83fd91fbf9d1/substrate/client/consensus/grandpa/src/tests.rs#L1823-L1931 + EXPECT_EQ(block_tree_->getBestContaining(C1_hash).value().hash, C1_hash); +} + +/** + * @given a block tree with two branches-chains + * @when trying to obtain the best chain containing the root of the split on + two + * chains + * @then the longest chain with is returned + */ +TEST_F(BlockTreeTest, Reorganize) { + // GIVEN + auto A_hash = addHeaderToRepository(kFinalizedBlockInfo.hash, 43); + auto B_hash = addHeaderToRepository(A_hash, 44); + + // 42 43 44 45 46 47 + // + // LF - A - B + + // WHEN.1 + auto C1_hash = addHeaderToRepository(B_hash, 45, "1"_hash256); + auto D1_hash = addHeaderToRepository(C1_hash, 46, "1"_hash256); + auto E1_hash = addHeaderToRepository(D1_hash, 47, "1"_hash256); + + // 42 43 44 45 46 47 + // + // LF - A - B - C1 - D1 - E1 + + // THEN.2 + ASSERT_TRUE(block_tree_->bestBlock() == BlockInfo(47, E1_hash)); + + // WHEN.2 + auto C2_hash = addHeaderToRepository(B_hash, 45, "2"_hash256); + auto D2_hash = addHeaderToRepository(C2_hash, 46, "2"_hash256); + auto E2_hash = addHeaderToRepository(D2_hash, 47, "2"_hash256); + + // 42 43 44 45 46 47 + // + // _C2 - D2 - E2 + // / + // LF - A - B - C1 - D1 - E1 + + // THEN.2 + ASSERT_TRUE(block_tree_->bestBlock() == BlockInfo(47, E1_hash)); + + // WHEN.3 + EXPECT_CALL(*storage_, putJustification(_, _)) + .WillOnce(Return(outcome::success())); + + EXPECT_CALL(*storage_, getBlockBody(_)) + .WillRepeatedly(Return(outcome::success(BlockBody{}))); + + EXPECT_CALL(*storage_, removeJustification(kFinalizedBlockInfo.hash)) + .WillRepeatedly(Return(outcome::success())); + EXPECT_CALL(*justification_storage_policy_, + shouldStoreFor(finalized_block_header_, _)) + .WillOnce(Return(outcome::success(false))); + + ASSERT_OUTCOME_SUCCESS(block_tree_->finalize(C2_hash, {})); + + // 42 43 44 45 46 47 + // + // LF - A - B - C2 - D2 - E2 + + // THEN.3 + ASSERT_TRUE(block_tree_->bestBlock() == BlockInfo(47, E2_hash)); +} + +TEST_F(BlockTreeTest, CleanupObsoleteJustificationOnFinalized) { + auto b43 = addHeaderToRepository(kFinalizedBlockInfo.hash, 43); + auto b55 = addHeaderToRepository(b43, 55); + auto b56 = addHeaderToRepository(b55, 56); + EXPECT_CALL(*storage_, getBlockBody(b56)).WillOnce(Return(BlockBody{})); + + Justification new_justification{"justification_56"_buf}; + + // shouldn't keep old justification + EXPECT_CALL(*justification_storage_policy_, + shouldStoreFor(finalized_block_header_, _)) + .WillOnce(Return(false)); + // store new justification + EXPECT_CALL(*storage_, putJustification(new_justification, b56)) + .WillOnce(Return(outcome::success())); + // remove old justification + EXPECT_CALL(*storage_, removeJustification(kFinalizedBlockInfo.hash)) + .WillOnce(Return(outcome::success())); + ASSERT_OUTCOME_SUCCESS(block_tree_->finalize(b56, new_justification)); +} + +TEST_F(BlockTreeTest, KeepLastFinalizedJustificationIfItShouldBeStored) { + auto b43 = addHeaderToRepository(kFinalizedBlockInfo.hash, 43); + auto b55 = addHeaderToRepository(b43, 55); + auto b56 = addHeaderToRepository(b55, 56); + EXPECT_CALL(*storage_, getBlockBody(b56)).WillOnce(Return(BlockBody{})); + + Justification new_justification{"justification_56"_buf}; + + // shouldn't keep old justification + EXPECT_CALL(*justification_storage_policy_, + shouldStoreFor(finalized_block_header_, _)) + .WillOnce(Return(true)); + // store new justification + EXPECT_CALL(*storage_, putJustification(new_justification, b56)) + .WillOnce(Return(outcome::success())); + ASSERT_OUTCOME_SUCCESS(block_tree_->finalize(b56, new_justification)); +} + +/** + * @given a block tree with two branches-chains + * @when trying to obtain the best chain containing the root of the split on + two + * chains + * @then the longest chain with is returned + */ +TEST_F(BlockTreeTest, GetBestBlock) { + auto T_hash = addHeaderToRepository(kFinalizedBlockInfo.hash, 43); + auto A_hash = addHeaderToRepository(T_hash, 44); + auto B_hash = addHeaderToRepository(A_hash, 45); + + [[maybe_unused]] // + auto C1_hash = addHeaderToRepository(B_hash, 46); + + auto C2_hash = addHeaderToRepository(B_hash, 46); + auto D2_hash = addHeaderToRepository(C2_hash, 47); + + auto C3_hash = addHeaderToRepository(B_hash, 46); + auto D3_hash = addHeaderToRepository(C3_hash, 47); + auto E3_hash = addHeaderToRepository(D3_hash, 48); + auto F3_hash = addHeaderToRepository(E3_hash, 49); + + // 42 43 44 45 46 47 48 49 50 + // + // _C1 + // / + // LF - T - A - B - C2 - D2 + // \_ + // C3 - D3 - E3 - F3 + + { + ASSERT_OUTCOME_SUCCESS(best_info, block_tree_->getBestContaining(T_hash)); + ASSERT_EQ(best_info.hash, F3_hash); + } + + // --------------------------------------------------------------------------- + + auto E2_hash = addHeaderToRepository(D2_hash, 48, SlotType::Primary); + + // 42 43 44 45 46 47 48 49 50 + // + // _C1 + // / + // LF - T - A - B - C2 - D2 - E2* + // \_ + // C3 - D3 - E3 - F3 + + { + ASSERT_OUTCOME_SUCCESS(best_info, block_tree_->getBestContaining(T_hash)); + ASSERT_EQ(best_info.hash, E2_hash); + } + + // --------------------------------------------------------------------------- + + auto G3_hash = addHeaderToRepository(F3_hash, 50, SlotType::Primary); + + // 42 43 44 45 46 47 48 49 50 + // + // _C1 + // / + // LF - T - A - B - C2 - D2 - E2* + // \_ + // C3 - D3 - E3 - F3 - G3** + + { + ASSERT_OUTCOME_SUCCESS(best_info, block_tree_->getBestContaining(T_hash)); + ASSERT_EQ(best_info.hash, G3_hash); + } + + // --------------------------------------------------------------------------- + + ASSERT_OUTCOME_SUCCESS(block_tree_->markAsRevertedBlocks({E3_hash})); + + // 42 43 44 45 46 47 48 49 50 + // + // _C1 + // / + // LF - T - A - B - C2 - D2 - E2* + // \_ + // C3 - D3 - E3 - F3 - G3** + + { + ASSERT_OUTCOME_SUCCESS(best_info, block_tree_->getBestContaining(T_hash)); + ASSERT_EQ(best_info.hash, E2_hash); + } +} From fd9f2c0406d320c9d92987c892c253996abd0f5a Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Fri, 23 May 2025 02:00:57 +0300 Subject: [PATCH 10/14] feature: block tree tests --- .../justification_storage_policy_mock.hpp | 26 + tests/unit/blockchain/CMakeLists.txt | 26 +- tests/unit/blockchain/block_tree_test.cpp | 528 +++++++++--------- 3 files changed, 321 insertions(+), 259 deletions(-) create mode 100644 tests/mock/blockchain/justification_storage_policy_mock.hpp diff --git a/tests/mock/blockchain/justification_storage_policy_mock.hpp b/tests/mock/blockchain/justification_storage_policy_mock.hpp new file mode 100644 index 00000000..3bbdd6ec --- /dev/null +++ b/tests/mock/blockchain/justification_storage_policy_mock.hpp @@ -0,0 +1,26 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include "blockchain/impl/justification_storage_policy.hpp" + +namespace jam::blockchain { + + class JustificationStoragePolicyMock : public JustificationStoragePolicy { + public: + MOCK_METHOD(outcome::result, + shouldStoreFor, + (const BlockHeader &header, + BlockNumber last_finalized_number), + (const, override)); + + private: + }; + +} // namespace kagome::blockchain diff --git a/tests/unit/blockchain/CMakeLists.txt b/tests/unit/blockchain/CMakeLists.txt index ed52e570..d86c53d4 100644 --- a/tests/unit/blockchain/CMakeLists.txt +++ b/tests/unit/blockchain/CMakeLists.txt @@ -11,4 +11,28 @@ target_link_libraries(block_storage_test blockchain #??? logger_for_tests storage - ) \ No newline at end of file + ) + +addtest(block_tree_test + block_tree_test.cpp +) +target_link_libraries(block_tree_test + blockchain +# block_tree +# network +# consensus + hasher + app_configuration + logger_for_tests +# dummy_error + qtils::qtils + se_async +) + +#addtest(justification_storage_policy_test +# justification_storage_policy_test.cpp +# ) +#target_link_libraries(justification_storage_policy_test +# blockchain +# scale::scale +# ) diff --git a/tests/unit/blockchain/block_tree_test.cpp b/tests/unit/blockchain/block_tree_test.cpp index daad1203..b0a78961 100644 --- a/tests/unit/blockchain/block_tree_test.cpp +++ b/tests/unit/blockchain/block_tree_test.cpp @@ -4,63 +4,70 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include "blockchain/impl/block_tree_impl.hpp" - #include #include +#include #include +#include "app/configuration.hpp" #include "blockchain/block_tree_error.hpp" +#include "blockchain/impl/block_tree_impl.hpp" #include "blockchain/impl/cached_tree.hpp" -#include "common/main_thread_pool.hpp" -#include "consensus/babe/types/babe_block_header.hpp" -#include "consensus/babe/types/seal.hpp" +// #include "common/main_thread_pool.hpp" +#include "blockchain/impl/block_storage_initializer.hpp" +// #include "consensus/babe/types/babe_block_header.hpp" +// #include "consensus/babe/types/seal.hpp" #include "crypto/hasher/hasher_impl.hpp" -#include "mock/core/application/app_configuration_mock.hpp" -#include "mock/core/application/app_state_manager_mock.hpp" -#include "mock/core/blockchain/block_storage_mock.hpp" -#include "mock/core/blockchain/justification_storage_policy.hpp" -#include "mock/core/consensus/babe/babe_config_repository_mock.hpp" -#include "mock/core/storage/trie_pruner/trie_pruner_mock.hpp" -#include "mock/core/transaction_pool/transaction_pool_mock.hpp" -#include "network/impl/extrinsic_observer_impl.hpp" -#include "scale/kagome_scale.hpp" +// #include "mock/core/app/configuration_mock.hpp" +#include "mock/app/state_manager_mock.hpp" +#include "mock/blockchain/block_storage_mock.hpp" +#include "mock/blockchain/justification_storage_policy_mock.hpp" +// #include "mock/core/consensus/babe/babe_config_repository_mock.hpp" +// #include "mock/core/storage/trie_pruner/trie_pruner_mock.hpp" +// #include "mock/core/transaction_pool/transaction_pool_mock.hpp" +// #include "network/impl/extrinsic_observer_impl.hpp" +// #include "scale/kagome_scale.hpp" +#include + +#include "jam_types/justification.hpp" +#include "se/subscription.hpp" #include "testutil/literals.hpp" #include "testutil/prepare_loggers.hpp" -using namespace kagome; -using application::AppConfigurationMock; -using application::AppStateManagerMock; -using blockchain::BlockStorageMock; -using blockchain::BlockTreeError; -using blockchain::BlockTreeImpl; -using blockchain::JustificationStoragePolicyMock; -using blockchain::TreeNode; -using common::Buffer; -using common::MainThreadPool; -using consensus::SlotNumber; -using consensus::babe::BabeBlockHeader; -using consensus::babe::SlotType; -using crypto::HasherImpl; -using ::kagome::scale::encode; -using network::ExtrinsicObserverImpl; -using primitives::Block; -using primitives::BlockBody; -using primitives::BlockHash; -using primitives::BlockHeader; -using primitives::BlockId; -using primitives::BlockInfo; -using primitives::BlockNumber; -using primitives::calculateBlockHash; -using primitives::Consensus; -using primitives::Digest; -using primitives::Justification; -using primitives::PreRuntime; -using storage::trie_pruner::TriePrunerMock; -using transaction_pool::TransactionPoolMock; -using BabeSeal = consensus::babe::Seal; -using Seal = primitives::Seal; +using jam::Block; +using jam::BlockBody; +using jam::BlockHash; +using jam::BlockHeader; +using jam::BlockId; +using jam::BlockInfo; +using jam::BlockNumber; +using jam::calculateBlockHash; +using jam::encode; +using jam::TimeSlot; +using jam::app::Configuration; +using jam::app::StateManagerMock; +using jam::blockchain::BlockStorageMock; +using jam::blockchain::BlockTreeError; +using jam::blockchain::BlockTreeImpl; +using jam::blockchain::BlockTreeInitializer; +using jam::blockchain::JustificationStoragePolicyMock; +using jam::blockchain::TreeNode; +// using jam::consensus::babe::BabeBlockHeader; +// using jam::consensus::babe::SlotType; +using jam::crypto::HasherImpl; +// using jam::network::ExtrinsicObserverImpl; +// using jam::primitives::Consensus; +// using jam::primitives::Digest; +using jam::Justification; +// using jam::primitives::PreRuntime; +// using jam::storage::trie_pruner::TriePrunerMock; +// using jam::transaction_pool::TransactionPoolMock; +using qtils::ByteVec; +using qtils::literals::operator""_vec; +// using BabeSeal = consensus::babe::Seal; +// using Seal = primitives::Seal; +using RootHash = jam::OpaqueHash; using testing::_; using testing::Invoke; @@ -71,28 +78,32 @@ using testing::StrictMock; namespace kagome::primitives { void PrintTo(const BlockHeader &header, std::ostream *os) { *os << "BlockHeader {\n" - << "\tnumber: " << header.number << ",\n" - << "\tparent: " << header.parent_hash << ",\n" - << "\tstate_root: " << header.state_root << ",\n" - << "\text_root: " << header.extrinsics_root << "\n}"; + << "\tslot: " << header.slot << ",\n" + << "\tparent: " << header.parent << ",\n" + << "\tparent_state_root: " << header.parent_state_root << ",\n" + << "\textrinsic_hash: " << header.extrinsic_hash << "\n}"; } } // namespace kagome::primitives -struct BlockTreeTest : public testing::Test { +struct BlockTreeTest : testing::Test { static void SetUpTestCase() { testutil::prepareLoggers(); } + static void TearDownTestCase() {} void SetUp() override { + auto dispatcher = std::make_shared>(); + se_ = std::make_shared(dispatcher); + EXPECT_CALL(*storage_, getBlockTreeLeaves()) - .WillOnce(Return(std::vector{kFinalizedBlockInfo.hash})); + .WillOnce(Return(std::vector{kFinalizedBlockInfo.hash})); EXPECT_CALL(*storage_, setBlockTreeLeaves(_)) .WillRepeatedly(Return(outcome::success())); for (BlockNumber i = 1; i < 100; ++i) { EXPECT_CALL(*storage_, getBlockHash(i)) - .WillRepeatedly(Return(kFirstBlockInfo.hash)); + .WillRepeatedly(Return(std::vector{kFirstBlockInfo.hash})); } EXPECT_CALL(*storage_, hasBlockHeader(kFirstBlockInfo.hash)) @@ -112,71 +123,67 @@ struct BlockTreeTest : public testing::Test { EXPECT_CALL(*storage_, removeBlock(_)) .WillRepeatedly(Invoke([&](const auto &hash) { - delNumToHash(hash); + delSlotToHash(hash); return outcome::success(); })); EXPECT_CALL(*storage_, getBlockHash(testing::Matcher(_))) .WillRepeatedly(Invoke( - [&](BlockNumber n) - -> outcome::result> { - auto it = num_to_hash_.find(n); - if (it == num_to_hash_.end()) { + [&](BlockNumber n) -> outcome::result> { + auto it = slot_to_hash_.find(n); + if (it == slot_to_hash_.end()) { return BlockTreeError::HEADER_NOT_FOUND; } - return it->second; + return std::vector{it->second}; })); - EXPECT_CALL(*storage_, - getBlockHeader({finalized_block_header_.parent_hash})) + EXPECT_CALL(*storage_, getBlockHeader({finalized_block_header_.parent})) .WillRepeatedly(Return(BlockTreeError::HEADER_NOT_FOUND)); EXPECT_CALL(*storage_, getBlockHeader(kFinalizedBlockInfo.hash)) .WillRepeatedly(Return(finalized_block_header_)); - EXPECT_CALL(*storage_, assignNumberToHash(_)) + EXPECT_CALL(*storage_, assignHashToSlot(_)) .WillRepeatedly( Invoke([&](const BlockInfo &b) -> outcome::result { - putNumToHash(b); + putSlotToHash(b); return outcome::success(); })); - EXPECT_CALL(*storage_, deassignNumberToHash(_)) - .WillRepeatedly(Invoke([&](const auto &number) { - delNumToHash(number); - return outcome::success(); - })); + EXPECT_CALL(*storage_, deassignHashToSlot(_)) + .WillRepeatedly( + Invoke([&](const BlockInfo &b) -> outcome::result { + delSlotToHash(b); + return outcome::success(); + })); - ON_CALL(*state_pruner_, recoverState(_)) - .WillByDefault(Return(outcome::success())); + // ON_CALL(*state_pruner_, recoverState(_)) + // .WillByDefault(Return(outcome::success())); - ON_CALL(*state_pruner_, schedulePrune(_, _, _)).WillByDefault(Return()); + // ON_CALL(*state_pruner_, schedulePrune(_, _, _)).WillByDefault(Return()); - putNumToHash(kGenesisBlockInfo); - putNumToHash(kFinalizedBlockInfo); + putSlotToHash(kGenesisBlockInfo); + putSlotToHash(kFinalizedBlockInfo); - auto chain_events_engine = - std::make_shared(); - auto ext_events_engine = - std::make_shared(); + auto logsys = testutil::prepareLoggers(); - auto extrinsic_event_key_repo = - std::make_shared(); + // Initialize the block tree + qtils::SharedRef initializer = + std::make_shared(logsys, storage_); - block_tree_ = BlockTreeImpl::create(*app_config_, - storage_, - hasher_, - chain_events_engine, - ext_events_engine, - extrinsic_event_key_repo, - justification_storage_policy_, - state_pruner_, - *main_thread_pool_) - .value(); + block_tree_ = std::make_shared(logsys, + app_config_, + storage_, + hasher_, + justification_storage_policy_, + se_, + initializer); } void TearDown() override { - watchdog_->stop(); + // watchdog_->stop(); + se_->dispose(); + se_.reset(); } /** @@ -186,12 +193,12 @@ struct BlockTreeTest : public testing::Test { BlockHash addBlock(const Block &block) { auto encoded_block = encode(block).value(); auto hash = hasher_->blake2b_256(encoded_block); - BlockInfo block_info(block.header.number, hash); + BlockInfo block_info(block.header.slot, hash); const_cast(block.header).hash_opt.emplace(hash); EXPECT_CALL(*storage_, putBlock(block)) .WillRepeatedly(Invoke([&](const auto &block) { - putNumToHash(block_info); + putSlotToHash(block_info); return hash; })); @@ -207,22 +214,23 @@ struct BlockTreeTest : public testing::Test { /** * Creates block and add it to block tree * @param parent - hash of parent block - * @param number - number of new block - * @param state - hash of state root + * @param slot - slot of new block + * @param state - hash of parent state root + * @param is_primary - true if block created by ticket winning * @return hash newly created block - * @note To create different block with same number and parent, use different + * @note To create different block with same slot and parent, use different * hash ot state root */ std::tuple addHeaderToRepositoryAndGet( - const BlockHash &parent, - BlockNumber number, - storage::trie::RootHash state, - SlotType slot_type) { + const BlockHash &parent, TimeSlot slot, RootHash state, bool is_primary) { BlockHeader header; - header.parent_hash = parent; - header.number = number; - header.state_root = state; - header.digest = make_digest(number, slot_type); + header.parent = parent; + header.slot = slot; + header.parent_state_root = state; + + // Trick with fake extrinsic hash to mark a block producing way + header.extrinsic_hash = is_primary ? "ticket"_arr32 : "fallback"_arr32; + calculateBlockHash(header, *hasher_); auto hash = addBlock(Block{header, {}}); @@ -236,38 +244,40 @@ struct BlockTreeTest : public testing::Test { uint32_t state_nonce_ = 0; BlockHash addHeaderToRepository(const BlockHash &parent, BlockNumber number) { - Hash256 state; + RootHash state; memcpy(state.data(), &state_nonce_, sizeof(state_nonce_)); ++state_nonce_; - return std::get<0>(addHeaderToRepositoryAndGet( - parent, number, state, SlotType::SecondaryPlain)); + return std::get<0>( + addHeaderToRepositoryAndGet(parent, number, state, false)); } BlockHash addHeaderToRepository(const BlockHash &parent, BlockNumber number, - SlotType slot_type) { + bool is_primary) { return std::get<0>( - addHeaderToRepositoryAndGet(parent, number, {}, slot_type)); + addHeaderToRepositoryAndGet(parent, number, {}, is_primary)); } BlockHash addHeaderToRepository(const BlockHash &parent, BlockNumber number, - storage::trie::RootHash state) { - return std::get<0>(addHeaderToRepositoryAndGet( - parent, number, state, SlotType::SecondaryPlain)); + RootHash state) { + return std::get<0>( + addHeaderToRepositoryAndGet(parent, number, state, false)); } - std::shared_ptr app_config_ = - std::make_shared(); + std::shared_ptr se_; + + std::shared_ptr app_config_ = + std::make_shared(); std::shared_ptr storage_ = std::make_shared(); - std::shared_ptr pool_ = - std::make_shared(); + // std::shared_ptr pool_ = + // std::make_shared(); - std::shared_ptr extrinsic_observer_ = - std::make_shared(pool_); + // std::shared_ptr extrinsic_observer_ = + // std::make_shared(pool_); std::shared_ptr hasher_ = std::make_shared(); @@ -275,54 +285,42 @@ struct BlockTreeTest : public testing::Test { justification_storage_policy_ = std::make_shared>(); - std::shared_ptr state_pruner_ = - std::make_shared(); - - std::shared_ptr app_state_manager_ = - std::make_shared(); + // std::shared_ptr state_pruner_ = + // std::make_shared(); - std::shared_ptr watchdog_ = - std::make_shared(std::chrono::milliseconds(1)); - - std::shared_ptr main_thread_pool_ = - std::make_shared( - watchdog_, std::make_shared()); + std::shared_ptr app_state_manager_ = + std::make_shared(); std::shared_ptr block_tree_; const BlockId kLastFinalizedBlockId = kFinalizedBlockInfo.hash; - static Digest make_digest(SlotNumber slot, - SlotType slot_type = SlotType::SecondaryPlain) { - Digest digest; - - BabeBlockHeader babe_header{ - .slot_assignment_type = slot_type, - .authority_index = 0, - .slot_number = slot, - }; - Buffer encoded_header{encode(babe_header).value()}; - digest.emplace_back( - PreRuntime{{primitives::kBabeEngineId, encoded_header}}); - - BabeSeal seal{}; - Buffer encoded_seal{encode(seal).value()}; - digest.emplace_back(Seal{{primitives::kBabeEngineId, encoded_seal}}); - - return digest; - } + // static Digest make_digest(SlotNumber slot, + // SlotType slot_type = SlotType::SecondaryPlain) { + // Digest digest; + // + // BabeBlockHeader babe_header{ + // .slot_assignment_type = slot_type, + // .authority_index = 0, + // .slot_number = slot, + // }; + // ByteVec encoded_header{encode(babe_header).value()}; + // digest.emplace_back( + // PreRuntime{{primitives::kBabeEngineId, encoded_header}}); + // + // BabeSeal seal{}; + // ByteVec encoded_seal{encode(seal).value()}; + // digest.emplace_back(Seal{{primitives::kBabeEngineId, encoded_seal}}); + // + // return digest; + // } const BlockInfo kGenesisBlockInfo{ 0ul, BlockHash::fromString("genesis_block___________________").value()}; - BlockHeader first_block_header_{ - 1, // number - kGenesisBlockInfo.hash, // parent - {}, // state root - {}, // extrinsics root - make_digest(1), // digests - kGenesisBlockInfo.hash // hash - }; + BlockHeader first_block_header_{.parent = kGenesisBlockInfo.hash, + .slot = 1, + .hash_opt = kGenesisBlockInfo.hash}; const BlockInfo kFirstBlockInfo{ 1ul, BlockHash::fromString("first_block_____________________").value()}; @@ -331,40 +329,60 @@ struct BlockTreeTest : public testing::Test { 42ull, BlockHash::fromString("finalized_block_________________").value()}; BlockHeader finalized_block_header_{ - kFinalizedBlockInfo.number, // number - // parent - BlockHash::fromString("parent_of_finalized_____________").value(), - {}, // state root - {}, // extrinsics root - make_digest(kFinalizedBlockInfo.number), // digests - kFinalizedBlockInfo.hash // hash - }; - - BlockBody finalized_block_body_{{Buffer{0x22, 0x44}}, {Buffer{0x55, 0x66}}}; + .parent = + BlockHash::fromString("parent_of_finalized_____________").value(), + .slot = kFinalizedBlockInfo.slot, + .hash_opt = kFinalizedBlockInfo.hash}; - std::map num_to_hash_; + BlockBody finalized_block_body_{ + //{ByteVec{0x22, 0x44}}, {ByteVec{0x55, 0x66}} + }; - void putNumToHash(const BlockInfo &b) { - num_to_hash_.emplace(b.number, b.hash); + std::map> slot_to_hash_; + + void putSlotToHash(const BlockInfo &b) { + auto it = slot_to_hash_.find(b.slot); + if (it == slot_to_hash_.end()) { + slot_to_hash_.emplace(b.slot, std::vector{b.hash}); + } else { + auto &hashes = it->second; + if (not qtils::cxx23::ranges::contains(hashes, b.hash)) { + hashes.emplace_back(b.hash); + } + } } - void delNumToHash(BlockHash hash) { - auto it = std::ranges::find_if( - num_to_hash_.begin(), num_to_hash_.end(), [&](const auto &it) { - return it.second == hash; - }); - if (it == num_to_hash_.end()) { + void delSlotToHash(const BlockInfo &b) { + auto it = slot_to_hash_.find(b.slot); + if (it == slot_to_hash_.end()) { return; + ; + } + auto &hashes = it->second; + auto to_erase = std::ranges::remove(hashes, b.hash); + if (not to_erase.empty()) { + hashes.erase(to_erase.begin(), to_erase.end()); + if (hashes.empty()) { + slot_to_hash_.erase(it); + } } - num_to_hash_.erase(it); } - void delNumToHash(BlockNumber number) { - num_to_hash_.erase(number); + void delSlotToHash(const BlockHash &hash) { + for (auto it = slot_to_hash_.begin(); it != slot_to_hash_.end();) { + auto &hashes = it->second; + auto to_erase = std::ranges::remove(hashes, hash); + if (not to_erase.empty()) { + hashes.erase(to_erase.begin(), to_erase.end()); + if (hashes.empty()) { + it = slot_to_hash_.erase(it); + continue; + } + } + ++it; + } } - BlockHeader makeBlockHeader(BlockNumber number, - BlockHash parent, - Digest digest) { - BlockHeader header{number, std::move(parent), {}, {}, std::move(digest)}; + BlockHeader makeBlockHeader(TimeSlot slot, BlockHash parent) const { + BlockHeader header{.parent = parent, .slot = slot}; calculateBlockHash(header, *hasher_); return header; } @@ -389,7 +407,7 @@ TEST_F(BlockTreeTest, GetBody) { /** * @given block tree with at least one block inside - * @when adding a new block, which is a child of that block + * @when adding a new block, which is a child of that block. * @then block is added */ TEST_F(BlockTreeTest, AddBlock) { @@ -406,10 +424,8 @@ TEST_F(BlockTreeTest, AddBlock) { ASSERT_TRUE(children_res.value().empty()); // WHEN - BlockHeader header = makeBlockHeader( - kFinalizedBlockInfo.number + 1, kFinalizedBlockInfo.hash, {PreRuntime{}}); - BlockBody body{{Buffer{0x55, 0x55}}}; - Block new_block{header, body}; + Block new_block{.header = makeBlockHeader(kFinalizedBlockInfo.slot + 1, + kFinalizedBlockInfo.hash)}; auto hash = addBlock(new_block); // THEN @@ -432,9 +448,7 @@ TEST_F(BlockTreeTest, AddBlock) { */ TEST_F(BlockTreeTest, AddBlockNoParent) { // GIVEN - BlockHeader header = makeBlockHeader(123, {}, {PreRuntime{}}); - BlockBody body{{Buffer{0x55, 0x55}}}; - Block new_block{header, body}; + Block new_block{.header = makeBlockHeader(123, {})}; // WHEN-THEN ASSERT_OUTCOME_ERROR(block_tree_->addBlock(new_block), @@ -451,10 +465,8 @@ TEST_F(BlockTreeTest, Finalize) { auto &&last_finalized_hash = block_tree_->getLastFinalized().hash; ASSERT_EQ(last_finalized_hash, kFinalizedBlockInfo.hash); - BlockHeader header = makeBlockHeader( - kFinalizedBlockInfo.number + 1, kFinalizedBlockInfo.hash, {PreRuntime{}}); - BlockBody body{{Buffer{0x55, 0x55}}}; - Block new_block{header, body}; + Block new_block{.header = makeBlockHeader(kFinalizedBlockInfo.slot + 1, + kFinalizedBlockInfo.hash)}; auto hash = addBlock(new_block); Justification justification{{0x45, 0xF4}}; @@ -468,9 +480,9 @@ TEST_F(BlockTreeTest, Finalize) { EXPECT_CALL(*storage_, removeJustification(kFinalizedBlockInfo.hash)) .WillRepeatedly(Return(outcome::success())); EXPECT_CALL(*storage_, getBlockHeader(hash)) - .WillRepeatedly(Return(outcome::success(header))); + .WillRepeatedly(Return(outcome::success(new_block.header))); EXPECT_CALL(*storage_, getBlockBody(hash)) - .WillRepeatedly(Return(outcome::success(body))); + .WillRepeatedly(Return(outcome::success(new_block.extrinsic))); EXPECT_CALL(*justification_storage_policy_, shouldStoreFor(finalized_block_header_, _)) .WillOnce(Return(outcome::success(false))); @@ -491,7 +503,8 @@ TEST_F(BlockTreeTest, Finalize) { * ---A*---B * * @when finalizing non-finalized block B1 - * @then finalization completes successfully: block B pruned, block C1 persists, + * @then finalization completes successfully: block B pruned, block C1 + persists, * metadata valid */ TEST_F(BlockTreeTest, FinalizeWithPruning) { @@ -499,21 +512,21 @@ TEST_F(BlockTreeTest, FinalizeWithPruning) { auto &&A_finalized_hash = block_tree_->getLastFinalized().hash; ASSERT_EQ(A_finalized_hash, kFinalizedBlockInfo.hash); - BlockHeader B_header = makeBlockHeader( - kFinalizedBlockInfo.number + 1, A_finalized_hash, {PreRuntime{}}); - BlockBody B_body{{Buffer{0x55, 0x55}}}; + BlockHeader B_header = + makeBlockHeader(kFinalizedBlockInfo.slot + 1, A_finalized_hash); + BlockBody B_body{.preimages = {{.blob = {1}}}}; Block B_block{B_header, B_body}; auto B_hash = addBlock(B_block); - BlockHeader B1_header = makeBlockHeader( - kFinalizedBlockInfo.number + 1, A_finalized_hash, {PreRuntime{}}); - BlockBody B1_body{{Buffer{0x55, 0x56}}}; + BlockHeader B1_header = + makeBlockHeader(kFinalizedBlockInfo.slot + 1, A_finalized_hash); + BlockBody B1_body{.preimages = {{.blob = {2}}}}; Block B1_block{B1_header, B1_body}; auto B1_hash = addBlock(B1_block); BlockHeader C1_header = - makeBlockHeader(kFinalizedBlockInfo.number + 2, B1_hash, {PreRuntime{}}); - BlockBody C1_body{{Buffer{0x55, 0x57}}}; + makeBlockHeader(kFinalizedBlockInfo.slot + 2, B1_hash); + BlockBody C1_body{.preimages = {{.blob = {3}}}}; Block C1_block{C1_header, C1_body}; auto C1_hash = addBlock(C1_block); @@ -529,9 +542,10 @@ TEST_F(BlockTreeTest, FinalizeWithPruning) { .WillRepeatedly(Return(outcome::success(B1_body))); EXPECT_CALL(*storage_, getBlockBody(B_hash)) .WillRepeatedly(Return(outcome::success(B1_body))); - EXPECT_CALL(*pool_, submitExtrinsic(_, _)) - .WillRepeatedly( - Return(outcome::success(hasher_->blake2b_256(Buffer{0xaa, 0xbb})))); + // EXPECT_CALL(*pool_, submitExtrinsic(_, _)) + // .WillRepeatedly( + // Return(outcome::success(hasher_->blake2b_256(ByteVec{0xaa, + // 0xbb})))); EXPECT_CALL(*storage_, removeJustification(kFinalizedBlockInfo.hash)) .WillRepeatedly(Return(outcome::success())); EXPECT_CALL(*justification_storage_policy_, @@ -548,8 +562,8 @@ TEST_F(BlockTreeTest, FinalizeWithPruning) { } /** - * @given block tree with following topology (finalized blocks marked with an - * asterisk): + * @given block tree with the following topology (finalized blocks marked with + * an asterisk): * * +---B1---C1 * / @@ -564,21 +578,21 @@ TEST_F(BlockTreeTest, FinalizeWithPruningDeepestLeaf) { auto &&A_finalized_hash = block_tree_->getLastFinalized().hash; ASSERT_EQ(A_finalized_hash, kFinalizedBlockInfo.hash); - BlockHeader B_header = makeBlockHeader( - kFinalizedBlockInfo.number + 1, A_finalized_hash, {PreRuntime{}}); - BlockBody B_body{{Buffer{0x55, 0x55}}}; + BlockHeader B_header = + makeBlockHeader(kFinalizedBlockInfo.slot + 1, A_finalized_hash); + BlockBody B_body{.preimages = {{.blob = {1}}}}; Block B_block{B_header, B_body}; auto B_hash = addBlock(B_block); - BlockHeader B1_header = makeBlockHeader( - kFinalizedBlockInfo.number + 1, A_finalized_hash, {PreRuntime{}}); - BlockBody B1_body{{Buffer{0x55, 0x56}}}; + BlockHeader B1_header = + makeBlockHeader(kFinalizedBlockInfo.slot + 1, A_finalized_hash); + BlockBody B1_body{.preimages = {{.blob = {2}}}}; Block B1_block{B1_header, B1_body}; auto B1_hash = addBlock(B1_block); BlockHeader C1_header = - makeBlockHeader(kFinalizedBlockInfo.number + 2, B1_hash, {PreRuntime{}}); - BlockBody C1_body{{Buffer{0x55, 0x57}}}; + makeBlockHeader(kFinalizedBlockInfo.slot + 2, B1_hash); + BlockBody C1_body{.preimages = {{.blob = {3}}}}; Block C1_block{C1_header, C1_body}; auto C1_hash = addBlock(C1_block); @@ -594,9 +608,10 @@ TEST_F(BlockTreeTest, FinalizeWithPruningDeepestLeaf) { .WillRepeatedly(Return(outcome::success(B1_body))); EXPECT_CALL(*storage_, getBlockBody(C1_hash)) .WillRepeatedly(Return(outcome::success(C1_body))); - EXPECT_CALL(*pool_, submitExtrinsic(_, _)) - .WillRepeatedly( - Return(outcome::success(hasher_->blake2b_256(Buffer{0xaa, 0xbb})))); + // EXPECT_CALL(*pool_, submitExtrinsic(_, _)) + // .WillRepeatedly( + // Return(outcome::success(hasher_->blake2b_256(ByteVec{0xaa, + // 0xbb})))); EXPECT_CALL(*storage_, removeJustification(kFinalizedBlockInfo.hash)) .WillRepeatedly(Return(outcome::success())); EXPECT_CALL(*justification_storage_policy_, @@ -612,16 +627,17 @@ TEST_F(BlockTreeTest, FinalizeWithPruningDeepestLeaf) { ASSERT_EQ(block_tree_->bestBlock().hash, B_hash); } -std::shared_ptr makeFullTree(size_t depth, size_t branching_factor) { - auto make_subtree = [branching_factor](std::shared_ptr parent, - BlockNumber current_depth, - BlockNumber max_depth, - std::string name, - auto &make_subtree) { +std::shared_ptr makeFullTree(size_t depth, + const size_t branching_factor) { + auto make_subtree = [&branching_factor](std::shared_ptr parent, + BlockNumber current_depth, + BlockNumber max_depth, + std::string name, + auto &make_subtree) { BlockHash hash{}; std::copy_n(name.begin(), name.size(), hash.begin()); auto node = std::make_shared( - BlockInfo{hash, current_depth}, parent, false); + BlockInfo{current_depth, hash}, parent, false); if (current_depth + 1 == max_depth) { return node; } @@ -645,20 +661,19 @@ struct NodeProcessor { /** * @given block tree with at least three blocks inside - * @when asking for chain from the given block to top + * @when asking for a chain from the given block to top one * @then expected chain is returned */ TEST_F(BlockTreeTest, GetChainByBlockAscending) { // GIVEN - BlockHeader header = makeBlockHeader( - kFinalizedBlockInfo.number + 1, kFinalizedBlockInfo.hash, {PreRuntime{}}); - BlockBody body{{Buffer{0x55, 0x55}}}; + BlockHeader header = + makeBlockHeader(kFinalizedBlockInfo.slot + 1, kFinalizedBlockInfo.hash); + BlockBody body{.preimages = {{.blob = {0}}}}; Block new_block{header, body}; auto hash1 = addBlock(new_block); - header = - makeBlockHeader(kFinalizedBlockInfo.number + 2, hash1, {Consensus{}}); - body = BlockBody{{Buffer{0x55, 0x55}}}; + header = makeBlockHeader(kFinalizedBlockInfo.slot + 2, hash1); + body = BlockBody{.preimages = {{.blob = {0}}}}; new_block = Block{header, body}; auto hash2 = addBlock(new_block); @@ -679,14 +694,14 @@ TEST_F(BlockTreeTest, GetChainByBlockAscending) { */ TEST_F(BlockTreeTest, GetChainByBlockDescending) { // GIVEN - BlockHeader header = makeBlockHeader( - kFinalizedBlockInfo.number + 1, kFinalizedBlockInfo.hash, {PreRuntime{}}); - BlockBody body{{Buffer{0x55, 0x55}}}; + BlockHeader header = + makeBlockHeader(kFinalizedBlockInfo.slot + 1, kFinalizedBlockInfo.hash); + BlockBody body{.preimages = {{.blob = {0}}}}; Block new_block{header, body}; auto hash1 = addBlock(new_block); - header = makeBlockHeader(header.number + 1, hash1, {Consensus{}}); - body = BlockBody{{Buffer{0x55, 0x55}}}; + header = makeBlockHeader(header.slot + 1, hash1); + body = BlockBody{.preimages = {{.blob = {0}}}}; new_block = Block{header, body}; auto hash2 = addBlock(new_block); @@ -711,9 +726,9 @@ TEST_F(BlockTreeTest, GetChainByBlockDescending) { */ TEST_F(BlockTreeTest, GetBestChain_DiscardedBlock) { BlockInfo target = kFirstBlockInfo; - BlockInfo other(kFirstBlockInfo.number, "OtherBlock#1"_hash256); - EXPECT_CALL(*storage_, getBlockHash(target.number)) - .WillRepeatedly(Return(other.hash)); + BlockInfo other(kFirstBlockInfo.slot, "OtherBlock#1"_arr32); + EXPECT_CALL(*storage_, getBlockHash(target.slot)) + .WillRepeatedly(Return(std::vector{other.hash})); ASSERT_OUTCOME_ERROR(block_tree_->getBestContaining(target.hash), BlockTreeError::BLOCK_ON_DEAD_END); @@ -734,8 +749,7 @@ TEST_F(BlockTreeTest, GetBestChain_ShortChain) { /** * @given a block tree with two branches-chains - * @when trying to obtain the best chain containing the root of the split on - two + * @when trying to obtain the best chain containing the root of the split on two * chains * @then the longest chain with is returned */ @@ -769,8 +783,7 @@ TEST_F(BlockTreeTest, GetBestChain_TwoChains) { /** * @given a block tree with two branches-chains - * @when trying to obtain the best chain containing the root of the split on - two + * @when trying to obtain the best chain containing the root of the split on two * chains * @then the longest chain with is returned */ @@ -784,9 +797,9 @@ TEST_F(BlockTreeTest, Reorganize) { // LF - A - B // WHEN.1 - auto C1_hash = addHeaderToRepository(B_hash, 45, "1"_hash256); - auto D1_hash = addHeaderToRepository(C1_hash, 46, "1"_hash256); - auto E1_hash = addHeaderToRepository(D1_hash, 47, "1"_hash256); + auto C1_hash = addHeaderToRepository(B_hash, 45, "1"_arr32); + auto D1_hash = addHeaderToRepository(C1_hash, 46, "1"_arr32); + auto E1_hash = addHeaderToRepository(D1_hash, 47, "1"_arr32); // 42 43 44 45 46 47 // @@ -796,9 +809,9 @@ TEST_F(BlockTreeTest, Reorganize) { ASSERT_TRUE(block_tree_->bestBlock() == BlockInfo(47, E1_hash)); // WHEN.2 - auto C2_hash = addHeaderToRepository(B_hash, 45, "2"_hash256); - auto D2_hash = addHeaderToRepository(C2_hash, 46, "2"_hash256); - auto E2_hash = addHeaderToRepository(D2_hash, 47, "2"_hash256); + auto C2_hash = addHeaderToRepository(B_hash, 45, "2"_arr32); + auto D2_hash = addHeaderToRepository(C2_hash, 46, "2"_arr32); + auto E2_hash = addHeaderToRepository(D2_hash, 47, "2"_arr32); // 42 43 44 45 46 47 // @@ -838,7 +851,7 @@ TEST_F(BlockTreeTest, CleanupObsoleteJustificationOnFinalized) { auto b56 = addHeaderToRepository(b55, 56); EXPECT_CALL(*storage_, getBlockBody(b56)).WillOnce(Return(BlockBody{})); - Justification new_justification{"justification_56"_buf}; + Justification new_justification{"justification_56"_vec}; // shouldn't keep old justification EXPECT_CALL(*justification_storage_policy_, @@ -859,7 +872,7 @@ TEST_F(BlockTreeTest, KeepLastFinalizedJustificationIfItShouldBeStored) { auto b56 = addHeaderToRepository(b55, 56); EXPECT_CALL(*storage_, getBlockBody(b56)).WillOnce(Return(BlockBody{})); - Justification new_justification{"justification_56"_buf}; + Justification new_justification{"justification_56"_vec}; // shouldn't keep old justification EXPECT_CALL(*justification_storage_policy_, @@ -873,8 +886,7 @@ TEST_F(BlockTreeTest, KeepLastFinalizedJustificationIfItShouldBeStored) { /** * @given a block tree with two branches-chains - * @when trying to obtain the best chain containing the root of the split on - two + * @when trying to obtain the best chain containing the root of the split on two * chains * @then the longest chain with is returned */ @@ -909,7 +921,7 @@ TEST_F(BlockTreeTest, GetBestBlock) { // --------------------------------------------------------------------------- - auto E2_hash = addHeaderToRepository(D2_hash, 48, SlotType::Primary); + auto E2_hash = addHeaderToRepository(D2_hash, 48, true); // 42 43 44 45 46 47 48 49 50 // @@ -926,7 +938,7 @@ TEST_F(BlockTreeTest, GetBestBlock) { // --------------------------------------------------------------------------- - auto G3_hash = addHeaderToRepository(F3_hash, 50, SlotType::Primary); + auto G3_hash = addHeaderToRepository(F3_hash, 50, true); // 42 43 44 45 46 47 48 49 50 // From f2c343b0770878ad346e225e60029f5cbfdfe1a9 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Mon, 23 Jun 2025 10:20:00 +0300 Subject: [PATCH 11/14] fix: CI problems Signed-off-by: Dmitriy Khaustov aka xDimon --- src/app/impl/chain_spec_impl.hpp | 2 +- src/blockchain/block_storage_error.hpp | 4 +++- src/blockchain/block_tree.hpp | 7 +------ src/blockchain/block_tree_error.hpp | 1 + src/blockchain/impl/block_tree_impl.cpp | 3 ++- src/blockchain/impl/block_tree_impl.hpp | 17 ++++++++++------- src/storage/storage_error.hpp | 7 ++++--- 7 files changed, 22 insertions(+), 19 deletions(-) diff --git a/src/app/impl/chain_spec_impl.hpp b/src/app/impl/chain_spec_impl.hpp index 1818b9b9..ce1583a4 100644 --- a/src/app/impl/chain_spec_impl.hpp +++ b/src/app/impl/chain_spec_impl.hpp @@ -25,7 +25,7 @@ namespace jam::app { class ChainSpecImpl : public ChainSpec { public: - enum class Error { + enum class Error : uint8_t { MISSING_ENTRY = 1, MISSING_PEER_ID, PARSER_ERROR, diff --git a/src/blockchain/block_storage_error.hpp b/src/blockchain/block_storage_error.hpp index 7ce5ecf4..fcacfcd0 100644 --- a/src/blockchain/block_storage_error.hpp +++ b/src/blockchain/block_storage_error.hpp @@ -7,16 +7,18 @@ #pragma once #include +#include namespace jam::blockchain { +#undef BlockStorageError enum class BlockStorageError : uint8_t { BLOCK_EXISTS = 1, HEADER_NOT_FOUND, GENESIS_BLOCK_ALREADY_EXISTS, GENESIS_BLOCK_NOT_FOUND, FINALIZED_BLOCK_NOT_FOUND, - BLOCK_TREE_LEAVES_NOT_FOUND + BLOCK_TREE_LEAVES_NOT_FOUND, }; } diff --git a/src/blockchain/block_tree.hpp b/src/blockchain/block_tree.hpp index ffd82b1b..b56030bd 100644 --- a/src/blockchain/block_tree.hpp +++ b/src/blockchain/block_tree.hpp @@ -153,14 +153,9 @@ namespace jam::blockchain { virtual outcome::result finalize( const BlockHash &block, const Justification &justification) = 0; - // enum class GetChainDirection : uint8_t { - // ASCEND, - // DESCEND, - // }; - /** * Get a chain of blocks from provided block to direction of the best block - * @param block, from which the chain is started + * @param block from which the chain is started * @param maximum number of blocks to be retrieved * @return chain or blocks or error */ diff --git a/src/blockchain/block_tree_error.hpp b/src/blockchain/block_tree_error.hpp index 546ee0e4..4bc32e45 100644 --- a/src/blockchain/block_tree_error.hpp +++ b/src/blockchain/block_tree_error.hpp @@ -7,6 +7,7 @@ #pragma once #include +#include namespace jam::blockchain { diff --git a/src/blockchain/impl/block_tree_impl.cpp b/src/blockchain/impl/block_tree_impl.cpp index d2ae868a..3043219c 100644 --- a/src/blockchain/impl/block_tree_impl.cpp +++ b/src/blockchain/impl/block_tree_impl.cpp @@ -9,7 +9,8 @@ // #include // #include // #include -// #include +#include +#include #include diff --git a/src/blockchain/impl/block_tree_impl.hpp b/src/blockchain/impl/block_tree_impl.hpp index 67baabe1..aa8c1ad9 100644 --- a/src/blockchain/impl/block_tree_impl.hpp +++ b/src/blockchain/impl/block_tree_impl.hpp @@ -237,12 +237,14 @@ namespace jam::blockchain { == std::this_thread::get_id()) { return f(block_tree_data_.unsafeGet()); } - return block_tree_data_.exclusiveAccess([&f, - this](BlockTreeData &data) { - exclusive_owner_ = std::this_thread::get_id(); - qtils::FinalAction reset([&] { exclusive_owner_ = std::nullopt; }); - return f(data); - }); + return block_tree_data_.exclusiveAccess( + [&f, this](BlockTreeData &data) { + exclusive_owner_ = std::this_thread::get_id(); + qtils::FinalAction reset([&] { + exclusive_owner_ = decltype(std::this_thread::get_id()){}; + }); + return f(data); + }); } template @@ -258,7 +260,8 @@ namespace jam::blockchain { private: se::utils::SafeObject block_tree_data_; - std::atomic> exclusive_owner_; + std::atomic exclusive_owner_ = + decltype(std::this_thread::get_id()){}; }; log::Logger log_; diff --git a/src/storage/storage_error.hpp b/src/storage/storage_error.hpp index 4c6b82ed..5136b8e6 100644 --- a/src/storage/storage_error.hpp +++ b/src/storage/storage_error.hpp @@ -14,6 +14,7 @@ #pragma once #include +#include namespace jam::storage { @@ -23,7 +24,7 @@ namespace jam::storage { * Defines common error conditions returned by storage operations, * such as missing entries, corruption, or IO failures. */ - enum class StorageError : int { // NOLINT(performance-enum-size) + enum class StorageError : uint8_t { // NOLINT(performance-enum-size) OK = 0, ///< success (no error) @@ -31,11 +32,11 @@ namespace jam::storage { CORRUPTION = 2, ///< data corruption in storage INVALID_ARGUMENT = 3, ///< invalid argument to storage IO_ERROR = 4, ///< IO error in storage - NOT_FOUND = 5, ///< entry not found in storage + NOT_FOUND = 5, ///< entry isn't found in storage DB_PATH_NOT_CREATED = 6, ///< storage path was not created STORAGE_GONE = 7, ///< storage instance has been uninitialized - UNKNOWN = 1000, ///< unknown error + UNKNOWN = 255, ///< unknown error }; } // namespace jam::storage From d357f692c413f968fe9bee25f7a9a2498ea5f582 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Tue, 1 Jul 2025 17:01:38 +0300 Subject: [PATCH 12/14] fix: review issues Signed-off-by: Dmitriy Khaustov aka xDimon --- src/blockchain/impl/block_storage_impl.cpp | 8 ++++---- src/blockchain/impl/storage_util.cpp | 4 ++-- src/storage/rocksdb/rocksdb_spaces.cpp | 2 +- src/storage/spaces.hpp | 10 +++++----- src/third_party/keccak/CMakeLists.txt | 1 - tests/unit/blockchain/block_storage_test.cpp | 14 ++++++++------ vcpkg.json | 2 +- 7 files changed, 21 insertions(+), 20 deletions(-) diff --git a/src/blockchain/impl/block_storage_impl.cpp b/src/blockchain/impl/block_storage_impl.cpp index 88c059d6..788c0b14 100644 --- a/src/blockchain/impl/block_storage_impl.cpp +++ b/src/blockchain/impl/block_storage_impl.cpp @@ -96,7 +96,7 @@ namespace jam::blockchain { const BlockInfo &block_index) { SL_DEBUG(logger_, "Add slot-to-hash for {}", block_index); auto slot_to_hash_key = slotToHashLookupKey(block_index.slot); - auto storage = storage_->getSpace(storage::Space::LookupKey); + auto storage = storage_->getSpace(storage::Space::SlotToHashes); OUTCOME_TRY(hashes, getBlockHash(block_index.slot)); if (not qtils::cxx23::ranges::contains(hashes, block_index.hash)) { hashes.emplace_back(block_index.hash); @@ -109,7 +109,7 @@ namespace jam::blockchain { const BlockIndex &block_index) { SL_DEBUG(logger_, "Remove num-to-idx for {}", block_index); auto slot_to_hash_key = slotToHashLookupKey(block_index.slot); - auto storage = storage_->getSpace(storage::Space::LookupKey); + auto storage = storage_->getSpace(storage::Space::SlotToHashes); OUTCOME_TRY(hashes, getBlockHash(block_index.slot)); auto to_erase = std::ranges::remove(hashes, block_index.hash); if (not to_erase.empty()) { @@ -125,7 +125,7 @@ namespace jam::blockchain { outcome::result> BlockStorageImpl::getBlockHash( TimeSlot slot) const { - auto storage = storage_->getSpace(storage::Space::LookupKey); + auto storage = storage_->getSpace(storage::Space::SlotToHashes); OUTCOME_TRY(data_opt, storage->tryGet(slotToHashLookupKey(slot))); if (data_opt.has_value()) { return decode>(data_opt.value()); @@ -300,7 +300,7 @@ namespace jam::blockchain { { // Remove slot-to-hash assigning auto num_to_hash_key = slotToHashLookupKey(block_index.slot); - auto key_space = storage_->getSpace(storage::Space::LookupKey); + auto key_space = storage_->getSpace(storage::Space::SlotToHashes); OUTCOME_TRY(hash_opt, key_space->tryGet(num_to_hash_key.view())); if (hash_opt == block_hash) { if (auto res = key_space->remove(num_to_hash_key); res.has_error()) { diff --git a/src/blockchain/impl/storage_util.cpp b/src/blockchain/impl/storage_util.cpp index eb61bc71..eab8f6c7 100644 --- a/src/blockchain/impl/storage_util.cpp +++ b/src/blockchain/impl/storage_util.cpp @@ -24,7 +24,7 @@ namespace jam::blockchain { block_id, [&](const BlockNumber &block_number) -> outcome::result> { - auto key_space = storage.getSpace(storage::Space::LookupKey); + auto key_space = storage.getSpace(storage::Space::SlotToHashes); return key_space->tryGet(slotToHashLookupKey(block_number)); }, [](const BlockHash &block_hash) { @@ -34,7 +34,7 @@ namespace jam::blockchain { outcome::result> blockHashByNumber( storage::SpacedStorage &storage, BlockNumber block_number) { - auto key_space = storage.getSpace(storage::Space::LookupKey); + auto key_space = storage.getSpace(storage::Space::SlotToHashes); OUTCOME_TRY(data_opt, key_space->tryGet(slotToHashLookupKey(block_number))); if (data_opt.has_value()) { OUTCOME_TRY(hash, BlockHash::fromSpan(data_opt.value())); diff --git a/src/storage/rocksdb/rocksdb_spaces.cpp b/src/storage/rocksdb/rocksdb_spaces.cpp index c7b82478..65b32eaa 100644 --- a/src/storage/rocksdb/rocksdb_spaces.cpp +++ b/src/storage/rocksdb/rocksdb_spaces.cpp @@ -18,7 +18,7 @@ namespace jam::storage { // Names of non-default space static constexpr std::string_view kNamesArr[] = { - "lookup_key", + "slot_to_hash", "header", "extrinsic", "justification", diff --git a/src/storage/spaces.hpp b/src/storage/spaces.hpp index 98e96ac3..fd313f0f 100644 --- a/src/storage/spaces.hpp +++ b/src/storage/spaces.hpp @@ -29,16 +29,16 @@ namespace jam::storage { */ enum class Space : uint8_t { Default = 0, ///< Default space used for general-purpose storage - LookupKey, ///< Space used for mapping lookup keys // application-defined spaces - Header, - Extrinsic, - Justification, + SlotToHashes, ///< mapping time slot to block hashes + Header, ///< mapping block hash to header + Extrinsic, ///< mapping block hash to extrinsic + Justification, ///< mapping block hash to finality justification // ... append here Total ///< Total number of defined spaces (must be last) }; constexpr size_t SpacesCount = static_cast(Space::Total); -} +} // namespace jam::storage diff --git a/src/third_party/keccak/CMakeLists.txt b/src/third_party/keccak/CMakeLists.txt index 3c452a59..6f2f6eba 100644 --- a/src/third_party/keccak/CMakeLists.txt +++ b/src/third_party/keccak/CMakeLists.txt @@ -9,6 +9,5 @@ add_library(keccak ) set_source_files_properties(keccak.с PROPERTIES LANGUAGE C) target_link_libraries(keccak - OpenSSL::Crypto qtils::qtils ) diff --git a/tests/unit/blockchain/block_storage_test.cpp b/tests/unit/blockchain/block_storage_test.cpp index 4319ddb2..4eefbc01 100644 --- a/tests/unit/blockchain/block_storage_test.cpp +++ b/tests/unit/blockchain/block_storage_test.cpp @@ -23,8 +23,8 @@ #include #include "qtils/error_throw.hpp" -#include "testutil/prepare_loggers.hpp" #include "testutil/literals.hpp" +#include "testutil/prepare_loggers.hpp" using jam::Block; using jam::BlockBody; @@ -60,11 +60,13 @@ class BlockStorageTest : public testing::Test { hasher = std::make_shared(); spaced_storage = std::make_shared(); - std::set required_spaces = {Space::Default, - Space::Header, - Space::Justification, - Space::Extrinsic, - Space::LookupKey}; + std::set required_spaces = { + Space::Default, + Space::SlotToHashes, + Space::Header, + Space::Extrinsic, + Space::Justification, + }; for (auto space : required_spaces) { auto storage = std::make_shared(); diff --git a/vcpkg.json b/vcpkg.json index ca86b6d2..003311ef 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -16,7 +16,7 @@ "ftxui", "rocksdb", "boost-property-tree", - "openssl", + "boringssl", "xxhash" ], "features": { From 1878d723b5c55995a695a156cf8aa9a044fc8a30 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Fri, 23 May 2025 02:01:10 +0300 Subject: [PATCH 13/14] for tests --- src/app/impl/application_impl.cpp | 9 +++++++-- src/app/impl/application_impl.hpp | 13 ++++++++++--- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/app/impl/application_impl.cpp b/src/app/impl/application_impl.cpp index d17649d9..43cb468e 100644 --- a/src/app/impl/application_impl.cpp +++ b/src/app/impl/application_impl.cpp @@ -32,8 +32,13 @@ namespace jam::app { qtils::SharedRef state_manager, qtils::SharedRef watchdog, qtils::SharedRef metrics_exposer, - qtils::SharedRef system_clock, - std::shared_ptr) + qtils::SharedRef system_clock + + , + qtils::SharedRef + // qtils::SharedRef + + ) : logger_(logsys->getLogger("Application", "application")), app_config_(std::move(config)), state_manager_(std::move(state_manager)), diff --git a/src/app/impl/application_impl.hpp b/src/app/impl/application_impl.hpp index 73dcf5e2..a053b669 100644 --- a/src/app/impl/application_impl.hpp +++ b/src/app/impl/application_impl.hpp @@ -8,9 +8,11 @@ #include +#include +#include #include -#include +#include "app/application.hpp" #include "app/application.hpp" #include "se/subscription_fwd.hpp" @@ -82,8 +84,13 @@ namespace jam::app { qtils::SharedRef state_manager, qtils::SharedRef watchdog, qtils::SharedRef metrics_exposer, - qtils::SharedRef system_clock, - std::shared_ptr); + qtils::SharedRef system_clock + + , + qtils::SharedRef + // qtils::SharedRef + + ); void run() override; From 3e89675587838a8fe3d44e294370553b3ea1c3a3 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Tue, 1 Jul 2025 18:09:33 +0300 Subject: [PATCH 14/14] fix: ci Signed-off-by: Dmitriy Khaustov aka xDimon --- .ci/.env | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.ci/.env b/.ci/.env index d17091bb..dd32c273 100644 --- a/.ci/.env +++ b/.ci/.env @@ -14,7 +14,8 @@ LINUX_PACKAGES="make \ python3-pip \ unzip \ zip \ - nano" + nano \ + nasm" MACOS_PACKAGES="make \ autoconf \ @@ -27,7 +28,9 @@ MACOS_PACKAGES="make \ pkg-config \ python@3.12 \ unzip \ - zip" + zip \ + nasm \ + go" CMAKE_VERSION=3.31.1 DEBIAN_FRONTEND=noninteractive