From 3e142816bcd506bcec0bf89b13dd9f11c28c2fe4 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Sat, 11 Jan 2025 03:21:25 +0800 Subject: [PATCH 01/45] draft Signed-off-by: Dmitriy Khaustov aka xDimon --- CMakeLists.txt | 15 +- python/asn1.py | 4 +- src/CMakeLists.txt | 9 +- src/apps/CMakeLists.txt | 38 ++++ src/apps/dlopen.c | 35 +++ src/apps/jam_node.cpp | 209 ++++++++++++++++++ src/log/CMakeLists.txt | 26 +++ src/log/configurator.cpp | 173 +++++++++++++++ src/log/configurator.hpp | 31 +++ src/log/formatters/filepath.hpp | 18 ++ src/log/formatters/optional.hpp | 36 +++ src/log/formatters/ref_and_ptr.hpp | 136 ++++++++++++ src/log/formatters/variant.hpp | 40 ++++ src/log/logger.cpp | 166 ++++++++++++++ src/log/logger.hpp | 56 +++++ src/log/profiling_logger.cpp | 12 + src/log/profiling_logger.hpp | 73 ++++++ src/log/trace_macros.hpp | 75 +++++++ src_/CMakeLists.txt | 7 + {src => src_}/TODO_qtils/bytes_std_hash.hpp | 0 .../TODO_qtils/cxx23/ranges/contains.hpp | 0 {src => src_}/jam/CMakeLists.txt | 0 {src => src_}/jam/bandersnatch.hpp | 0 {src => src_}/jam/blake.hpp | 0 {src => src_}/jam/ed25519.hpp | 0 {src => src_}/jam/empty.hpp | 0 {src => src_}/jam/keccak.hpp | 0 {src => src_}/jam/tagged.hpp | 0 {src => src_}/jam/unused.hpp | 0 tests/CMakeLists.txt | 7 + vcpkg-overlay/soralog/portfile.cmake | 11 + vcpkg-overlay/soralog/vcpkg.json | 18 ++ vcpkg.json | 6 +- 33 files changed, 1191 insertions(+), 10 deletions(-) create mode 100644 src/apps/CMakeLists.txt create mode 100644 src/apps/dlopen.c create mode 100644 src/apps/jam_node.cpp create mode 100644 src/log/CMakeLists.txt create mode 100644 src/log/configurator.cpp create mode 100644 src/log/configurator.hpp create mode 100644 src/log/formatters/filepath.hpp create mode 100644 src/log/formatters/optional.hpp create mode 100644 src/log/formatters/ref_and_ptr.hpp create mode 100644 src/log/formatters/variant.hpp create mode 100644 src/log/logger.cpp create mode 100644 src/log/logger.hpp create mode 100644 src/log/profiling_logger.cpp create mode 100644 src/log/profiling_logger.hpp create mode 100644 src/log/trace_macros.hpp create mode 100644 src_/CMakeLists.txt rename {src => src_}/TODO_qtils/bytes_std_hash.hpp (100%) rename {src => src_}/TODO_qtils/cxx23/ranges/contains.hpp (100%) rename {src => src_}/jam/CMakeLists.txt (100%) rename {src => src_}/jam/bandersnatch.hpp (100%) rename {src => src_}/jam/blake.hpp (100%) rename {src => src_}/jam/ed25519.hpp (100%) rename {src => src_}/jam/empty.hpp (100%) rename {src => src_}/jam/keccak.hpp (100%) rename {src => src_}/jam/tagged.hpp (100%) rename {src => src_}/jam/unused.hpp (100%) create mode 100644 tests/CMakeLists.txt create mode 100644 vcpkg-overlay/soralog/portfile.cmake create mode 100644 vcpkg-overlay/soralog/vcpkg.json diff --git a/CMakeLists.txt b/CMakeLists.txt index 3f50388..9da8994 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,14 +11,15 @@ if(TESTING) list(APPEND VCPKG_MANIFEST_FEATURES test) endif() -project(cpp-jam - VERSION 0.0.1 - LANGUAGES C CXX -) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) +project(cpp-jam + VERSION 0.0.1 + LANGUAGES CXX +) + find_package(Python3 REQUIRED) find_package(PkgConfig REQUIRED) @@ -28,14 +29,17 @@ find_package(Boost CONFIG REQUIRED) find_package(fmt CONFIG REQUIRED) find_package(jam_crust CONFIG REQUIRED) find_package(scale CONFIG REQUIRED) +find_package(soralog CONFIG REQUIRED) find_package(schnorrkel_crust CONFIG REQUIRED) add_library(headers INTERFACE) target_include_directories(headers INTERFACE - $ + $ $ ) +add_subdirectory(src_) + add_subdirectory(src) if(TESTING) @@ -45,4 +49,5 @@ if(TESTING) set(GTEST_DEPS GTest::gtest GTest::gtest_main GTest::gmock GTest::gmock_main) add_subdirectory(test-vectors) + add_subdirectory(tests) endif () diff --git a/python/asn1.py b/python/asn1.py index 1459057..ee4e364 100755 --- a/python/asn1.py +++ b/python/asn1.py @@ -406,8 +406,8 @@ def __init__(self, cpp_namespace: str, path: str): "", "#include ", "#include ", - "#include ", - "#include ", + "#include ", + "#include ", "", *self.g_types, ] diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 185da66..3a6ad77 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -4,4 +4,11 @@ # SPDX-License-Identifier: Apache-2.0 # -add_subdirectory(jam) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) + +# Applications (should contain `main()` function) +add_subdirectory(apps) + +# Logging subsystem +add_subdirectory(log) + diff --git a/src/apps/CMakeLists.txt b/src/apps/CMakeLists.txt new file mode 100644 index 0000000..c2445bc --- /dev/null +++ b/src/apps/CMakeLists.txt @@ -0,0 +1,38 @@ +# +# Copyright Quadrivium LLC +# All Rights Reserved +# SPDX-License-Identifier: Apache-2.0 +# + +# fix for dyld bug on mac +# we should actually check the target platform here, not the host platform, +# but not that we explicitly support cross-compilation anyway +if (CMAKE_HOST_APPLE AND CMAKE_BUILD_TYPE STREQUAL "Debug") + add_library(jam_node SHARED jam_node.cpp) + set_target_properties(jam_node PROPERTIES PREFIX "" DEBUG_POSTFIX "") + + add_executable(jam_node_dlopen dlopen.c) + set_target_properties(jam_node_dlopen PROPERTIES OUTPUT_NAME jam_node) + set_target_properties(jam_node_dlopen PROPERTIES LINKER_LANGUAGE CXX) +else () + add_executable(jam_node jam_node.cpp) +endif () + +target_link_libraries(jam_node + Boost::boost +# Boost::program_options +# kagome_application +# app_config +# fd_limit +# p2p::p2p_identify +# p2p::p2p_ping +# kagome-benchmark +# kagome-db-editor +# storage_explorer +# filesystem +# kagome_key + ) + +if (BACKWARD) + add_backward(jam_node) +endif () diff --git a/src/apps/dlopen.c b/src/apps/dlopen.c new file mode 100644 index 0000000..23b712d --- /dev/null +++ b/src/apps/dlopen.c @@ -0,0 +1,35 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include + +int main(int argc, char **argv) { + uint32_t size = 0; + _NSGetExecutablePath(NULL, &size); + char *exe = (char *)malloc(size); + _NSGetExecutablePath(exe, &size); + + const char *suffix = ".dylib"; + char *dylib = (char *)malloc(strlen(exe) + strlen(suffix) + 1); + strcpy(dylib, exe); + strcpy(dylib + strlen(exe), suffix); + + void *lib = dlopen(dylib, RTLD_NOW); + if (lib == NULL) { + printf("dlopen: %s\n", dlerror()); + return -1; + } + void *sym = dlsym(lib, "main"); + if (sym == NULL) { + printf("dlsym: %s\n", dlerror()); + return -1; + } + return ((int (*)(int, char **))sym)(argc, argv); +} diff --git a/src/apps/jam_node.cpp b/src/apps/jam_node.cpp new file mode 100644 index 0000000..cd1d3ce --- /dev/null +++ b/src/apps/jam_node.cpp @@ -0,0 +1,209 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +#if defined(BACKWARD_HAS_BACKTRACE) +#include +#endif + +#undef TRUE +#undef FALSE + +// #include +// #include +// +// #include "application/impl/app_configuration_impl.hpp" +// #include "application/impl/kagome_application_impl.hpp" +// #include "common/fd_limit.hpp" +// #include "injector/application_injector.hpp" +// #include "log/configurator.hpp" +#include "log/logger.hpp" +// #include "parachain/pvf/kagome_pvf_worker.hpp" + +// NOLINTBEGIN(cppcoreguidelines-pro-bounds-pointer-arithmetic) + +// using kagome::application::AppConfiguration; +// using kagome::application::AppConfigurationImpl; +// +// int storage_explorer_main(int argc, const char **argv); +// int db_editor_main(int argc, const char **argv); +// +// namespace kagome { +// int benchmark_main(int argc, const char **argv); +// int key_main(int argc, const char **argv); +// } +// +// namespace { +// int run_node(int argc, const char **argv) { +// auto configuration = std::make_shared(); +// +// if (not configuration->initializeFromArgs(argc, argv)) { +// return EXIT_FAILURE; +// } +// +// kagome::log::tuneLoggingSystem(configuration->log()); +// +// auto injector = +// std::make_unique(configuration); +// +// kagome::common::setFdLimit(SIZE_MAX); +// +// auto app = +// std::make_shared(*injector); +// +// if (configuration->subcommand().has_value()) { +// switch (configuration->subcommand().value()) { +// using kagome::application::Subcommand; +// case Subcommand::ChainInfo: +// return app->chainInfo(); +// } +// } +// +// if (configuration->precompileWasm()) { +// return app->precompileWasm(); +// } +// +// // Recovery mode +// if (configuration->recoverState().has_value()) { +// return app->recovery(); +// } +// +// auto logger = +// kagome::log::createLogger("Main", kagome::log::defaultGroupName); +// +// SL_INFO( +// logger, "Kagome started. Version: {} ", configuration->nodeVersion()); +// +// app->run(); +// +// SL_INFO(logger, "Kagome stopped"); +// logger->flush(); +// +// return EXIT_SUCCESS; +// } +// +// void wrong_usage() { +// std::cerr << "Wrong usage.\n" +// "Available subcommands: storage-explorer db-editor benchmark\n" +// "Run with `--help' argument to print usage\n"; +// } +// +// } // namespace + +int main(int argc, const char **argv, const char **env) { +#if defined(BACKWARD_HAS_BACKTRACE) + backward::SignalHandling sh; +#endif + + // // Needed for zombienet + // setvbuf(stdout, nullptr, _IOLBF, 0); + // setvbuf(stderr, nullptr, _IOLBF, 0); + + // libp2p::common::FinalAction flush_std_streams_at_exit([] { + // std::cout.flush(); + // std::cerr.flush(); + // }); + + // if (argc > 1) { + // std::string_view name{argv[1]}; + // if (name == "pvf-worker") { + // return kagome::parachain::pvf_worker_main(argc - 1, argv + 1, env); + // } + // if (name == "check-secure-mode") { + // return kagome::parachain::secureModeCheckMain(argc, argv); + // } + // } + + soralog::util::setThreadName("kagome"); + + // Logging system + auto logging_system = [&] { + auto custom_log_config_path = + kagome::log::Configurator::getLogConfigFile(argc, argv); + if (custom_log_config_path.has_value()) { + if (not std::filesystem::is_regular_file( + custom_log_config_path.value())) { + std::cerr << "Provided wrong path to logger config file\n"; + exit(EXIT_FAILURE); + } + } + + auto libp2p_log_configurator = + std::make_shared(); + + auto kagome_log_configurator = + custom_log_config_path.has_value() + ? std::make_shared( + std::move(libp2p_log_configurator), + custom_log_config_path.value()) + : std::make_shared( + std::move(libp2p_log_configurator)); + + return std::make_shared( + std::move(kagome_log_configurator)); + }(); + + auto r = logging_system->configure(); + if (not r.message.empty()) { + (r.has_error ? std::cerr : std::cout) << r.message << '\n'; + } + if (r.has_error) { + return EXIT_FAILURE; + } + + kagome::log::setLoggingSystem(logging_system); + + int exit_code = EXIT_FAILURE; + + if (argc == 0) { + // Abnormal run + wrong_usage(); + } + + else if (argc == 1) { + // Run without arguments + wrong_usage(); + } + + else { + std::string_view name{argv[1]}; + + if (name == "storage-explorer") { + exit_code = storage_explorer_main(argc - 1, argv + 1); + } + + else if (name == "db-editor") { + exit_code = db_editor_main(argc - 1, argv + 1); + } + + else if (name == "benchmark") { + exit_code = kagome::benchmark_main(argc - 1, argv + 1); + } + + else if (name == "key") { + exit_code = kagome::key_main(argc - 1, argv + 1); + } + + else if (name.substr(0, 1) == "-") { + // The first argument isn't subcommand, run as node + exit_code = run_node(argc, argv); + + } else { + // No subcommand, but argument is not a valid option: begins not with dash + wrong_usage(); + } + } + + auto logger = + kagome::log::createLogger("Main", kagome::log::defaultGroupName); + SL_INFO(logger, "All components are stopped"); + logger->flush(); + + return exit_code; +} + +// NOLINTEND(cppcoreguidelines-pro-bounds-pointer-arithmetic) diff --git a/src/log/CMakeLists.txt b/src/log/CMakeLists.txt new file mode 100644 index 0000000..6a0381e --- /dev/null +++ b/src/log/CMakeLists.txt @@ -0,0 +1,26 @@ +# +# Copyright Quadrivium LLC +# All Rights Reserved +# SPDX-License-Identifier: Apache-2.0 +# + +#add_library(logger +# logger.cpp +# profiling_logger.cpp +# ) +#target_link_libraries(logger +# fmt::fmt +# soralog::soralog +# p2p::p2p_logger +# clock +# ) +#kagome_install(logger) +# +#add_library(log_configurator +# configurator.cpp +# ) +#target_link_libraries(log_configurator +# soralog::configurator_yaml +# Boost::program_options +# ) +#kagome_install(log_configurator) diff --git a/src/log/configurator.cpp b/src/log/configurator.cpp new file mode 100644 index 0000000..e3a91dd --- /dev/null +++ b/src/log/configurator.cpp @@ -0,0 +1,173 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "log/configurator.hpp" + +#include + +namespace kagome::log { + + namespace { + // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) + std::string embedded_config(R"( +# ---------------- +sinks: + - name: console + type: console + stream: stderr + thread: name + color: false + latency: 0 +groups: + - name: main + sink: console + level: info + is_fallback: true + children: + - name: libp2p + level: off + - name: kagome + children: + - name: profile + - name: injector + - name: application + - name: rpc + children: + - name: rpc_transport + - name: api + children: + - name: author_api + - name: authorship + - name: blockchain + children: + - name: block_tree + - name: block_storage + - name: digest_tracker + - name: offchain + - name: authority + - name: crypto + children: + - name: bip39 + - name: key_store + - name: ed25519 + - name: ecdsa + - name: consensus + children: + - name: timeline + - name: babe + children: + - name: babe_lottery + - name: block_appender + - name: block_executor + - name: block_validator + - name: babe_config_repo + - name: grandpa + children: + - name: voting_round + - name: parachain + children: + - name: pvf_executor + - name: fragment_chain + - name: dispute + - name: runtime + children: + - name: runtime_api + - name: host_api + children: + - name: elliptic_curves_extension + - name: memory_extension + - name: io_extension + - name: crypto_extension + - name: storage_extension + - name: child_storage_extension + - name: offchain_extension + - name: misc_extension + - name: runtime_cache + - name: binaryen + - name: wavm + - name: wasmedge + - name: metrics + - name: telemetry + - name: network + children: + - name: reputation + - name: synchronizer + - name: authority_discovery + - name: kagome_protocols + children: + - name: block_announce_protocol + - name: grandpa_protocol + - name: propagate_transactions_protocol + - name: sync_protocol + - name: state_protocol + - name: warp_sync_protocol + - name: parachain_protocols + children: + - name: collation_protocol_vstaging + - name: validation_protocol_vstaging + - name: req_collation_protocol + - name: req_chunk_protocol + - name: req_available_data_protocol + - name: req_statement_protocol + - name: req_pov_protocol + - name: dispute_protocol + - name: req_attested_candidate_protocol + - name: changes_trie + - name: storage + children: + - name: trie + - name: trie_pruner + - name: transactions + - name: pubsub + - name: threads + - name: others + children: + - name: testing + - name: debug +# ---------------- + )"); + } // namespace + + Configurator::Configurator(std::shared_ptr previous) + : ConfiguratorFromYAML(std::move(previous), embedded_config) {} + + Configurator::Configurator(std::shared_ptr previous, + std::string config) + : ConfiguratorFromYAML(std::move(previous), std::move(config)) {} + + Configurator::Configurator(std::shared_ptr previous, + filesystem::path path) + : ConfiguratorFromYAML(std::move(previous), std::move(path)) {} + + std::optional Configurator::getLogConfigFile( + int argc, const char **argv) { + namespace po = boost::program_options; + po::options_description desc("General options"); + desc.add_options() + // clang-format off + ("logcfg", po::value()) + ("log", po::value()) // needed to avoid mix `--logcfg` and `--log` + // clang-format on + ; + + po::variables_map vm; + + po::parsed_options parsed = po::command_line_parser(argc, argv) + .options(desc) + .allow_unregistered() + .run(); + po::store(parsed, vm); + po::notify(vm); + + if (auto it = vm.find("logcfg"); it != vm.end()) { + if (not it->second.defaulted()) { + return it->second.as(); + } + } + return std::nullopt; + } + +} // namespace kagome::log diff --git a/src/log/configurator.hpp b/src/log/configurator.hpp new file mode 100644 index 0000000..e1503f3 --- /dev/null +++ b/src/log/configurator.hpp @@ -0,0 +1,31 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include "filesystem/common.hpp" + +namespace kagome::log { + + class Configurator : public soralog::ConfiguratorFromYAML { + using PrevConfigurator = soralog::Configurator; + + public: + Configurator(std::shared_ptr previous); + + explicit Configurator(std::shared_ptr previous, + std::string config); + + explicit Configurator(std::shared_ptr previous, + filesystem::path path); + + static std::optional getLogConfigFile(int argc, + const char **argv); + }; + +} // namespace kagome::log diff --git a/src/log/formatters/filepath.hpp b/src/log/formatters/filepath.hpp new file mode 100644 index 0000000..d03c635 --- /dev/null +++ b/src/log/formatters/filepath.hpp @@ -0,0 +1,18 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include + +template <> +struct fmt::formatter + : fmt::formatter { + auto format(const std::filesystem::path &path, format_context &ctx) { + return fmt::formatter::format(path.native(), ctx); + } +}; diff --git a/src/log/formatters/optional.hpp b/src/log/formatters/optional.hpp new file mode 100644 index 0000000..c263221 --- /dev/null +++ b/src/log/formatters/optional.hpp @@ -0,0 +1,36 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include +#include +#include + +#include + +template