diff --git a/contrib/epee/include/net/http_server_handlers_map2.h b/contrib/epee/include/net/http_server_handlers_map2.h index ffb3f3b7ea5..72c78b505d2 100644 --- a/contrib/epee/include/net/http_server_handlers_map2.h +++ b/contrib/epee/include/net/http_server_handlers_map2.h @@ -131,7 +131,7 @@ } \ uint64_t ticks2 = misc_utils::get_tick_count(); \ epee::byte_slice buffer; \ - epee::serialization::store_t_to_binary(static_cast(resp), buffer, 64 * 1024); \ + epee::serialization::store_t_to_binary(static_cast(resp), buffer); \ uint64_t ticks3 = epee::misc_utils::get_tick_count(); \ response_info.m_body.assign(reinterpret_cast(buffer.data()), buffer.size()); \ response_info.m_mime_tipe = " application/octet-stream"; \ diff --git a/contrib/epee/include/storages/http_abstract_invoke.h b/contrib/epee/include/storages/http_abstract_invoke.h index c615b20e6c0..ae75c3a4585 100644 --- a/contrib/epee/include/storages/http_abstract_invoke.h +++ b/contrib/epee/include/storages/http_abstract_invoke.h @@ -76,7 +76,7 @@ namespace epee bool invoke_http_bin(const boost::string_ref uri, const t_request& out_struct, t_response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref method = "POST") { byte_slice req_param; - if(!serialization::store_t_to_binary(out_struct, req_param, 16 * 1024)) + if(!serialization::store_t_to_binary(out_struct, req_param)) return false; const http::http_response_info* pri = NULL; @@ -98,12 +98,7 @@ namespace epee return false; } - static const constexpr epee::serialization::portable_storage::limits_t default_http_bin_limits = { - 65536 * 3, // objects - 65536 * 3, // fields - 65536 * 3, // strings - }; - return serialization::load_t_from_binary(result_struct, epee::strspan(pri->m_body), &default_http_bin_limits); + return serialization::load_t_from_binary(result_struct, epee::strspan(pri->m_body)); } template diff --git a/contrib/epee/include/storages/levin_abstract_invoke2.h b/contrib/epee/include/storages/levin_abstract_invoke2.h index 383d67cc221..6f6b345e9f0 100644 --- a/contrib/epee/include/storages/levin_abstract_invoke2.h +++ b/contrib/epee/include/storages/levin_abstract_invoke2.h @@ -27,6 +27,7 @@ #pragma once #include "portable_storage_template_helper.h" +#include "p2p/portable_scheme/load_store_wrappers.h" #include #include #include @@ -43,15 +44,6 @@ void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool template void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, int command); -namespace -{ - static const constexpr epee::serialization::portable_storage::limits_t default_levin_limits = { - 8192, // objects - 16384, // fields - 16384, // strings - }; -} - namespace epee { namespace net_utils @@ -108,11 +100,9 @@ namespace epee bool invoke_remote_command2(const epee::net_utils::connection_context_base context, int command, const t_arg& out_struct, t_result& result_struct, t_transport& transport) { const boost::uuids::uuid &conn_id = context.m_connection_id; - typename serialization::portable_storage stg; - out_struct.store(stg); levin::message_writer to_send{16 * 1024}; std::string buff_to_recv; - stg.store_to_binary(to_send.buffer); + portable_scheme::store_to_binary(out_struct, to_send.buffer); int res = transport.invoke(command, std::move(to_send), buff_to_recv, conn_id); if( res <=0 ) @@ -120,25 +110,22 @@ namespace epee LOG_PRINT_L1("Failed to invoke command " << command << " return code " << res); return false; } - typename serialization::portable_storage stg_ret; - if(!stg_ret.load_from_binary(buff_to_recv, &default_levin_limits)) + if(!portable_scheme::load_from_binary(result_struct, buff_to_recv)) { on_levin_traffic(context, true, false, true, buff_to_recv.size(), command); LOG_ERROR("Failed to load_from_binary on command " << command); return false; } on_levin_traffic(context, true, false, false, buff_to_recv.size(), command); - return result_struct.load(stg_ret); + return true; } template bool async_invoke_remote_command2(const epee::net_utils::connection_context_base &context, int command, const t_arg& out_struct, t_transport& transport, const callback_t &cb, size_t inv_timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED) { const boost::uuids::uuid &conn_id = context.m_connection_id; - typename serialization::portable_storage stg; - const_cast(out_struct).store(stg);//TODO: add true const support to searilzation levin::message_writer to_send{16 * 1024}; - stg.store_to_binary(to_send.buffer); + portable_scheme::store_to_binary(out_struct, to_send.buffer); int res = transport.invoke_async(command, std::move(to_send), conn_id, [cb, command](int code, const epee::span buff, typename t_transport::connection_context& context)->bool { t_result result_struct = AUTO_VAL_INIT(result_struct); @@ -150,21 +137,13 @@ namespace epee cb(code, result_struct, context); return false; } - serialization::portable_storage stg_ret; - if(!stg_ret.load_from_binary(buff, &default_levin_limits)) + if(!portable_scheme::load_from_binary(result_struct, buff)) { on_levin_traffic(context, true, false, true, buff.size(), command); LOG_ERROR("Failed to load_from_binary on command " << command); cb(LEVIN_ERROR_FORMAT, result_struct, context); return false; } - if (!result_struct.load(stg_ret)) - { - on_levin_traffic(context, true, false, true, buff.size(), command); - LOG_ERROR("Failed to load result struct on command " << command); - cb(LEVIN_ERROR_FORMAT, result_struct, context); - return false; - } on_levin_traffic(context, true, false, false, buff.size(), command); cb(code, result_struct, context); return true; @@ -181,10 +160,8 @@ namespace epee bool notify_remote_command2(const typename t_transport::connection_context &context, int command, const t_arg& out_struct, t_transport& transport) { const boost::uuids::uuid &conn_id = context.m_connection_id; - serialization::portable_storage stg; - out_struct.store(stg); levin::message_writer to_send; - stg.store_to_binary(to_send.buffer); + portable_scheme::store_to_binary(out_struct, to_send.buffer); int res = transport.send(to_send.finalize_notify(command), conn_id); if(res <=0 ) @@ -199,28 +176,18 @@ namespace epee template int buff_to_t_adapter(int command, const epee::span in_buff, byte_stream& buff_out, callback_t cb, t_context& context ) { - serialization::portable_storage strg; - if(!strg.load_from_binary(in_buff, &default_levin_limits)) + boost::value_initialized in_struct; + if(!portable_scheme::load_from_binary(static_cast(in_struct), in_buff)) { on_levin_traffic(context, false, false, true, in_buff.size(), command); LOG_ERROR("Failed to load_from_binary in command " << command); return -1; } - boost::value_initialized in_struct; boost::value_initialized out_struct; - if (!static_cast(in_struct).load(strg)) - { - on_levin_traffic(context, false, false, true, in_buff.size(), command); - LOG_ERROR("Failed to load in_struct in command " << command); - return -1; - } on_levin_traffic(context, false, false, false, in_buff.size(), command); int res = cb(command, static_cast(in_struct), static_cast(out_struct), context); - serialization::portable_storage strg_out; - static_cast(out_struct).store(strg_out); - - if(!strg_out.store_to_binary(buff_out)) + if(!portable_scheme::store_to_binary(static_cast(out_struct), buff_out)) { LOG_ERROR("Failed to store_to_binary in command" << command); return -1; @@ -232,18 +199,11 @@ namespace epee template int buff_to_t_adapter(t_owner* powner, int command, const epee::span in_buff, callback_t cb, t_context& context) { - serialization::portable_storage strg; - if(!strg.load_from_binary(in_buff, &default_levin_limits)) - { - on_levin_traffic(context, false, false, true, in_buff.size(), command); - LOG_ERROR("Failed to load_from_binary in notify " << command); - return -1; - } boost::value_initialized in_struct; - if (!static_cast(in_struct).load(strg)) + if(!portable_scheme::load_from_binary(static_cast(in_struct), in_buff)) { on_levin_traffic(context, false, false, true, in_buff.size(), command); - LOG_ERROR("Failed to load in_struct in notify " << command); + LOG_ERROR("Failed to load_from_binary in notify " << command); return -1; } on_levin_traffic(context, false, false, false, in_buff.size(), command); diff --git a/contrib/epee/include/storages/portable_storage_template_helper.h b/contrib/epee/include/storages/portable_storage_template_helper.h index 7f6596f36b5..9df6a74318c 100644 --- a/contrib/epee/include/storages/portable_storage_template_helper.h +++ b/contrib/epee/include/storages/portable_storage_template_helper.h @@ -30,6 +30,7 @@ #include "byte_slice.h" #include "parserse_base_utils.h" /// TODO: (mj-xmr) This will be reduced in an another PR +#include "p2p/portable_scheme/load_store_wrappers.h" #include "portable_storage.h" #include "file_io_utils.h" #include "span.h" @@ -88,14 +89,9 @@ namespace epee } //----------------------------------------------------------------------------------------------------------- template - bool load_t_from_binary(t_struct& out, const epee::span binary_buff, const epee::serialization::portable_storage::limits_t *limits = NULL) + bool load_t_from_binary(t_struct& out, const epee::span binary_buff) { - portable_storage ps; - bool rs = ps.load_from_binary(binary_buff, limits); - if(!rs) - return false; - - return out.load(ps); + return portable_scheme::load_from_binary(out, binary_buff); } //----------------------------------------------------------------------------------------------------------- template @@ -115,27 +111,23 @@ namespace epee } //----------------------------------------------------------------------------------------------------------- template - bool store_t_to_binary(t_struct& str_in, byte_slice& binary_buff, size_t initial_buffer_size = 8192) + bool store_t_to_binary(t_struct& str_in, byte_slice& binary_buff) { - portable_storage ps; - str_in.store(ps); - return ps.store_to_binary(binary_buff, initial_buffer_size); + return portable_scheme::store_to_binary(str_in, binary_buff); } //----------------------------------------------------------------------------------------------------------- template - byte_slice store_t_to_binary(t_struct& str_in, size_t initial_buffer_size = 8192) + byte_slice store_t_to_binary(t_struct& str_in) { byte_slice binary_buff; - store_t_to_binary(str_in, binary_buff, initial_buffer_size); + store_t_to_binary(str_in, binary_buff); return binary_buff; } //----------------------------------------------------------------------------------------------------------- template bool store_t_to_binary(t_struct& str_in, byte_stream& binary_buff) { - portable_storage ps; - str_in.store(ps); - return ps.store_to_binary(binary_buff); + return portable_scheme::store_to_binary(str_in, binary_buff); } } diff --git a/src/cryptonote_basic/CMakeLists.txt b/src/cryptonote_basic/CMakeLists.txt index e386ec4eabd..3e2d69c4595 100644 --- a/src/cryptonote_basic/CMakeLists.txt +++ b/src/cryptonote_basic/CMakeLists.txt @@ -46,6 +46,7 @@ target_link_libraries(cryptonote_format_utils_basic set(cryptonote_basic_sources account.cpp + account_epee_serialization.cpp connection_context.cpp cryptonote_basic_impl.cpp cryptonote_format_utils.cpp diff --git a/src/cryptonote_basic/account_epee_serialization.cpp b/src/cryptonote_basic/account_epee_serialization.cpp new file mode 100644 index 00000000000..d88e97cc90f --- /dev/null +++ b/src/cryptonote_basic/account_epee_serialization.cpp @@ -0,0 +1,6 @@ +#include "account_epee_serialization.h" +#include "p2p/portable_scheme/load_store_wrappers_impl.h" + +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(cryptonote::account_public_address) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(cryptonote::account_keys) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(cryptonote::account_base) diff --git a/src/cryptonote_basic/account_epee_serialization.h b/src/cryptonote_basic/account_epee_serialization.h new file mode 100644 index 00000000000..63e17a3a539 --- /dev/null +++ b/src/cryptonote_basic/account_epee_serialization.h @@ -0,0 +1,100 @@ +#pragma once + +#include "account.h" +#include "p2p/portable_scheme/scheme.h" + +namespace portable_scheme { + +namespace scheme_space { + +template<> +struct scheme { + using T = cryptonote::account_public_address; + using tags = map_tag< + field_tag>, + field_tag> + >; + BEGIN_READ(T) + READ_FIELD_POD(m_spend_public_key) + READ_FIELD_POD(m_view_public_key) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD_POD(m_spend_public_key) + WRITE_FIELD_POD(m_view_public_key) + END_WRITE() +}; + +template<> +struct scheme { + using T = cryptonote::account_keys; + using tags = map_tag< + field_tag::tags>, + field_tag>, + field_tag>, + field_tag>, + field_tag> + >; + BEGIN_READ(T) + READ_FIELD(m_account_address) + READ_FIELD_CUSTOM(m_spend_secret_key, scheme>) + READ_FIELD_CUSTOM(m_view_secret_key, scheme>) + READ_FIELD_LIST_CUSTOM(m_multisig_keys, scheme>) + READ_FIELD_POD(m_encryption_iv) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD(m_account_address) + WRITE_FIELD_CUSTOM(m_spend_secret_key, scheme>) + WRITE_FIELD_CUSTOM(m_view_secret_key, scheme>) + WRITE_FIELD_LIST_CUSTOM(m_multisig_keys, scheme>) + WRITE_FIELD_POD_WITH_DEFAULT(m_encryption_iv, crypto::chacha_iv{{}}) + END_WRITE() +}; + +template<> +struct scheme { + using T = cryptonote::account_base; + using tags = map_tag< + field_tag>, + field_tag::tags> + >; + BEGIN_READ_RAW( + private: + const T &t; + uint64_t m_creation_timestamp; + public: + read_t(const T &t): t(t), m_creation_timestamp(t.get_createtime()) {} + ) + READ_FIELD_NAME("m_keys", t.get_keys()) + READ_FIELD_NAME("m_creation_timestamp", m_creation_timestamp) + END_READ() + BEGIN_WRITE_RAW( + private: + T &t; + cryptonote::account_keys m_keys{}; + uint64_t m_creation_timestamp{}; + public: + write_t(T &t): t(t) {} + ) + WRITE_FIELD_NAME("m_keys", m_keys); + WRITE_FIELD_NAME("m_creation_timestamp", m_creation_timestamp); + WRITE_CHECK( + struct archive_t { + const cryptonote::account_keys &m_keys; + const uint64_t &m_creation_timestamp; + void operator&(cryptonote::account_keys& m_keys) const { + m_keys = this->m_keys; + } + void operator&(std::uint64_t &m_creation_timestamp) const { + m_creation_timestamp = this->m_creation_timestamp; + } + }; + archive_t archive{m_keys, m_creation_timestamp}; + t.serialize(archive, 0); + return true; + ) + END_WRITE() +}; + +} + +} diff --git a/src/cryptonote_protocol/cryptonote_protocol_serialization.cpp b/src/cryptonote_protocol/cryptonote_protocol_serialization.cpp new file mode 100644 index 00000000000..e263cc99f9a --- /dev/null +++ b/src/cryptonote_protocol/cryptonote_protocol_serialization.cpp @@ -0,0 +1,13 @@ +#include "cryptonote_protocol/cryptonote_protocol_serialization.h" +#include "p2p/portable_scheme/load_store_wrappers_impl.h" + +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(cryptonote::NOTIFY_NEW_BLOCK::request) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(cryptonote::NOTIFY_NEW_TRANSACTIONS::request) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(cryptonote::NOTIFY_REQUEST_GET_OBJECTS::request) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(cryptonote::NOTIFY_RESPONSE_GET_OBJECTS::request) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(cryptonote::CORE_SYNC_DATA) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(cryptonote::NOTIFY_REQUEST_CHAIN::request) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(cryptonote::NOTIFY_RESPONSE_CHAIN_ENTRY::request) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(cryptonote::NOTIFY_NEW_FLUFFY_BLOCK::request) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(cryptonote::NOTIFY_REQUEST_FLUFFY_MISSING_TX::request) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(cryptonote::NOTIFY_GET_TXPOOL_COMPLEMENT::request) diff --git a/src/cryptonote_protocol/cryptonote_protocol_serialization.h b/src/cryptonote_protocol/cryptonote_protocol_serialization.h new file mode 100644 index 00000000000..c1d97d6a2a7 --- /dev/null +++ b/src/cryptonote_protocol/cryptonote_protocol_serialization.h @@ -0,0 +1,337 @@ +#pragma once + +#include "cryptonote_protocol_defs.h" +#include "p2p/portable_scheme/scheme.h" +#include "cryptonote_basic/cryptonote_format_utils.h" + +namespace portable_scheme { + +namespace scheme_space { + +template<> +struct scheme { + using S = scheme>; + using tags = S::tags; + using read_t = S::read_t; + using write_t = S::write_t; +}; + +constexpr std::size_t MIN_TX_BLOB_SIZE = 41; +constexpr std::size_t MIN_BLOCK_BLOB_SIZE = 72; + +template<> +struct scheme { + using T = cryptonote::tx_blob_entry; + using tags = map_tag< + field_tag, optional_t::required>, + field_tag, optional_t::required> + >; + BEGIN_READ(T) + READ_FIELD_LIST_POD(blob) + READ_FIELD_POD(prunable_hash) + END_READ() + + BEGIN_WRITE(T) + WRITE_FIELD_LIST_POD(blob) + WRITE_FIELD_POD(prunable_hash) + END_WRITE() +}; + +struct compact_tx_t { + using tags = span_tag; + using T = cryptonote::tx_blob_entry; + using S = scheme>; + struct read_t: S::read_t { + read_t(const T &t): S::read_t(t.blob) {} + }; + struct write_t: S::write_t { + write_t(T &t): S::write_t(t.blob) {} + }; +}; + +template<> +struct scheme { + using T = cryptonote::block_complete_entry; + using compact_txs_t = scheme>; + using tags = map_tag< + field_tag, optional_t::required>, + field_tag>, + field_tag>, + field_tag>::tags>, + field_tag + >; + BEGIN_READ(T) + READ_FIELD_LIST_POD(block) + READ_FIELD_WITH_DEFAULT(block_weight, 0) + READ_FIELD_WITH_DEFAULT(pruned, false) + READ_FIELD_LIST_OPT(txs, {return t.pruned;}) + READ_FIELD_LIST_CUSTOM_OPT(txs, compact_txs_t, {return not t.pruned;}) + END_READ() + BEGIN_WRITE_RAW( + private: + T &t; + public: + write_t(T &t): t(t) { t.txs.clear(); } + ) + WRITE_FIELD_LIST_POD(block) + WRITE_FIELD_WITH_DEFAULT(block_weight, 0) + WRITE_FIELD_WITH_DEFAULT(pruned, false) + WRITE_FIELD_LIST_OPT(txs) + WRITE_FIELD_LIST_CUSTOM_OPT(txs, compact_txs_t) + END_WRITE() +}; + +template<> +struct scheme { + using T = cryptonote::NOTIFY_NEW_BLOCK::request; + using tags = map_tag< + field_tag::tags>, + field_tag> + >; + BEGIN_READ(T) + READ_FIELD(b) + READ_FIELD(current_blockchain_height) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD(b) + WRITE_FIELD(current_blockchain_height) + END_WRITE() +}; + +template<> +struct scheme { + using T = cryptonote::NOTIFY_NEW_TRANSACTIONS::request; + using tags = map_tag< + field_tag>, + field_tag>, + field_tag>> + >; + BEGIN_READ(T) + READ_FIELD_LIST_POD(_) + READ_FIELD_WITH_DEFAULT(dandelionpp_fluff, true) + READ_FIELD_LIST(txs) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD_LIST_POD(_) + WRITE_FIELD_WITH_DEFAULT(dandelionpp_fluff, true) + WRITE_FIELD_LIST(txs) + END_WRITE() +}; + +template<> +struct scheme { + using T = cryptonote::NOTIFY_REQUEST_GET_OBJECTS::request; + using tags = map_tag< + field_tag>, + field_tag> + >; + BEGIN_READ(T) + READ_FIELD_LIST_POD(blocks) + READ_FIELD_WITH_DEFAULT(prune, false) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD_LIST_POD(blocks) + WRITE_FIELD_WITH_DEFAULT(prune, false) + END_WRITE() +}; + +template<> +struct scheme { + using T = cryptonote::NOTIFY_RESPONSE_GET_OBJECTS::request; + using tags = map_tag< + field_tag>::tags>, + field_tag>, + field_tag> + >; + BEGIN_READ(T) + READ_FIELD_LIST(blocks) + READ_FIELD(current_blockchain_height) + READ_FIELD_LIST_POD(missed_ids) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD_LIST(blocks) + WRITE_FIELD(current_blockchain_height) + WRITE_FIELD_LIST_POD(missed_ids) + END_WRITE() +}; + +template<> +struct scheme { + using T = cryptonote::CORE_SYNC_DATA; + using tags = map_tag< + field_tag>, + field_tag>, + field_tag>, + field_tag>, + field_tag>, + field_tag> + >; + BEGIN_READ(T) + READ_FIELD(cumulative_difficulty) + READ_FIELD(cumulative_difficulty_top64) + READ_FIELD(current_height) + READ_FIELD_WITH_DEFAULT(pruning_seed, 0) + READ_FIELD_POD(top_id) + READ_FIELD_WITH_DEFAULT(top_version, 0) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD(cumulative_difficulty) + WRITE_FIELD_WITH_DEFAULT(cumulative_difficulty_top64, 0) + WRITE_FIELD(current_height) + WRITE_FIELD_WITH_DEFAULT(pruning_seed, 0) + WRITE_FIELD_POD(top_id) + WRITE_FIELD_WITH_DEFAULT(top_version, 0) + END_WRITE() +}; + +template<> +struct scheme { + using T = cryptonote::NOTIFY_REQUEST_CHAIN::request; + using tags = map_tag< + field_tag>, + field_tag> + >; + BEGIN_READ(T) + READ_FIELD_LIST_POD(block_ids) + READ_FIELD_WITH_DEFAULT(prune, false) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD_LIST_POD(block_ids) + WRITE_FIELD_WITH_DEFAULT(prune, false) + END_WRITE() +}; + +template<> +struct scheme { + using T = cryptonote::NOTIFY_RESPONSE_CHAIN_ENTRY::request; + using tags = map_tag< + field_tag>, + field_tag>, + field_tag>, + field_tag>, + #if 1 + field_tag>, + #else + field_tag>, + field_tag>>, + #endif + field_tag>, + field_tag> + >; + BEGIN_READ(T) + READ_FIELD(cumulative_difficulty) + READ_FIELD(cumulative_difficulty_top64) + READ_FIELD_LIST_POD(first_block) + READ_FIELD_LIST_POD(m_block_ids) + #if 1 + READ_FIELD_LIST_POD_FORCE(m_block_weights) + #else + READ_FIELD_LIST_POD_OPT_FORCE(m_block_weights) + READ_FIELD_LIST_OPT_NAME("m_block_weights_", t.m_block_weights) + #endif + READ_FIELD(start_height) + READ_FIELD(total_height) + END_READ() + #if 1 + BEGIN_WRITE(T) + #else + BEGIN_WRITE_RAW( + private: + T &t; + public: + write_t(T &t): t(t) { t.m_block_weights.clear(); } + ) + #endif + WRITE_FIELD(cumulative_difficulty) + WRITE_FIELD(cumulative_difficulty_top64) + WRITE_FIELD_LIST_POD(first_block) + WRITE_FIELD_LIST_POD(m_block_ids) + #if 1 + WRITE_FIELD_LIST_POD_FORCE(m_block_weights) + #else + WRITE_FIELD_LIST_POD_OPT_FORCE(m_block_weights) + WRITE_FIELD_LIST_OPT_NAME("m_block_weights_", t.m_block_weights) + #endif + WRITE_FIELD(start_height) + WRITE_FIELD(total_height) + END_WRITE() +}; + +template<> +struct scheme { + using T = cryptonote::NOTIFY_NEW_FLUFFY_BLOCK::request; + using tags = map_tag< + field_tag::tags>, + field_tag> + >; + BEGIN_READ(T) + READ_FIELD(b) + READ_FIELD(current_blockchain_height) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD(b) + WRITE_FIELD(current_blockchain_height) + END_WRITE() +}; + +template<> +struct scheme { + using T = cryptonote::NOTIFY_REQUEST_FLUFFY_MISSING_TX::request; + using tags = map_tag< + field_tag>, + field_tag>, + #if 1 + field_tag> + #else + field_tag>, + field_tag>> + #endif + >; + BEGIN_READ(T) + READ_FIELD_POD(block_hash) + READ_FIELD(current_blockchain_height) + #if 1 + READ_FIELD_LIST_POD_FORCE(missing_tx_indices) + #else + READ_FIELD_LIST_POD_OPT_FORCE(missing_tx_indices) + READ_FIELD_LIST_OPT_NAME("missing_tx_indices_", t.missing_tx_indices) + #endif + END_READ() + #if 1 + BEGIN_WRITE(T) + #else + BEGIN_WRITE_RAW( + private: + T &t; + public: + write_t(T &t): t(t) { t.missing_tx_indices.clear(); } + ) + #endif + WRITE_FIELD_POD(block_hash) + WRITE_FIELD(current_blockchain_height) + #if 1 + WRITE_FIELD_LIST_POD_FORCE(missing_tx_indices) + #else + WRITE_FIELD_LIST_POD_OPT_FORCE(missing_tx_indices) + WRITE_FIELD_LIST_OPT_NAME("missing_tx_indices_", t.missing_tx_indices) + #endif + END_WRITE() +}; + +template<> +struct scheme { + using T = cryptonote::NOTIFY_GET_TXPOOL_COMPLEMENT::request; + using tags = map_tag< + field_tag> + >; + BEGIN_READ(T) + READ_FIELD_LIST_POD(hashes) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD_LIST_POD(hashes) + END_WRITE() +}; + +} + +} diff --git a/src/p2p/node_server_serialization.cpp b/src/p2p/node_server_serialization.cpp new file mode 100644 index 00000000000..978bee238a1 --- /dev/null +++ b/src/p2p/node_server_serialization.cpp @@ -0,0 +1,8 @@ +#include "p2p/p2p_protocol_serialization.h" +#include "cryptonote_protocol/cryptonote_protocol_serialization.h" +#include "p2p/portable_scheme/load_store_wrappers_impl.h" + +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(nodetool::COMMAND_HANDSHAKE_T::request) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(nodetool::COMMAND_HANDSHAKE_T::response) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(nodetool::COMMAND_TIMED_SYNC_T::request) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(nodetool::COMMAND_TIMED_SYNC_T::response) diff --git a/src/p2p/p2p_protocol_serialization.cpp b/src/p2p/p2p_protocol_serialization.cpp new file mode 100644 index 00000000000..a281ce52d99 --- /dev/null +++ b/src/p2p/p2p_protocol_serialization.cpp @@ -0,0 +1,7 @@ +#include "p2p/p2p_protocol_serialization.h" +#include "p2p/portable_scheme/load_store_wrappers_impl.h" + +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(nodetool::COMMAND_PING::request) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(nodetool::COMMAND_PING::response) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(nodetool::COMMAND_REQUEST_SUPPORT_FLAGS::request) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(nodetool::COMMAND_REQUEST_SUPPORT_FLAGS::response) diff --git a/src/p2p/p2p_protocol_serialization.h b/src/p2p/p2p_protocol_serialization.h new file mode 100644 index 00000000000..4df793a4158 --- /dev/null +++ b/src/p2p/p2p_protocol_serialization.h @@ -0,0 +1,320 @@ +#pragma once + +#include "p2p/p2p_protocol_defs.h" +#include "p2p/portable_scheme/scheme.h" + +namespace portable_scheme { + +namespace scheme_space { + +struct addr_t { + struct { + bool addr; + bool m_ip; + bool host; + bool m_port; + bool port; + } absent; + boost::asio::ip::address_v6::bytes_type ipv6; + std::uint32_t ipv4; + std::string host; + std::uint16_t port; +}; + +template<> +struct scheme { + using T = addr_t; + using tags = map_tag< + field_tag>, + field_tag>, + field_tag>, + field_tag>, + field_tag> + >; + BEGIN_READ(T) + READ_FIELD_POD_OPT_NAME("addr", t.ipv6, {return not t.absent.addr;}) + READ_FIELD_OPT_NAME("m_ip", t.ipv4, {return not t.absent.m_ip;}) + READ_FIELD_LIST_POD_OPT(host, {return not t.absent.host;}) + READ_FIELD_OPT_NAME("m_port", t.port, {return not t.absent.m_port;}) + READ_FIELD_OPT(port, {return not t.absent.port;}) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD_POD_OPT_NAME("addr", t.ipv6, {t.absent.addr = true; return true;}) + WRITE_FIELD_OPT_NAME("m_ip", t.ipv4, {t.absent.m_ip = true; return true;}) + WRITE_FIELD_LIST_POD_OPT(host, {t.absent.host = true; return true;}) + WRITE_FIELD_OPT_NAME("m_port", t.port, {t.absent.m_port = true; return true;}) + WRITE_FIELD_OPT(port, {t.absent.port = true; return true;}) + END_WRITE() +}; + +template<> +struct scheme { + using T = epee::net_utils::network_address; + using tags = map_tag< + field_tag::tags>, + field_tag> + >; + using ipv4_addr = epee::net_utils::ipv4_network_address; + using ipv6_addr = epee::net_utils::ipv6_network_address; + using tor_addr = net::tor_address; + using i2p_addr = net::i2p_address; + using addr_type = epee::net_utils::address_type; + BEGIN_READ_RAW( + private: + addr_t addr; + std::uint8_t type_id; + public: + read_t(const T &t): + addr{ + .absent = { + .addr = t.get_type_id() != addr_type::ipv6, + .m_ip = t.get_type_id() != addr_type::ipv4, + .host = t.get_type_id() != addr_type::tor and t.get_type_id() != addr_type::i2p, + .m_port = t.get_type_id() != addr_type::ipv4 and t.get_type_id() != addr_type::ipv6, + .port = t.get_type_id() != addr_type::tor and t.get_type_id() != addr_type::i2p, + }, + }, + type_id{ + static_cast(t.get_type_id()) + } + { + switch(t.get_type_id()) { + case addr_type::ipv4: {addr.ipv4 = t.as().ip(); addr.port = t.as().port(); break;} + case addr_type::ipv6: {addr.ipv6 = t.as().ip().to_bytes(); addr.port = t.as().port(); break;} + case addr_type::i2p: {addr.host = t.as().host_str(); addr.port = t.as().port(); break;} + case addr_type::tor: {addr.host = t.as().host_str(); addr.port = t.as().port(); break;} + default: break; + } + } + ) + READ_FIELD_NAME("addr", addr) + READ_FIELD_NAME("type", type_id) + END_READ() + BEGIN_WRITE_RAW( + private: + T &t; + addr_t addr{}; + std::uint8_t type_id{}; + public: + write_t(T &t): t(t) {} + ) + WRITE_FIELD_NAME("addr", addr) + WRITE_FIELD_NAME("type", type_id) + WRITE_CHECK( + switch (static_cast(type_id)) { + case addr_type::ipv4: + if (addr.absent.m_ip or addr.absent.m_port or not addr.absent.port) + return false; + t = ipv4_addr(addr.ipv4, addr.port); + break; + case addr_type::ipv6: + if (addr.absent.addr or addr.absent.m_port or not addr.absent.port) + return false; + t = ipv6_addr(boost::asio::ip::address_v6(addr.ipv6), addr.port); + break; + case addr_type::tor: { + if (addr.absent.host or addr.absent.port or not addr.absent.m_port) + return false; + auto&& result = tor_addr::make(addr.host, addr.port); + if (not result) + return false; + t = result.value(); + break; + } + case addr_type::i2p: { + if (addr.absent.host or addr.absent.port or not addr.absent.m_port) + return false; + auto&& result = i2p_addr::make(addr.host, addr.port); + if (not result) + return false; + t = result.value(); + break; + } + default: + return false; + } + return true; + ) + END_WRITE() +}; + +template<> +struct scheme> { + using T = nodetool::peerlist_entry_base; + using tags = map_tag< + field_tag::tags>, + field_tag>, + field_tag>, + field_tag>, + field_tag>, + field_tag> + >; + BEGIN_READ(T) + READ_FIELD(adr) + READ_FIELD_WITH_DEFAULT(id, 0) + READ_FIELD_WITH_DEFAULT(last_seen, 0) + READ_FIELD_WITH_DEFAULT(pruning_seed, 0) + READ_FIELD_WITH_DEFAULT(rpc_credits_per_hash, 0) + READ_FIELD_WITH_DEFAULT(rpc_port, 0) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD(adr) + WRITE_FIELD_WITH_DEFAULT(id, 0) + WRITE_FIELD_WITH_DEFAULT(last_seen, 0) + WRITE_FIELD_WITH_DEFAULT(pruning_seed, 0) + WRITE_FIELD_WITH_DEFAULT(rpc_credits_per_hash, 0) + WRITE_FIELD_WITH_DEFAULT(rpc_port, 0) + END_WRITE() +}; + +template<> +struct scheme { + using T = nodetool::basic_node_data; + using tags = map_tag< + field_tag>, + field_tag>, + field_tag>, + field_tag>, + field_tag>, + field_tag> + >; + BEGIN_READ(T) + READ_FIELD(my_port) + READ_FIELD_POD(network_id) + READ_FIELD(peer_id) + READ_FIELD_WITH_DEFAULT(rpc_credits_per_hash, 0) + READ_FIELD_WITH_DEFAULT(rpc_port, 0) + READ_FIELD_WITH_DEFAULT(support_flags, 0) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD(my_port) + WRITE_FIELD_POD(network_id) + WRITE_FIELD(peer_id) + WRITE_FIELD_WITH_DEFAULT(rpc_credits_per_hash, 0) + WRITE_FIELD_WITH_DEFAULT(rpc_port, 0) + WRITE_FIELD_WITH_DEFAULT(support_flags, 0) + END_WRITE() +}; + +template +struct scheme::request>::value), void>::type> { + using tags = map_tag< + field_tag::tags>, + field_tag::tags> + >; + BEGIN_READ(T) + READ_FIELD(node_data) + READ_FIELD(payload_data) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD(node_data) + WRITE_FIELD(payload_data) + END_WRITE() +}; + +struct peerlist_t { + using T = std::vector>; + using S = scheme, 0, P2P_MAX_PEERS_IN_HANDSHAKE>>; + using tags = S::tags; + using read_t = S::read_t; + using write_t = S::write_t; +}; + +template +struct scheme::response>::value), void>::type> { + using tags = map_tag< + field_tag, + field_tag::tags>, + field_tag::tags> + >; + BEGIN_READ(T) + READ_FIELD_LIST_CUSTOM(local_peerlist_new, peerlist_t) + READ_FIELD(node_data) + READ_FIELD(payload_data) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD_LIST_CUSTOM(local_peerlist_new, peerlist_t) + WRITE_FIELD(node_data) + WRITE_FIELD(payload_data) + END_WRITE() +}; + +template +struct scheme::request>::value), void>::type> { + using tags = map_tag< + field_tag::tags> + >; + BEGIN_READ(T) + READ_FIELD(payload_data) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD(payload_data) + END_WRITE() +}; + +template +struct scheme::response>::value), void>::type> { + using tags = map_tag< + field_tag, + field_tag::tags> + >; + BEGIN_READ(T) + READ_FIELD_LIST_CUSTOM(local_peerlist_new, peerlist_t) + READ_FIELD(payload_data) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD_LIST_CUSTOM(local_peerlist_new, peerlist_t) + WRITE_FIELD(payload_data) + END_WRITE() +}; + +template<> +struct scheme { + using T = nodetool::COMMAND_PING::request; + using tags = map_tag<>; + READ(T) + WRITE(T) +}; + +template<> +struct scheme { + using T = nodetool::COMMAND_PING::response; + using tags = map_tag< + field_tag>, + field_tag>::tags> + >; + BEGIN_READ(T) + READ_FIELD_LIST_POD(status) + READ_FIELD(peer_id) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD_LIST_POD(status) + WRITE_FIELD(peer_id) + END_WRITE() +}; + +template<> +struct scheme { + using T = nodetool::COMMAND_REQUEST_SUPPORT_FLAGS::request; + using tags = map_tag<>; + READ(T) + WRITE(T) +}; + +template<> +struct scheme { + using T = nodetool::COMMAND_REQUEST_SUPPORT_FLAGS::response; + using tags = map_tag< + field_tag> + >; + BEGIN_READ(T) + READ_FIELD(support_flags) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD(support_flags) + END_WRITE() +}; + +} + +} diff --git a/src/p2p/portable_scheme/binary_codec.h b/src/p2p/portable_scheme/binary_codec.h new file mode 100644 index 00000000000..29ae110d9d6 --- /dev/null +++ b/src/p2p/portable_scheme/binary_codec.h @@ -0,0 +1,831 @@ +#pragma once + +#include "tags.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace portable_scheme { + +namespace binary_codec { + +enum : std::uint8_t { + kInt64 = 0x01, + kInt32 = 0x02, + kInt16 = 0x03, + kInt8 = 0x04, + kUInt64 = 0x05, + kUInt32 = 0x06, + kUInt16 = 0x07, + kUInt8 = 0x08, + kDouble = 0x09, + kSpan = 0x0A, + kBool = 0x0B, + kMap = 0x0C, + kList = 0x0D, + kListAny = 0x80, +}; + +constexpr std::uint64_t kSign = 0x0102010101011101; +constexpr std::uint8_t kVersion = 1; + +using namespace tags_space; + +#if BOOST_ENDIAN_LITTLE_BYTE == BOOST_VERSION_NUMBER_AVAILABLE +constexpr endian_t native_endian = endian_t::little; +#elif BOOST_ENDIAN_BIG_BYTE == BOOST_VERSION_NUMBER_AVAILABLE +constexpr endian_t native_endian = endian_t::big; +#endif + +static_assert(native_endian != endian_t::native, ""); + +template +struct mark_t; + +template +struct mark_t> { + static constexpr std::uint8_t value() { + return ( + std::is_same::value ? kInt64 : + std::is_same::value ? kInt32 : + std::is_same::value ? kInt16 : + std::is_same::value ? kInt8 : + std::is_same::value ? kUInt64 : + std::is_same::value ? kUInt32 : + std::is_same::value ? kUInt16 : + std::is_same::value ? kUInt8 : + std::is_same::value ? kDouble : + std::is_same::value ? kBool : + 0 + ); + } + static_assert(value() != 0, ""); +}; + +template +struct mark_t> { + static constexpr std::uint8_t value() { return kSpan; } +}; + +template +struct mark_t> { + static constexpr std::uint8_t value() { return kMap; } +}; + +template +struct mark_t> { + static constexpr std::uint8_t value() { return kListAny | mark_t::value(); } +}; + +template +struct mark_t, MIN_>> { + static constexpr std::uint8_t value() { return kListAny | kList; } +}; + +template +T swap_endian(const T &b) { + static_assert(std::is_pod::value, ""); + if (endian != endian_t::native && endian != native_endian && sizeof(T) >= 2) { + T a; + for (std::size_t i = 0; i < sizeof(T); ++i) { + reinterpret_cast(&a)[i] = reinterpret_cast(&b)[sizeof(T) - i - 1]; + } + return a; + } + return b; +} + +template +struct codec; + +template +struct codec> { + static_assert(std::is_integral::value || std::is_floating_point::value, ""); + static size_t encode(const T &t, std::uint8_t *out, std::size_t out_len) { + if (size() <= out_len) { + *reinterpret_cast(out) = swap_endian(t); + } + return size(); + } + static constexpr std::size_t size(const T &) { + return sizeof(T); + } + static constexpr std::size_t size() { + return sizeof(T); + } + static std::pair decode(T &t, const std::uint8_t *in, std::size_t in_len, std::size_t recursion_limit = 0) { + if (size() <= in_len) { + t = swap_endian(*reinterpret_cast(in)); + return {true, size()}; + } + return {false, in_len}; + } + static std::pair skip(const std::uint8_t *in, std::size_t in_len, std::size_t recursion_limit = 0) { + if (size() <= in_len) { + return {true, size()}; + } + return {false, in_len}; + } +}; + +template +struct codec> { + using T = bool; + static size_t encode(const T &t, std::uint8_t *out, std::size_t out_len) { + if (size() <= out_len) { + *reinterpret_cast(out) = t; + } + return size(); + } + static constexpr std::size_t size(const T &) { + return size(); + } + static constexpr std::size_t size() { + return sizeof(std::uint8_t); + } + static std::pair decode(T &t, const std::uint8_t *in, std::size_t in_len, std::size_t recursion_limit = 0) { + if (size() <= in_len) { + t = *reinterpret_cast(in); + return {true, size()}; + } + return {false, in_len}; + } + static std::pair skip(const std::uint8_t *in, std::size_t in_len, std::size_t recursion_limit = 0) { + if (size() <= in_len) { + return {true, size()}; + } + return {false, in_len}; + } +}; + +template +struct codec> { + static_assert(std::numeric_limits::is_integer, ""); + static_assert(not std::numeric_limits::is_signed, ""); + static_assert(std::numeric_limits::max() <= std::numeric_limits::max(), ""); + static std::size_t encode(const T &t, std::uint8_t *out, std::size_t out_len) { + assert(t <= (std::numeric_limits::max() >> 2)); + const std::size_t size = codec>::size(t); + const std::uint8_t size_id = codec>::size_id(t); + if (size <= out_len) { + switch (size_id) { + case 0: *reinterpret_cast(out) = swap_endian((t << 2) | size_id); break; + case 1: *reinterpret_cast(out) = swap_endian((t << 2) | size_id); break; + case 2: *reinterpret_cast(out) = swap_endian((t << 2) | size_id); break; + case 3: *reinterpret_cast(out) = swap_endian((t << 2) | size_id); break; + default: break; + } + } + return size; + } + static constexpr std::size_t size(const T &t) { + return 1 << size_id(t); + } + static constexpr std::size_t size() { + return 1; + } + static constexpr std::uint8_t size_id(const T &t) { + return ( + t < (1 << (8 * sizeof(std::uint8_t) - 2)) ? 0 : + t < (1 << (8 * sizeof(std::uint16_t) - 2)) ? 1 : + t < (1 << (8 * sizeof(std::uint32_t) - 2)) ? 2 : 3 + ); + } + static std::pair decode(T &t, const std::uint8_t *in, std::size_t in_len) { + if (not in_len) { + return {false, 0}; + } + const std::uint8_t size_id = (*reinterpret_cast(in) & 0x3); + const std::size_t size = (1 << size_id); + if (size > in_len) { + return {false, size}; + } + std::uint64_t value = {}; + switch (size_id) { + case 0: value = (swap_endian(*reinterpret_cast(in)) >> 2); break; + case 1: value = (swap_endian(*reinterpret_cast(in)) >> 2); break; + case 2: value = (swap_endian(*reinterpret_cast(in)) >> 2); break; + case 3: value = (swap_endian(*reinterpret_cast(in)) >> 2); break; + default: assert(false); break; + } + t = value; + return {value <= std::numeric_limits::max(), size}; + } + static std::pair skip(const std::uint8_t *in, std::size_t in_len) { + if (not in_len) { + return {false, 0}; + } + const std::uint8_t size_id = (*reinterpret_cast(in) & 0x3); + const std::size_t size = (1 << size_id); + return {size <= in_len, size}; + } +}; + +template +struct codec> { + template + static size_t encode(const T &t, std::uint8_t *out, std::size_t out_len) { + std::size_t size = codec>::encode(t.size(), out, out_len); + if (size + t.size() <= out_len) { + t.get(out + size); + } + size += t.size(); + return size; + } + template + static size_t size(const T &t) { + return codec>::size(t.size()) + t.size(); + } + static constexpr size_t size() { + return codec>::size(MIN) + MIN; + } + template + static std::pair decode(T &&t, const std::uint8_t *in, std::size_t in_len, std::size_t = 0) { + void const *data = {}; + std::size_t size = {}; + const auto parse_varint = codec>::decode(size, in, in_len); + if (not parse_varint.first) { + return {false, parse_varint.second}; + } + data = in + parse_varint.second; + if (parse_varint.second + size <= in_len) { + return {t.set(data, static_cast(size)), parse_varint.second + size}; + } + return {false, in_len}; + } + static std::pair skip(const std::uint8_t *in, std::size_t in_len, std::size_t = 0) { + std::uint64_t data_size; + const auto parse_varint = codec>::decode(data_size, in, in_len); + const std::size_t size = parse_varint.second; + if (not parse_varint.first) { + return {false, size}; + } + if (size + data_size <= in_len) { + return {true, size + data_size}; + } + return {false, in_len}; + } +}; + +template +struct codec> { + template + static size_t encode(const T &t, std::uint8_t *out, std::size_t out_len) { + std::size_t size = codec>::encode(t.size(), out, out_len); + auto &&it = t.cbegin(); + for (std::size_t i = 0; i < t.size(); ++i) { + size += codec::encode(it.next(), out + size, size <= out_len ? out_len - size : 0); + } + return size; + } + template + static std::size_t size(const T &t) { + std::size_t size = codec>::size(t.size()); + auto &&it = t.cbegin(); + for (std::size_t i = 0; i < t.size(); ++i) { + size += codec::size(it.next()); + } + return size; + } + static constexpr std::size_t size() { + return codec>::size(MIN) + MIN * codec::size(); + } + template + static std::pair decode(T &&t, const std::uint8_t *in, std::size_t in_len, std::size_t recursion_limit = 0) { + std::size_t list_size{}; + const auto parse_list_size = codec>::decode(list_size, in, in_len); + std::size_t size = parse_list_size.second; + if (not parse_list_size.first) { + return {false, size}; + } + static_assert(codec::size(), ""); + const bool force_item_size = t.template force_item_size::size()>(list_size); + if (force_item_size and list_size > (in_len - size) / codec::size()) { + return {false, 0}; + } + if (not t.resize(list_size)) { + return {false, size}; + } + auto &&it = t.begin(); + if (force_item_size) { + std::size_t effective_in_len = in_len - list_size * codec::size(); + while (effective_in_len < in_len) { + effective_in_len += codec::size(); + const auto parse_item = codec::decode(it.next(), in + size, effective_in_len - size, recursion_limit); + size += parse_item.second; + if (not parse_item.first) { + return {false, size}; + } + } + } + else { + for (std::size_t i = 0; i < list_size; ++i) { + const auto parse_item = codec::decode(it.next(), in + size, in_len - size, recursion_limit); + size += parse_item.second; + if (not parse_item.first) { + return {false, size}; + } + } + } + return {true, size}; + } + static std::pair skip(const std::uint8_t *in, std::size_t in_len, std::size_t recursion_limit = 0) { + std::size_t list_size{}; + const auto parse_list_size = codec>::decode(list_size, in, in_len); + size_t size = parse_list_size.second; + if (not parse_list_size.first) { + return {false, size}; + } + for (std::size_t i = 0; i < list_size; ++i) { + const auto skip_item = codec::skip(in + size, in_len - size, recursion_limit); + size += skip_item.second; + if (not skip_item.first) { + return {false, size}; + } + } + return {true, size}; + } +}; + +template<> +struct codec>> { + static std::pair skip(const std::uint8_t *in, std::size_t in_len, std::size_t recursion_limit = 0) { + if (recursion_limit == 0) { + return {false, 0}; + } + std::size_t list_size; + const auto parse_list_size = codec>::decode(list_size, in, in_len); + std::size_t size = parse_list_size.second; + if (not parse_list_size.first) { + return {false, size}; + } + for (std::size_t i = 0; i < list_size; ++i) { + std::uint8_t mark = {}; + const auto parse_mark = codec>::decode(mark, in + size, in_len - size); + size += parse_mark.second; + if (not parse_mark.first) { + return {false, size}; + } + std::pair skip_item = {}; + switch (mark) { + case kListAny | kInt64: skip_item = codec> >::skip(in, in_len); break; + case kListAny | kInt32: skip_item = codec> >::skip(in, in_len); break; + case kListAny | kInt16: skip_item = codec> >::skip(in, in_len); break; + case kListAny | kInt8: skip_item = codec> >::skip(in, in_len); break; + case kListAny | kUInt64: skip_item = codec>>::skip(in, in_len); break; + case kListAny | kUInt32: skip_item = codec>>::skip(in, in_len); break; + case kListAny | kUInt16: skip_item = codec>>::skip(in, in_len); break; + case kListAny | kUInt8: skip_item = codec> >::skip(in, in_len); break; + case kListAny | kDouble: skip_item = codec> >::skip(in, in_len); break; + case kListAny | kBool: skip_item = codec> >::skip(in, in_len); break; + case kListAny | kSpan: skip_item = codec> >::skip(in, in_len); break; + case kListAny | kMap: skip_item = codec> >::skip(in, in_len, recursion_limit - 1); break; + case kListAny | kList: skip_item = codec> >::skip(in, in_len, recursion_limit - 1); break; + default: skip_item = {false, 0}; break; + } + size += skip_item.second; + if (not skip_item.first) { + return {false, size}; + } + } + return {true, size}; + } +}; + +struct key_codec { + template + static std::size_t encode(const T &t, std::uint8_t *out, std::size_t out_len) { + const std::size_t size = codec>::encode(t.size, out, out_len); + if (size + t.size <= out_len) { + std::memcpy(out + size, t.data, t.size); + } + return size + t.size; + } + template + static constexpr std::size_t size() { + return codec>::size(K::size) + K::size; + } + template + static std::pair decode(T &&t, const std::uint8_t *in, std::size_t in_len) { + const auto parse_size = codec>::decode(t.size, in, in_len); + const std::size_t size = parse_size.second; + if (not parse_size.first) { + return {false, size}; + } + t.data = reinterpret_cast(in + size); + if (size + t.size <= in_len) { + return {true, size + t.size}; + } + return {false, in_len}; + } + static std::pair skip(const std::uint8_t *in, std::size_t in_len) { + std::uint8_t key_size{}; + const auto parse_size = codec>::decode(key_size, in, in_len); + if (not parse_size.first) { + return {false, parse_size.second}; + } + if (parse_size.second + key_size <= in_len) { + return {true, parse_size.second + key_size}; + } + return {false, in_len}; + } +}; + +template<> +struct codec> { + static std::pair skip(const std::uint8_t *in, std::size_t in_len, std::size_t recursion_limit = 0) { + if (recursion_limit == 0) { + return {false, 0}; + } + std::size_t map_size{}; + const auto parse_map_size = codec>::decode(map_size, in, in_len); + size_t size = parse_map_size.second; + if (not parse_map_size.first) { + return {false, size}; + } + const auto skip_key_value = codec>::skip_key_value(map_size, in + size, in_len - size, recursion_limit); + size += skip_key_value.second; + if (not skip_key_value.first) { + return {false, size}; + } + return {true, size}; + } + static std::pair skip_key_value(std::size_t left_keys, const std::uint8_t *in, std::size_t in_len, std::size_t recursion_limit = 0) { + if (recursion_limit == 0) { + return {false, 0}; + } + std::size_t size = 0; + for (; left_keys > 0; --left_keys) { + const auto skip_key = key_codec::skip(in + size, in_len - size); + size += skip_key.second; + if (skip_key.first) { + std::uint8_t mark = {}; + const auto parse_mark = codec>::decode(mark, in + size, in_len - size); + size += parse_mark.second; + if (not parse_mark.first) { + return {false, size}; + } + const auto skip_val = codec>::skip_value(mark, in + size, in_len - size, recursion_limit); + size += skip_val.second; + if (not skip_val.first) { + return {false, size}; + } + } + else { + return {false, size}; + } + } + return {true, size}; + } + static std::pair skip_value(const std::uint8_t &mark, const std::uint8_t *in, std::size_t in_len, std::size_t recursion_limit = 0) { + if (recursion_limit == 0) { + return {false, 0}; + } + switch (mark) { + case kInt64: return codec >::skip(in, in_len); + case kInt32: return codec >::skip(in, in_len); + case kInt16: return codec >::skip(in, in_len); + case kInt8: return codec >::skip(in, in_len); + case kUInt64: return codec>::skip(in, in_len); + case kUInt32: return codec>::skip(in, in_len); + case kUInt16: return codec>::skip(in, in_len); + case kUInt8: return codec >::skip(in, in_len); + case kDouble: return codec >::skip(in, in_len); + case kBool: return codec >::skip(in, in_len); + case kSpan: return codec >::skip(in, in_len); + case kMap: return codec >::skip(in, in_len, recursion_limit - 1); + case kListAny | kInt64: return codec> >::skip(in, in_len); + case kListAny | kInt32: return codec> >::skip(in, in_len); + case kListAny | kInt16: return codec> >::skip(in, in_len); + case kListAny | kInt8: return codec> >::skip(in, in_len); + case kListAny | kUInt64: return codec>>::skip(in, in_len); + case kListAny | kUInt32: return codec>>::skip(in, in_len); + case kListAny | kUInt16: return codec>>::skip(in, in_len); + case kListAny | kUInt8: return codec> >::skip(in, in_len); + case kListAny | kDouble: return codec> >::skip(in, in_len); + case kListAny | kBool: return codec> >::skip(in, in_len); + case kListAny | kSpan: return codec> >::skip(in, in_len); + case kListAny | kMap: return codec> >::skip(in, in_len, recursion_limit - 1); + case kListAny | kList: return codec> >::skip(in, in_len, recursion_limit - 1); + default: return {false, 0}; + } + } + template + static std::pair skip_to_key( + const T &key, + int &key_cmp, + std::size_t &left_keys, + const std::uint8_t &mark, + bool &mark_match, + const std::uint8_t *in, + std::size_t in_len, + std::size_t recursion_limit = 0 + ) { + if (recursion_limit == 0) { + return {false, 0}; + } + size_t size = 0; + for (; left_keys > 0; --left_keys) { + struct { + char const *data; + std::uint8_t size; + } found_key; + const auto parse_key = key_codec::decode(found_key, in + size, in_len - size); + size += parse_key.second; + if (parse_key.first) { + std::uint8_t found_mark; + const auto parse_mark = codec>::decode(found_mark, in + size, in_len - size); + size += parse_mark.second; + if (not parse_mark.first) { + return {false, size}; + } + auto lexicographical_compare = [](const void *lhs, std::size_t lhs_size, const void *rhs, std::size_t rhs_size) { + int cmp = std::memcmp(lhs, rhs, lhs_size < rhs_size ? lhs_size : rhs_size); + return ( + cmp != 0 ? cmp : + lhs_size < rhs_size ? -1 : + lhs_size > rhs_size ? 1 : + 0 + ); + }; + const int cmp = lexicographical_compare(found_key.data, found_key.size, key.data, key.size); + if (cmp == 0) { + key_cmp = cmp; + mark_match = (found_mark == mark); + if (mark_match) { + return {true, size}; + } + else { + size -= parse_key.second; + size -= parse_mark.second; + return {true, size}; + } + } + else if (cmp > 0) { + size -= parse_key.second; + size -= parse_mark.second; + key_cmp = cmp; + mark_match = (found_mark == mark); + return {true, size}; + } + const auto skip_val = skip_value(found_mark, in + size, in_len - size, recursion_limit); + size += skip_val.second; + if (not skip_val.first) { + return {false, size}; + } + } + else { + return {false, size}; + } + } + return {true, size}; + } + template + struct less; + template + struct less, key_space::key_t> { + using L = key_space::key_t; + using R = key_space::key_t; + template struct arg_t {}; + template + static constexpr bool less_t(arg_t) { + return L::data[N] != R::data[N] ? L::data[N] < R::data[N] : less_t(arg_t{}); + } + static constexpr bool less_t(arg_t<(sizeof...(C) < sizeof...(CC) ? sizeof...(C) : sizeof...(CC))>) { + return sizeof...(C) < sizeof...(CC); + }; + static constexpr bool value = less_t(arg_t<0>{}); + }; + template class S, typename ...T> + struct sorted { + template struct arg_t {}; + static constexpr bool value_t(arg_t<>) { return true; } + template + static constexpr bool value_t(arg_t) { return true; } + template + static constexpr bool value_t(arg_t) { + return not less::type, typename S::type>::value and value_t(arg_t{}); + } + static constexpr bool value = value_t(arg_t{}); + }; + template + struct key_selector; + template + struct key_selector> { using type = T; }; +}; + +template +struct map_codec; +template +struct map_codec, Tags...> { + template + static std::size_t fields(const T &t) { + return (optional == optional_t::required or t.template enabled::type>()) + map_codec::fields(t); + } + static constexpr std::size_t fields() { + return (optional != optional_t::optional ? 1 : 0) + map_codec::fields(); + } + template + static std::size_t encode(const T &t, std::uint8_t *out, std::size_t out_len) { + size_t size{}; + if (optional == optional_t::required or t.template enabled::type>()) { + size = key_codec::encode(Key::to_array(), out, out_len); + size += codec>::encode(mark_t::value(), out + size, size <= out_len ? out_len - size : 0); + size += codec::encode(t.template get::type>(), out + size, size <= out_len ? out_len - size : 0); + } + size += map_codec::encode(t, out + size, size <= out_len ? out_len - size : 0); + return size; + + } + template + static std::size_t size(const T &t) { + return ( + (optional == optional_t::required or t.template enabled::type>()) ? + ( + key_codec::size() + + codec>::size(mark_t::value()) + + codec::size(t.template get::type>()) + ) : 0 + ) + map_codec::size(t); + } + static constexpr std::size_t size() { + return ( + (optional != optional_t::optional ? codec::size() : 0) ? + ( + key_codec::size() + + codec>::size(mark_t::value()) + + codec::size() + ) : 0 + ) + map_codec::size(); + } + template + static std::pair decode(T &&t, std::size_t left_keys, const std::uint8_t *in, std::size_t in_len, std::size_t recursion_limit = 0) { + std::size_t size = 0; + if (left_keys > 0) { + int key_cmp{}; + bool mark_match{}; + const auto skip_to_key = codec>::skip_to_key( + Key::to_array(), + key_cmp, + left_keys, + mark_t::value(), + mark_match, + in, + in_len, + recursion_limit + ); + size += skip_to_key.second; + if (skip_to_key.first && key_cmp == 0 && mark_match) { + --left_keys; + auto &&field = t.template get::type>(); + const auto decode_field = codec::decode( + field, + in + size, + size <= in_len ? in_len - size : 0, + recursion_limit + ); + size += decode_field.second; + if (not decode_field.first) { + return {false, size}; + } + } + else if (not skip_to_key.first) { + return {false, size}; + } + else if (not t.template set_default::type>()) { + return {false, size}; + } + } + else { + if (not t.template set_default::type>()) { + return {false, size}; + } + } + const auto decode_fields = map_codec::decode( + t, + left_keys, + in + size, + size <= in_len ? in_len - size : 0, + recursion_limit + ); + size += decode_fields.second; + return {decode_fields.first, size}; + } +}; + +template<> +struct map_codec<> { + template + static std::size_t fields(const T &) { + return 0; + } + static constexpr std::size_t fields() { + return 0; + } + template + static size_t encode(const T &, std::uint8_t *out, std::size_t out_len) { + return 0; + } + template + static size_t size(const T &) { + return 0; + } + static constexpr std::size_t size() { + return 0; + } + template + static std::pair decode(T &&, std::size_t left_keys, const std::uint8_t *in, std::size_t in_len, size_t recursion_limit = 0) { + return codec>::skip_key_value(left_keys, in, in_len, recursion_limit); + } +}; + +template +struct codec> { + static_assert(codec>::sorted>::key_selector, TT...>::value, ""); + template + static std::size_t encode(const T &t, std::uint8_t *out, std::size_t out_len) { + const std::size_t size = codec>::encode(map_codec::fields(t), out, out_len); + return size + map_codec::encode(t, out + size, size <= out_len ? out_len - size : 0); + } + template + static std::size_t size(const T &t) { + return codec>::size(map_codec::fields(t)) + map_codec::size(t); + } + static constexpr std::size_t size() { + return codec>::size(map_codec::fields()) + map_codec::size(); + } + template + static std::pair decode(T &&t, const std::uint8_t *in, std::size_t in_len, std::size_t recursion_limit = 0) { + std::size_t keys{}; + const auto parse_varint = codec>::decode(keys, in, in_len); + std::size_t size = parse_varint.second; + if (not parse_varint.first) { + return {false, size}; + } + const auto parse_fields = map_codec::decode(t, keys, in + size, size <= in_len ? in_len - size : 0, recursion_limit); + size += parse_fields.second; + return {parse_fields.first ? t.set() : false, size}; + } +}; + +template +std::size_t encode(const T &t, std::uint8_t *out, const std::size_t out_len) { + struct magic_codec { + static std::size_t encode(std::uint8_t *out, const std::size_t out_len) { + std::size_t size = codec>::encode(kSign, out, out_len); + size += codec>::encode(kVersion, out + size, size <= out_len ? out_len - size: 0); + return size; + } + }; + std::size_t size = magic_codec::encode(out, out_len); + size += codec::encode(t, out + size, size <= out_len ? out_len - size: 0); + return size; +} + +template +std::size_t size(const T &t) { + struct magic_codec { + static constexpr std::size_t size() { + return codec>::size(kSign) + codec>::size(kVersion); + } + }; + return magic_codec::size() + codec::size(t); +} + +template +std::pair decode(T &&t, const std::uint8_t *in, const std::size_t in_len, std::size_t recursion_limit = 0) { + struct magic_codec { + static std::pair skip(const std::uint8_t *in, std::size_t in_len) { + std::uint64_t sign{}; + auto parse_sign = codec>::decode(sign, in, in_len); + std::size_t size = parse_sign.second; + if (not parse_sign.first or sign != kSign) { + return {false, size}; + } + std::uint8_t version{}; + auto parse_version = codec>::decode(version, in + size, in_len - size); + size += parse_version.second; + if (not parse_version.first or version != kVersion) { + return {false, size}; + } + return {true, size}; + } + }; + const auto skip_magic = magic_codec::skip(in, in_len); + std::size_t size = skip_magic.second; + if (not skip_magic.first) { + return {false, size}; + } + const auto parse_obj = codec::decode(t, in + size, in_len - size, recursion_limit); + size += parse_obj.second; + return {parse_obj.first, size}; +} + +} + +} diff --git a/src/p2p/portable_scheme/load_store_wrappers.h b/src/p2p/portable_scheme/load_store_wrappers.h new file mode 100644 index 00000000000..dee356d0886 --- /dev/null +++ b/src/p2p/portable_scheme/load_store_wrappers.h @@ -0,0 +1,15 @@ +#pragma once + +#include "byte_slice.h" +#include "byte_stream.h" + +namespace portable_scheme { + +template +bool store_to_binary(const T &t, epee::byte_stream &out); +template +bool store_to_binary(const T &t, epee::byte_slice &out); +template +bool load_from_binary(T &t, const epee::span &in); + +} diff --git a/src/p2p/portable_scheme/load_store_wrappers_impl.h b/src/p2p/portable_scheme/load_store_wrappers_impl.h new file mode 100644 index 00000000000..5a6f4cf2de0 --- /dev/null +++ b/src/p2p/portable_scheme/load_store_wrappers_impl.h @@ -0,0 +1,41 @@ +#pragma once + +#include "load_store_wrappers.h" +#include "scheme.h" +#include "binary_codec.h" + +namespace portable_scheme { + +template +bool store_to_binary(const T &t, epee::byte_stream &out) { + using scheme = scheme_space::scheme; + const auto &&readable = typename scheme::read_t(t); + const std::size_t size = binary_codec::size(readable); + std::vector buf(size); + binary_codec::encode(readable, buf.data(), buf.size()); + out.reserve(std::max(size, out.available()) - out.available()); + out.write(static_cast(buf.data()), buf.size()); + return true; +} +template +bool store_to_binary(const T &t, epee::byte_slice &out) { + using scheme = scheme_space::scheme; + const auto &&readable = typename scheme::read_t(t); + const std::size_t size = binary_codec::size(readable); + std::vector buf(size); + binary_codec::encode(readable, buf.data(), buf.size()); + out = epee::byte_slice(std::move(buf)); + return true; +} +template +bool load_from_binary(T &t, const epee::span &in) { + using scheme = typename scheme_space::scheme; + return binary_codec::decode(typename scheme::write_t(t), in.data(), in.size(), 100).first; +} + +} + +#define PORTABLE_SCHEME_LOAD_STORE_INSTANCE(T) \ + template bool portable_scheme::store_to_binary(const T &t, epee::byte_stream &out);\ + template bool portable_scheme::store_to_binary(const T &t, epee::byte_slice &out);\ + template bool portable_scheme::load_from_binary(T &t, const epee::span &in); diff --git a/src/p2p/portable_scheme/scheme.h b/src/p2p/portable_scheme/scheme.h new file mode 100644 index 00000000000..503d16dff00 --- /dev/null +++ b/src/p2p/portable_scheme/scheme.h @@ -0,0 +1,364 @@ +#pragma once + +#include "tags.h" +#include +#include +#include +#include + +namespace portable_scheme { + +namespace scheme_space { + +using namespace tags_space; + +template +struct scheme; + +template +struct pod_tag; + +template +struct scheme> { + using tags = span_tag; + struct read_t { + private: + const T &t; + public: + read_t(const T &t): t(t) {} + std::size_t size() const { return sizeof(t); } + void get(std::uint8_t *out) const { + static_assert(std::is_pod::value, ""); + static_assert(sizeof(T) < 2 or not std::is_integral::value, ""); + static_assert(sizeof(T) < 2 or not std::is_floating_point::value, ""); + *reinterpret_cast(out) = t; + } + }; + struct write_t { + private: + T &t; + public: + write_t(T &t): t(t) {} + bool set(const void *data, std::size_t size) { + if (size != sizeof(t)) + return false; + static_assert(std::is_pod::value, ""); + static_assert(sizeof(T) < 2 or not std::is_integral::value, ""); + static_assert(sizeof(T) < 2 or not std::is_floating_point::value, ""); + t = *reinterpret_cast(data); + return true; + } + }; +}; + +template< + typename T, + bool force = false, + typename TT = typename T::value_type +> +struct container_pod_tag; + +template +struct scheme, typename std::enable_if<( + std::is_same>::value or + std::is_same, typename T::allocator_type>>::value +), void>::type> { + using TT = typename T::value_type; + using tags = span_tag<>; + struct read_t { + private: + const T &t; + public: + read_t(const T &t): t(t) {} + std::size_t size() const { return t.size() * sizeof(TT); } + void get(std::uint8_t *out) const { + static_assert(std::is_pod::value, ""); + static_assert(force or sizeof(TT) < 2 or not std::is_integral::value, ""); + static_assert(force or sizeof(TT) < 2 or not std::is_floating_point::value, ""); + std::memcpy(out, reinterpret_cast(t.data()), t.size() * sizeof(TT)); + } + }; + struct write_t { + private: + T &t; + public: + write_t(T &t): t(t) {} + bool set(const void *data, std::size_t size) { + static_assert(std::is_pod::value, ""); + static_assert(force or sizeof(TT) < 2 or not std::is_integral::value, ""); + static_assert(force or sizeof(TT) < 2 or not std::is_floating_point::value, ""); + if (sizeof(TT) == 0 || size % sizeof(TT) != 0) + return false; + t.assign( + reinterpret_cast(data), + reinterpret_cast(data) + size / sizeof(TT) + ); + return true; + } + }; +}; + +template +struct scheme, typename std::enable_if<( + std::is_same>::value +), void>::type> { + using TT = typename T::value_type; + using tags = span_tag<>; + struct read_t { + private: + const T &t; + public: + read_t(const T &t): t(t) {} + std::size_t size() const { return t.size() * sizeof(TT); } + void get(std::uint8_t *out) const { + static_assert(std::is_pod::value, ""); + static_assert(sizeof(TT) < 2 or not std::is_integral::value, ""); + static_assert(sizeof(TT) < 2 or not std::is_floating_point::value, ""); + for (auto it = t.cbegin(); it != t.cend(); ++it, out += sizeof(TT)) { + *reinterpret_cast(out) = *it; + } + } + }; + struct write_t { + private: + T &t; + public: + write_t(T &t): t(t) {} + bool set(const void *data, std::size_t size) { + static_assert(std::is_pod::value, ""); + static_assert(sizeof(TT) < 2 or not std::is_integral::value, ""); + static_assert(sizeof(TT) < 2 or not std::is_floating_point::value, ""); + if (sizeof(TT) == 0 || size % sizeof(TT) != 0) + return false; + t.assign( + reinterpret_cast(data), + reinterpret_cast(data) + size / sizeof(TT) + ); + return true; + } + }; +}; + +template +struct scheme, typename std::enable_if<( + not std::is_same::value and ( + std::is_same>::value or + std::is_same>::value + ) +), void>::type> { + using tags = span_tag<>; + struct read_t { + private: + const T &t; + public: + read_t(const T &t): t(t) {} + std::size_t size() const { return t.size() * sizeof(TT); } + void get(std::uint8_t *out) const { + static_assert(std::is_pod::value, ""); + static_assert(sizeof(TT) < 2 or not std::is_integral::value, ""); + static_assert(sizeof(TT) < 2 or not std::is_floating_point::value, ""); + for (auto it = t.cbegin(); it != t.cend(); ++it, out += sizeof(TT)) { + *reinterpret_cast(out) = *it; + } + } + }; + struct write_t { + private: + T &t; + public: + write_t(T &t): t(t) {} + bool set(const void *data, std::size_t size) { + static_assert(std::is_pod::value, ""); + static_assert(sizeof(TT) < 2 or not std::is_integral::value, ""); + static_assert(sizeof(TT) < 2 or not std::is_floating_point::value, ""); + if (sizeof(TT) == 0 || size % sizeof(TT) != 0) + return false; + t.resize(size / sizeof(TT)); + const TT *ptr = reinterpret_cast(data); + for (TT &e: t) { + e = *(ptr++); + } + return true; + } + }; +}; + +template< + typename T, + typename = scheme, + std::size_t MIN = 0, + std::size_t MAX = std::size_t(-1), + std::size_t THRESHOLD = 0, + std::size_t COMPRESSION_RATIO = 1 +> +struct container_tag; + +template +struct scheme> { + using tags = list_tag; + struct read_t { + private: + const T &t; + public: + read_t(const T &t): t(t) {} + std::size_t size() const { return t.size(); } + struct const_iterator_t { + private: + typename T::const_iterator it; + public: + const_iterator_t(const T &t): it(t.cbegin()) {} + typename S::read_t next() { + return *(it++); + } + }; + const_iterator_t cbegin() const { return t; } + }; + struct write_t { + private: + T &t; + public: + write_t(T &t): t(t) {} + template + static constexpr bool force_item_size(std::size_t items) { + static_assert(MAX != std::size_t(-1) or sizeof(typename T::value_type) <= COMPRESSION_RATIO * MIN_ITEM_SIZE, ""); + return items >= THRESHOLD; + } + bool resize(std::size_t items) { + if (items > MAX) + return false; + t.resize(items); + return true; + } + struct iterator_t { + private: + typename T::iterator it; + public: + iterator_t(T &t): it(t.begin()) {} + typename S::write_t next() { + return *(it++); + } + }; + iterator_t begin() { return t; } + }; +}; + +template +struct scheme> { + using tags = base_tag; + using read_t = const T &; + using write_t = T &; +}; + +template +struct scheme::value or std::is_floating_point::value), void>::type> { + using tags = base_tag; + using read_t = const T &; + using write_t = T &; +}; + +template +constexpr auto is_lvalue_reference(T &&) -> typename std::enable_if::value, void>::type; +#define AS_IS(...) __VA_ARGS__ +#define BEGIN_READ_RAW(...) \ + struct read_t {\ + __VA_ARGS__\ + public:\ +\ + template struct selector{}; +#define BEGIN_READ(TYPE) \ + BEGIN_READ_RAW(\ + private:\ + const TYPE &t;\ + public:\ + read_t(const TYPE &t): t(t) {}\ + ) +#define READ_FIELD_CUSTOM_OPT_NAME(NAME, MEMBER, SCHEME, ENABLER) \ + bool enabled(selector::type>) const {ENABLER}\ + typename SCHEME::read_t get(selector::type>) const {static_assert(std::is_same::value, ""); return MEMBER;} +#define READ_FIELD_CUSTOM_OPT(MEMBER, SCHEME, ENABLER) READ_FIELD_CUSTOM_OPT_NAME(#MEMBER, t.MEMBER, AS_IS(SCHEME), AS_IS(ENABLER)) +#define READ_FIELD_OPT_NAME(NAME, MEMBER, ENABLER) READ_FIELD_CUSTOM_OPT_NAME(NAME, MEMBER, scheme::type>, AS_IS(ENABLER)) +#define READ_FIELD_OPT(MEMBER, ENABLER) READ_FIELD_CUSTOM_OPT(MEMBER, scheme::type>, AS_IS(ENABLER)) +#define READ_FIELD_CUSTOM(MEMBER, SCHEME) READ_FIELD_CUSTOM_OPT(MEMBER, AS_IS(SCHEME), {return true;}) +#define READ_FIELD_WITH_DEFAULT(MEMBER, DEFAULT) READ_FIELD_OPT(MEMBER, {return t.MEMBER != DEFAULT;}) +#define READ_FIELD_NAME(NAME, MEMBER) READ_FIELD_OPT_NAME(NAME, MEMBER, {return true;}) +#define READ_FIELD(MEMBER) READ_FIELD_CUSTOM(MEMBER, scheme::type>) +#define READ_FIELD_LIST_CUSTOM_OPT(MEMBER, SCHEME, ...) READ_FIELD_CUSTOM_OPT(MEMBER, AS_IS(SCHEME), AS_IS({if (t.MEMBER.empty()) return false; __VA_ARGS__; return true;})) +#define READ_FIELD_LIST_CUSTOM_OPT_NAME(NAME, MEMBER, SCHEME, ...) READ_FIELD_CUSTOM_OPT_NAME(NAME, MEMBER, AS_IS(SCHEME), AS_IS({if (MEMBER.empty()) return false; __VA_ARGS__; return true;})) +#define READ_FIELD_LIST_OPT_NAME(NAME, MEMBER, ...) READ_FIELD_LIST_CUSTOM_OPT_NAME(NAME, MEMBER, scheme::type>>, AS_IS(__VA_ARGS__)) +#define READ_FIELD_LIST_OPT(MEMBER, ...) READ_FIELD_LIST_CUSTOM_OPT(MEMBER, scheme::type>>, AS_IS(__VA_ARGS__)) +#define READ_FIELD_LIST_CUSTOM(MEMBER, ...) READ_FIELD_LIST_CUSTOM_OPT(MEMBER, AS_IS(__VA_ARGS__)) +#define READ_FIELD_LIST_POD_OPT(MEMBER, ...) READ_FIELD_LIST_CUSTOM_OPT(MEMBER, scheme::type>>, AS_IS(__VA_ARGS__)) +#define READ_FIELD_LIST_POD_OPT_FORCE(MEMBER, ...) READ_FIELD_LIST_CUSTOM_OPT(MEMBER, AS_IS(scheme::type, true>>), AS_IS(__VA_ARGS__)) +#define READ_FIELD_LIST_POD_OPT_NAME(NAME, MEMBER, ...) READ_FIELD_LIST_CUSTOM_OPT_NAME(NAME, MEMBER, scheme::type>>, AS_IS(__VA_ARGS__)) +#define READ_FIELD_LIST_POD(MEMBER) READ_FIELD_LIST_CUSTOM(MEMBER, scheme::type>>) +#define READ_FIELD_LIST_POD_FORCE(MEMBER) READ_FIELD_LIST_CUSTOM(MEMBER, AS_IS(scheme::type, true>>)) +#define READ_FIELD_LIST_POD_OPT_NAME_FORCE(NAME, MEMBER, ...) READ_FIELD_LIST_CUSTOM_OPT_NAME(NAME, MEMBER, AS_IS(scheme::type, true>>), AS_IS(__VA_ARGS__)) +#define READ_FIELD_LIST(MEMBER) READ_FIELD_LIST_OPT(MEMBER) +#define READ_FIELD_POD(MEMBER) READ_FIELD_CUSTOM(MEMBER, scheme::type>>) +#define READ_FIELD_POD_OPT(MEMBER, ENABLER) READ_FIELD_CUSTOM_OPT(MEMBER, scheme::type>>, AS_IS(ENABLER)) +#define READ_FIELD_POD_OPT_NAME(NAME, MEMBER, ENABLER) READ_FIELD_CUSTOM_OPT_NAME(NAME, MEMBER, scheme::type>>, AS_IS(ENABLER)) +#define END_READ() \ + template bool enabled() const { return enabled(selector<__T...>()); }\ + template auto get() const -> decltype(get(selector<__T...>())) { return get(selector<__T...>()); }\ + }; +#define READ(TYPE) \ + BEGIN_READ_RAW(\ + public:\ + read_t(const TYPE &t) {}\ + )\ + END_READ() +#define BEGIN_WRITE_RAW(...) \ + struct write_t {\ + __VA_ARGS__\ + public:\ + template struct selector{}; +#define BEGIN_WRITE(TYPE) \ + BEGIN_WRITE_RAW(\ + private:\ + TYPE &t;\ + public:\ + write_t(TYPE &t): t(t) {}\ + ) +#define WRITE_FIELD_CUSTOM_OPT_NAME(NAME, MEMBER, SCHEME, SET_DEFAULT) \ + bool set_default(selector::type>) {SET_DEFAULT}\ + typename SCHEME::write_t get(selector::type>) {static_assert(std::is_same::value, ""); return MEMBER;} +#define WRITE_FIELD_CUSTOM_OPT(MEMBER, SCHEME, SET_DEFAULT) WRITE_FIELD_CUSTOM_OPT_NAME(#MEMBER, t.MEMBER, AS_IS(SCHEME), AS_IS(SET_DEFAULT)) +#define WRITE_FIELD_OPT_NAME(NAME, MEMBER, ENABLER) WRITE_FIELD_CUSTOM_OPT_NAME(NAME, MEMBER, scheme::type>, AS_IS(ENABLER)) +#define WRITE_FIELD_OPT(MEMBER, ENABLER) WRITE_FIELD_CUSTOM_OPT(MEMBER, scheme::type>, AS_IS(ENABLER)) +#define WRITE_FIELD_CUSTOM(MEMBER, SCHEME) WRITE_FIELD_CUSTOM_OPT(MEMBER, AS_IS(SCHEME), {return false;}) +#define WRITE_FIELD_WITH_DEFAULT(MEMBER, DEFAULT) WRITE_FIELD_OPT(MEMBER, {t.MEMBER = DEFAULT; return true;}) +#define WRITE_FIELD(MEMBER) WRITE_FIELD_OPT(MEMBER, {return false;}) +#define WRITE_FIELD_NAME(NAME, MEMBER) WRITE_FIELD_CUSTOM_OPT_NAME(NAME, MEMBER, scheme::type>, {return false;}) +#define WRITE_FIELD_LIST_CUSTOM_OPT(MEMBER, SCHEME, ...) WRITE_FIELD_CUSTOM_OPT(MEMBER, AS_IS(SCHEME), {AS_IS(__VA_ARGS__); return true;}) +#define WRITE_FIELD_LIST_CUSTOM_OPT_NAME(NAME, MEMBER, SCHEME, ...) WRITE_FIELD_CUSTOM_OPT_NAME(NAME, MEMBER, AS_IS(SCHEME), {AS_IS(__VA_ARGS__); return true;}) +#define WRITE_FIELD_LIST_OPT_NAME(NAME, MEMBER, ...) WRITE_FIELD_LIST_CUSTOM_OPT_NAME(NAME, MEMBER, scheme::type>>, AS_IS(__VA_ARGS__)) +#define WRITE_FIELD_LIST_OPT(MEMBER, ...) WRITE_FIELD_LIST_CUSTOM_OPT(MEMBER, scheme::type>>, AS_IS(__VA_ARGS__)) +#define WRITE_FIELD_LIST_CUSTOM(MEMBER, ...) WRITE_FIELD_LIST_CUSTOM_OPT(MEMBER, AS_IS(__VA_ARGS__), {t.MEMBER.clear();}) +#define WRITE_FIELD_LIST_POD_OPT(MEMBER, ...) WRITE_FIELD_LIST_CUSTOM_OPT(MEMBER, scheme::type>>, AS_IS(__VA_ARGS__)) +#define WRITE_FIELD_LIST_POD_OPT_FORCE(MEMBER, ...) WRITE_FIELD_LIST_CUSTOM_OPT(MEMBER, AS_IS(scheme::type, true>>), AS_IS(__VA_ARGS__)) +#define WRITE_FIELD_LIST_POD_OPT_NAME(NAME, MEMBER, ...) WRITE_FIELD_LIST_CUSTOM_OPT_NAME(NAME, MEMBER, scheme::type>>, AS_IS(__VA_ARGS__)) +#define WRITE_FIELD_LIST_POD(MEMBER) WRITE_FIELD_LIST_CUSTOM(MEMBER, scheme::type>>) +#define WRITE_FIELD_LIST_POD_FORCE(MEMBER) WRITE_FIELD_LIST_CUSTOM(MEMBER, AS_IS(scheme::type, true>>)) +#define WRITE_FIELD_LIST_POD_OPT_NAME_FORCE(NAME, MEMBER, ...) WRITE_FIELD_LIST_CUSTOM_OPT_NAME(NAME, MEMBER, AS_IS(scheme::type, true>>), AS_IS(__VA_ARGS__)) +#define WRITE_FIELD_LIST(MEMBER) WRITE_FIELD_LIST_CUSTOM(MEMBER, scheme::type>>) +#define WRITE_FIELD_POD(MEMBER) WRITE_FIELD_CUSTOM(MEMBER, scheme::type>>) +#define WRITE_FIELD_POD_WITH_DEFAULT(MEMBER, DEFAULT) WRITE_FIELD_CUSTOM_OPT(MEMBER, scheme::type>>, {t.MEMBER = DEFAULT; return true;}) +#define WRITE_FIELD_POD_OPT(MEMBER, ENABLER) WRITE_FIELD_CUSTOM_OPT(MEMBER, scheme::type>>, AS_IS(ENABLER)) +#define WRITE_FIELD_POD_OPT_NAME(NAME, MEMBER, ENABLER) WRITE_FIELD_CUSTOM_OPT_NAME(NAME, MEMBER, scheme::type>>, AS_IS(ENABLER)) +#define WRITE_CHECK(...) bool set(selector) {__VA_ARGS__} +#define END_WRITE() \ + template bool set_default() { return set_default(selector<__T...>()); }\ + template auto get() -> decltype(get(selector<__T...>())) { return get(selector<__T...>()); }\ + template bool set(selector<__T...>) { return true; }\ + bool set() { return set(selector()); }\ + }; +#define WRITE(TYPE) \ + BEGIN_WRITE_RAW(\ + public:\ + write_t(TYPE &t) {}\ + )\ + END_WRITE() + +} + +} diff --git a/src/p2p/portable_scheme/tags.h b/src/p2p/portable_scheme/tags.h new file mode 100644 index 00000000000..433bd82ce75 --- /dev/null +++ b/src/p2p/portable_scheme/tags.h @@ -0,0 +1,114 @@ +#pragma once + +#include + +namespace portable_scheme { + +namespace tags_space { + +enum struct endian_t: std::uint8_t { + little, + big, + native, +}; + +template +struct base_tag; + +template +struct varint_tag; + +template +struct span_tag; + +enum struct optional_t { + optional, + required, + exception, +}; + +template +struct field_tag; + +template +struct map_tag; + +template +struct list_tag; + +template +struct field_key; + +template +struct field_key> { + using type = base_tag; +}; + +template +struct field_key> { + using type = varint_tag; +}; + +template +struct field_key> { + using type = span_tag<>; +}; + +template +struct field_key> { + using type = map_tag<>; +}; + +template +struct field_key> { + using type = list_tag::type>; +}; + +} + +namespace key_space { + +#if 0 +https://github.com/irrequietus/typestring/blob/master/typestring.hh +#endif + +template +struct key_t { + static constexpr char const data[sizeof...(C)+1] = {C..., '\0'}; + static constexpr unsigned int size = sizeof...(C); + struct array_t { + char const data[sizeof...(C)+1] = {C..., '\0'}; + static constexpr unsigned int size = sizeof...(C); + }; + static constexpr array_t to_array() { return {}; } +}; + +template +auto second(key_t) -> key_t; + +template +auto second(key_t, key_t<'\0'>, key_t...) -> key_t; + +template +auto second(key_t, key_t, key_t...) -> decltype(second(key_t(), key_t()...)); + +template +auto third(key_t) -> decltype(second(key_t()...)); + +template +constexpr char fourth(const char (&c)[B]) { + static_assert(B < 32, ""); + return c[A < B ? A : B - 1]; +} + +#define REP2(F, I, A) F(I, A), F(I + 1, A) +#define REP4(F, I, A) REP2(F, I, A), REP2(F, I + 2, A) +#define REP8(F, I, A) REP4(F, I, A), REP4(F, I + 4, A) +#define REP16(F, I, A) REP8(F, I, A), REP8(F, I + 8, A) +#define REP32(F, I, A) REP16(F, I, A), REP16(F, I + 16, A) +#define THIRD(I, A) portable_scheme::key_space::fourth(A) +#define KEY(A) decltype(portable_scheme::key_space::third(portable_scheme::key_space::key_t())) + +} + +} diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt index 15e433e10ac..b72bed87c46 100644 --- a/src/rpc/CMakeLists.txt +++ b/src/rpc/CMakeLists.txt @@ -31,6 +31,7 @@ include_directories(SYSTEM ${ZMQ_INCLUDE_PATH}) set(rpc_base_sources rpc_args.cpp rpc_payment_signature.cpp + core_rpc_server_commands_serialization.cpp rpc_handler.cpp) set(rpc_sources diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index da36f3c6479..6c1c053f2de 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -2197,8 +2197,23 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - template - bool core_rpc_server::use_bootstrap_daemon_if_necessary(const invoke_http_mode &mode, const std::string &command_name, const typename COMMAND_TYPE::request& req, typename COMMAND_TYPE::response& res, bool &r) + template<> + struct core_rpc_server::bootstrap_daemon_invoke { + template + static bool invoke(bootstrap_daemon &daemon, T... t) { return daemon.invoke_http_json(t...); } + }; + template<> + struct core_rpc_server::bootstrap_daemon_invoke { + template + static bool invoke(bootstrap_daemon &daemon, T... t) { return daemon.invoke_http_bin(t...); } + }; + template<> + struct core_rpc_server::bootstrap_daemon_invoke { + template + static bool invoke(bootstrap_daemon &daemon, T... t) { return daemon.invoke_http_json_rpc(t...); } + }; + template + bool core_rpc_server::use_bootstrap_daemon_if_necessary(T, const std::string &command_name, const typename COMMAND_TYPE::request& req, typename COMMAND_TYPE::response& res, bool &r) { res.untrusted = false; @@ -2249,23 +2264,7 @@ namespace cryptonote } } - if (mode == invoke_http_mode::JON) - { - r = m_bootstrap_daemon->invoke_http_json(command_name, req, res); - } - else if (mode == invoke_http_mode::BIN) - { - r = m_bootstrap_daemon->invoke_http_bin(command_name, req, res); - } - else if (mode == invoke_http_mode::JON_RPC) - { - r = m_bootstrap_daemon->invoke_http_json_rpc(command_name, req, res); - } - else - { - MERROR("Unknown invoke_http_mode: " << mode); - return false; - } + r = bootstrap_daemon_invoke::invoke(*m_bootstrap_daemon, command_name, req, res); { boost::upgrade_to_unique_lock lock(upgrade_lock); diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 84b14383aa7..5e5f8eec9f3 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -282,9 +282,18 @@ namespace cryptonote const std::string &address, const boost::optional &credentials, const std::string &proxy); - enum invoke_http_mode { JON, BIN, JON_RPC }; - template - bool use_bootstrap_daemon_if_necessary(const invoke_http_mode &mode, const std::string &command_name, const typename COMMAND_TYPE::request& req, typename COMMAND_TYPE::response& res, bool &r); + struct invoke_http_mode { + enum struct mode_t { JON, BIN, JON_RPC }; + template + struct tag { static constexpr mode_t value = mode; }; + static constexpr tag JON{}; + static constexpr tag BIN{}; + static constexpr tag JON_RPC{}; + }; + template + struct bootstrap_daemon_invoke; + template + bool use_bootstrap_daemon_if_necessary(T, const std::string &command_name, const typename COMMAND_TYPE::request& req, typename COMMAND_TYPE::response& res, bool &r); bool get_block_template(const account_public_address &address, const crypto::hash *prev_block, const cryptonote::blobdata &extra_nonce, size_t &reserved_offset, cryptonote::difficulty_type &difficulty, uint64_t &height, uint64_t &expected_reward, block &b, uint64_t &seed_height, crypto::hash &seed_hash, crypto::hash &next_seed_hash, epee::json_rpc::error &error_resp); bool check_payment(const std::string &client, uint64_t payment, const std::string &rpc, bool same_ts, std::string &message, uint64_t &credits, std::string &top_hash); diff --git a/src/rpc/core_rpc_server_commands_serialization.cpp b/src/rpc/core_rpc_server_commands_serialization.cpp new file mode 100644 index 00000000000..6cc6b9d09fc --- /dev/null +++ b/src/rpc/core_rpc_server_commands_serialization.cpp @@ -0,0 +1,15 @@ +#include "rpc/core_rpc_server_commands_serialization.h" +#include "p2p/portable_scheme/load_store_wrappers_impl.h" + +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::request) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::response) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(cryptonote::COMMAND_RPC_GET_BLOCKS_BY_HEIGHT::request) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(cryptonote::COMMAND_RPC_GET_BLOCKS_BY_HEIGHT::response) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(cryptonote::COMMAND_RPC_GET_HASHES_FAST::request) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(cryptonote::COMMAND_RPC_GET_HASHES_FAST::response) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(cryptonote::COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(cryptonote::COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(cryptonote::COMMAND_RPC_GET_OUTPUTS_BIN::request) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(cryptonote::COMMAND_RPC_GET_OUTPUTS_BIN::response) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(cryptonote::COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(cryptonote::COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response) diff --git a/src/rpc/core_rpc_server_commands_serialization.h b/src/rpc/core_rpc_server_commands_serialization.h new file mode 100644 index 00000000000..80e47be3765 --- /dev/null +++ b/src/rpc/core_rpc_server_commands_serialization.h @@ -0,0 +1,460 @@ +#pragma once + +#include "rpc/core_rpc_server_commands_defs.h" +#include "p2p/portable_scheme/scheme.h" +#include "cryptonote_protocol/cryptonote_protocol_serialization.h" + +namespace portable_scheme { + +namespace scheme_space { + +template<> +struct scheme { + using T = cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::request; + using tags = map_tag< + field_tag>, + field_tag>, + field_tag>, + field_tag>, + field_tag> + >; + BEGIN_READ(T) + READ_FIELD_LIST_POD(block_ids) + READ_FIELD_LIST_POD(client) + READ_FIELD_WITH_DEFAULT(no_miner_tx, false) + READ_FIELD(prune) + READ_FIELD(start_height) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD_LIST_POD(block_ids) + WRITE_FIELD_LIST_POD(client) + WRITE_FIELD_WITH_DEFAULT(no_miner_tx, false) + WRITE_FIELD(prune) + WRITE_FIELD(start_height) + END_WRITE() +}; + +template<> +struct scheme { + using T = cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices; + using tags = map_tag< + field_tag, 1>, optional_t::exception> + >; + BEGIN_READ(T) + READ_FIELD_LIST(indices) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD_LIST(indices) + END_WRITE() +}; + +template<> +struct scheme { + using T = cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices; + using indices_t = scheme, 0, std::size_t(-1), 514 + 1, 2>>; + using tags = map_tag< + field_tag + >; + BEGIN_READ(T) + READ_FIELD_LIST_CUSTOM(indices, indices_t) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD_LIST_CUSTOM(indices, indices_t) + END_WRITE() +}; + +template<> +struct scheme { + using T = cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::response; + using output_indices_t = scheme, + 0, + COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT + >>; + using tags = map_tag< + field_tag>::tags>, + field_tag>, + field_tag>, + field_tag, + field_tag>, + field_tag>::tags>, + field_tag>, + field_tag> + >; + BEGIN_READ(T) + READ_FIELD_LIST(blocks) + READ_FIELD(credits) + READ_FIELD(current_height) + READ_FIELD_LIST_CUSTOM(output_indices, output_indices_t) + READ_FIELD(start_height) + READ_FIELD_LIST_POD(status) + READ_FIELD_LIST_POD(top_hash) + READ_FIELD(untrusted) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD_LIST(blocks) + WRITE_FIELD(credits) + WRITE_FIELD(current_height) + WRITE_FIELD_LIST_CUSTOM(output_indices, output_indices_t) + WRITE_FIELD(start_height) + WRITE_FIELD_LIST_POD(status) + WRITE_FIELD_LIST_POD(top_hash) + WRITE_FIELD(untrusted) + END_WRITE() +}; + +template<> +struct scheme { + using T = cryptonote::COMMAND_RPC_GET_BLOCKS_BY_HEIGHT::request; + using tags = map_tag< + field_tag>, + field_tag>> + >; + BEGIN_READ(T) + READ_FIELD_LIST_POD(client) + READ_FIELD_LIST(heights) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD_LIST_POD(client) + WRITE_FIELD_LIST(heights) + END_WRITE() +}; + +template<> +struct scheme { + using T = cryptonote::COMMAND_RPC_GET_BLOCKS_BY_HEIGHT::response; + using tags = map_tag< + field_tag>::tags>, + field_tag>, + field_tag>::tags>, + field_tag>, + field_tag> + >; + BEGIN_READ(T) + READ_FIELD_LIST(blocks) + READ_FIELD(credits) + READ_FIELD_LIST_POD(status) + READ_FIELD_LIST_POD(top_hash) + READ_FIELD(untrusted) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD_LIST(blocks) + WRITE_FIELD(credits) + WRITE_FIELD_LIST_POD(status) + WRITE_FIELD_LIST_POD(top_hash) + WRITE_FIELD(untrusted) + END_WRITE() +}; + +template<> +struct scheme { + using T = cryptonote::COMMAND_RPC_GET_HASHES_FAST::request; + using tags = map_tag< + field_tag>, + field_tag>, + field_tag> + >; + BEGIN_READ(T) + READ_FIELD_LIST_POD(block_ids) + READ_FIELD_LIST_POD(client) + READ_FIELD(start_height) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD_LIST_POD(block_ids) + WRITE_FIELD_LIST_POD(client) + WRITE_FIELD(start_height) + END_WRITE() +}; + +template<> +struct scheme { + using T = cryptonote::COMMAND_RPC_GET_HASHES_FAST::response; + using tags = map_tag< + field_tag>, + field_tag>, + field_tag>, + field_tag>, + field_tag>::tags>, + field_tag>, + field_tag> + >; + BEGIN_READ(T) + READ_FIELD(credits) + READ_FIELD(current_height) + READ_FIELD_LIST_POD(m_block_ids) + READ_FIELD(start_height) + READ_FIELD_LIST_POD(status) + READ_FIELD_LIST_POD(top_hash) + READ_FIELD(untrusted) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD(credits) + WRITE_FIELD(current_height) + WRITE_FIELD_LIST_POD(m_block_ids) + WRITE_FIELD(start_height) + WRITE_FIELD_LIST_POD(status) + WRITE_FIELD_LIST_POD(top_hash) + WRITE_FIELD(untrusted) + END_WRITE() +}; + +template<> +struct scheme { + using T = cryptonote::COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request; + using tags = map_tag< + field_tag>, + field_tag> + >; + BEGIN_READ(T) + READ_FIELD_LIST_POD(client) + READ_FIELD_POD(txid) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD_LIST_POD(client) + WRITE_FIELD_POD(txid) + END_WRITE() +}; + +template<> +struct scheme { + using T = cryptonote::COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response; + using tags = map_tag< + field_tag>, + field_tag>>, + field_tag>::tags>, + field_tag>, + field_tag> + >; + BEGIN_READ(T) + READ_FIELD(credits) + READ_FIELD_LIST(o_indexes) + READ_FIELD_LIST_POD(status) + READ_FIELD_LIST_POD(top_hash) + READ_FIELD(untrusted) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD(credits) + WRITE_FIELD_LIST(o_indexes) + WRITE_FIELD_LIST_POD(status) + WRITE_FIELD_LIST_POD(top_hash) + WRITE_FIELD(untrusted) + END_WRITE() +}; + +template<> +struct scheme { + using T = cryptonote::get_outputs_out; + using tags = map_tag< + field_tag, optional_t::required>, + field_tag, optional_t::required> + >; + BEGIN_READ(T) + READ_FIELD(amount) + READ_FIELD(index) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD(amount) + WRITE_FIELD(index) + END_WRITE() +}; + +template<> +struct scheme { + using T = cryptonote::COMMAND_RPC_GET_OUTPUTS_BIN::request; + using tags = map_tag< + field_tag>, + field_tag>, + field_tag>::tags> + >; + BEGIN_READ(T) + READ_FIELD_LIST_POD(client) + READ_FIELD_WITH_DEFAULT(get_txid, true) + READ_FIELD_LIST(outputs) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD_LIST_POD(client) + WRITE_FIELD_WITH_DEFAULT(get_txid, true) + WRITE_FIELD_LIST(outputs) + END_WRITE() +}; + +template<> +struct scheme { + using T = cryptonote::COMMAND_RPC_GET_OUTPUTS_BIN::outkey; + using tags = map_tag< + field_tag, optional_t::required>, + field_tag>::tags, optional_t::required>, + field_tag>::tags, optional_t::required>, + field_tag>::tags, optional_t::required>, + field_tag, optional_t::required> + >; + BEGIN_READ(T) + READ_FIELD(height) + READ_FIELD_POD(key) + READ_FIELD_POD(mask) + READ_FIELD_POD(txid) + READ_FIELD(unlocked) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD(height) + WRITE_FIELD_POD(key) + WRITE_FIELD_POD(mask) + WRITE_FIELD_POD(txid) + WRITE_FIELD(unlocked) + END_WRITE() +}; + +template<> +struct scheme { + using T = cryptonote::COMMAND_RPC_GET_OUTPUTS_BIN::response; + using tags = map_tag< + field_tag>, + field_tag>::tags>, + field_tag>::tags>, + field_tag>, + field_tag> + >; + BEGIN_READ(T) + READ_FIELD(credits) + READ_FIELD_LIST(outs) + READ_FIELD_LIST_POD(status) + READ_FIELD_LIST_POD(top_hash) + READ_FIELD(untrusted) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD(credits) + WRITE_FIELD_LIST(outs) + WRITE_FIELD_LIST_POD(status) + WRITE_FIELD_LIST_POD(top_hash) + WRITE_FIELD(untrusted) + END_WRITE() +}; + +template<> +struct scheme { + using T = cryptonote::COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request; + using tags = map_tag< + field_tag>>, + field_tag>, + field_tag>, + field_tag>, + field_tag>, + field_tag>, + field_tag> + >; + BEGIN_READ(T) + READ_FIELD_LIST(amounts) + READ_FIELD_WITH_DEFAULT(binary, true) + READ_FIELD_LIST_POD(client) + READ_FIELD_WITH_DEFAULT(compress, false) + READ_FIELD_WITH_DEFAULT(cumulative, false) + READ_FIELD_WITH_DEFAULT(from_height, 0) + READ_FIELD_WITH_DEFAULT(to_height, 0) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD_LIST(amounts) + WRITE_FIELD_WITH_DEFAULT(binary, true) + WRITE_FIELD_LIST_POD(client) + WRITE_FIELD_WITH_DEFAULT(compress, false) + WRITE_FIELD_WITH_DEFAULT(cumulative, false) + WRITE_FIELD_WITH_DEFAULT(from_height, 0) + WRITE_FIELD_WITH_DEFAULT(to_height, 0) + END_WRITE() +}; + +template<> +struct scheme { + using T = cryptonote::COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::distribution; + using tags = map_tag< + field_tag, optional_t::required>, + field_tag, optional_t::required>, + field_tag, optional_t::required>, + field_tag, optional_t::required>, + field_tag>, + field_tag>>, + #if 1 + field_tag>, + #endif + field_tag, optional_t::required> + >; + BEGIN_READ_RAW( + private: + const T &t; + const std::string compressed_data; + public: + read_t(const T &t): t(t), compressed_data(t.compress and t.binary ? compress_integer_array(t.data.distribution) : std::string{}) {} + ) + READ_FIELD(amount) + READ_FIELD(binary) + READ_FIELD(compress) + READ_FIELD_NAME("base", t.data.base) + READ_FIELD_NAME("start_height", t.data.start_height) + #if 1 + READ_FIELD_LIST_OPT_NAME("distribution", t.data.distribution, {return not t.binary;}) + READ_FIELD_LIST_POD_OPT_NAME_FORCE("distribution", t.data.distribution, {return t.binary and not t.compress;}) + READ_FIELD_LIST_POD_OPT_NAME("compressed_data", compressed_data, {return t.binary and t.compress;}) + #else + READ_FIELD_LIST_OPT_NAME("distribution", t.data.distribution, {return not t.binary and not t.compress;}) + READ_FIELD_LIST_POD_OPT_NAME("compressed_data", compressed_data, {return t.binary and t.compress;}) + #endif + END_READ() + + BEGIN_WRITE_RAW( + private: + T &t; + std::string compressed_data{}; + public: + write_t(T &t): t(t) { t.data.distribution.clear(); } + ) + WRITE_FIELD(amount) + WRITE_FIELD(binary) + WRITE_FIELD(compress) + WRITE_FIELD_NAME("base", t.data.base) + WRITE_FIELD_NAME("start_height", t.data.start_height) + #if 1 + WRITE_FIELD_LIST_OPT_NAME("distribution", t.data.distribution) + WRITE_FIELD_LIST_POD_OPT_NAME_FORCE("distribution", t.data.distribution) + WRITE_FIELD_LIST_POD_OPT_NAME("compressed_data", compressed_data) + #else + WRITE_FIELD_LIST_OPT_NAME("distribution", t.data.distribution) + WRITE_FIELD_LIST_POD_OPT_NAME("compressed_data", compressed_data) + #endif + WRITE_CHECK({ + if (t.binary and t.compress) + t.data.distribution = decompress_integer_array(compressed_data); + return true; + }) + END_WRITE() +}; + +template<> +struct scheme { + using T = cryptonote::COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response; + using distributions_t = scheme, 0, std::size_t(-1), 0, 2>>; + using tags = map_tag< + field_tag>, + field_tag, + field_tag>::tags>, + field_tag>, + field_tag> + >; + BEGIN_READ(T) + READ_FIELD(credits) + READ_FIELD_LIST_CUSTOM(distributions, distributions_t) + READ_FIELD_LIST_POD(status) + READ_FIELD_LIST_POD(top_hash) + READ_FIELD(untrusted) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD(credits) + WRITE_FIELD_LIST_CUSTOM(distributions, distributions_t) + WRITE_FIELD_LIST_POD(status) + WRITE_FIELD_LIST_POD(top_hash) + WRITE_FIELD(untrusted) + END_WRITE() +}; + +} + +} diff --git a/tests/net_load_tests/CMakeLists.txt b/tests/net_load_tests/CMakeLists.txt index c334df4ba1d..0440387c103 100644 --- a/tests/net_load_tests/CMakeLists.txt +++ b/tests/net_load_tests/CMakeLists.txt @@ -26,6 +26,9 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +monero_add_library(net_load_tests_serialization + net_load_tests_serialization.cpp) + set(clt_sources clt.cpp) @@ -40,6 +43,7 @@ target_link_libraries(net_load_tests_clt p2p cryptonote_core epee + net_load_tests_serialization ${GTEST_LIBRARIES} ${Boost_CHRONO_LIBRARY} ${Boost_DATE_TIME_LIBRARY} @@ -62,6 +66,7 @@ target_link_libraries(net_load_tests_srv p2p cryptonote_core epee + net_load_tests_serialization ${GTEST_LIBRARIES} ${Boost_CHRONO_LIBRARY} ${Boost_DATE_TIME_LIBRARY} diff --git a/tests/net_load_tests/net_load_tests_serialization.cpp b/tests/net_load_tests/net_load_tests_serialization.cpp new file mode 100644 index 00000000000..b93682d86a8 --- /dev/null +++ b/tests/net_load_tests/net_load_tests_serialization.cpp @@ -0,0 +1,14 @@ +#include "net_load_tests_serialization.h" +#include "p2p/portable_scheme/load_store_wrappers_impl.h" + +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(net_load_tests::CMD_CLOSE_ALL_CONNECTIONS::request) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(net_load_tests::CMD_START_OPEN_CLOSE_TEST::request) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(net_load_tests::CMD_START_OPEN_CLOSE_TEST::response) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(net_load_tests::CMD_GET_STATISTICS::request) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(net_load_tests::CMD_GET_STATISTICS::response) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(net_load_tests::CMD_RESET_STATISTICS::request) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(net_load_tests::CMD_RESET_STATISTICS::response) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(net_load_tests::CMD_SHUTDOWN::request) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(net_load_tests::CMD_SEND_DATA_REQUESTS::request) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(net_load_tests::CMD_DATA_REQUEST::request) +PORTABLE_SCHEME_LOAD_STORE_INSTANCE(net_load_tests::CMD_DATA_REQUEST::response) diff --git a/tests/net_load_tests/net_load_tests_serialization.h b/tests/net_load_tests/net_load_tests_serialization.h new file mode 100644 index 00000000000..83c1aa4f935 --- /dev/null +++ b/tests/net_load_tests/net_load_tests_serialization.h @@ -0,0 +1,133 @@ +#pragma once + +#include "net_load_tests.h" +#include "p2p/portable_scheme/scheme.h" + +namespace portable_scheme { + +namespace scheme_space { + +template<> +struct scheme { + using T = net_load_tests::CMD_CLOSE_ALL_CONNECTIONS::request; + using tags = map_tag<>; + READ(T) + WRITE(T) +}; + +template<> +struct scheme { + using T = net_load_tests::CMD_START_OPEN_CLOSE_TEST::request; + using tags = map_tag< + field_tag>, + field_tag> + >; + BEGIN_READ(T) + READ_FIELD(open_request_target) + READ_FIELD(max_opened_conn_count) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD(open_request_target) + WRITE_FIELD(max_opened_conn_count) + END_WRITE() +}; + +template<> +struct scheme { + using T = net_load_tests::CMD_START_OPEN_CLOSE_TEST::response; + using tags = map_tag<>; + READ(T) + WRITE(T) +}; + +template<> +struct scheme { + using T = net_load_tests::CMD_GET_STATISTICS::request; + using tags = map_tag<>; + READ(T) + WRITE(T) +}; + +template<> +struct scheme { + using T = net_load_tests::CMD_GET_STATISTICS::response; + using tags = map_tag< + field_tag>, + field_tag>, + field_tag> + >; + BEGIN_READ(T) + READ_FIELD(opened_connections_count) + READ_FIELD(new_connection_counter) + READ_FIELD(close_connection_counter) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD(opened_connections_count) + WRITE_FIELD(new_connection_counter) + WRITE_FIELD(close_connection_counter) + END_WRITE() +}; + +template<> +struct scheme { + using T = net_load_tests::CMD_RESET_STATISTICS::request; + using tags = map_tag<>; + READ(T) + WRITE(T) +}; + +template<> +struct scheme { + using T = net_load_tests::CMD_RESET_STATISTICS::response; + using tags = map_tag<>; + READ(T) + WRITE(T) +}; + +template<> +struct scheme { + using T = net_load_tests::CMD_SHUTDOWN::request; + using tags = map_tag<>; + READ(T) + WRITE(T) +}; + +template<> +struct scheme { + using T = net_load_tests::CMD_SEND_DATA_REQUESTS::request; + using tags = map_tag< + field_tag> + >; + BEGIN_READ(T) + READ_FIELD(request_size) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD(request_size) + END_WRITE() +}; + +template<> +struct scheme { + using T = net_load_tests::CMD_DATA_REQUEST::request; + using tags = map_tag< + field_tag> + >; + BEGIN_READ(T) + READ_FIELD_LIST_POD(data) + END_READ() + BEGIN_WRITE(T) + WRITE_FIELD_LIST_POD(data) + END_WRITE() +}; + +template<> +struct scheme { + using T = net_load_tests::CMD_DATA_REQUEST::response; + using tags = map_tag<>; + READ(T) + WRITE(T) +}; + +} + +} diff --git a/tests/unit_tests/CMakeLists.txt b/tests/unit_tests/CMakeLists.txt index 556e0ec4033..56573aa7563 100644 --- a/tests/unit_tests/CMakeLists.txt +++ b/tests/unit_tests/CMakeLists.txt @@ -74,6 +74,7 @@ set(unit_tests_sources output_distribution.cpp parse_amount.cpp pruning.cpp + p2p_portable_scheme.cpp random.cpp rolling_median.cpp serialization.cpp diff --git a/tests/unit_tests/node_server.cpp b/tests/unit_tests/node_server.cpp index 7907e9a9a6d..abf3b05be6c 100644 --- a/tests/unit_tests/node_server.cpp +++ b/tests/unit_tests/node_server.cpp @@ -1088,10 +1088,11 @@ TEST(node_server, race_condition) context_t context; conn->get_context(context); event_t handshaked; - typename messages::handshake::request_t msg{{ + typename messages::handshake::request msg; + msg.node_data = { ::config::NETWORK_ID, 58080, - }}; + }; epee::net_utils::async_invoke_remote_command2( context, messages::handshake::ID, diff --git a/tests/unit_tests/p2p_portable_scheme.cpp b/tests/unit_tests/p2p_portable_scheme.cpp new file mode 100644 index 00000000000..226b8e8e4f2 --- /dev/null +++ b/tests/unit_tests/p2p_portable_scheme.cpp @@ -0,0 +1,136 @@ +#include + +#include "byte_slice.h" +#include "cryptonote_basic/cryptonote_format_utils.h" +#include "cryptonote_protocol/cryptonote_protocol_defs.h" +#include "p2p/p2p_protocol_defs.h" +#include "storages/portable_storage_template_helper.h" + +#include +#include + +TEST(portable_scheme_binary_codec_large_packets, dos_1) +{ + std::vector sample{0x01, 0x11, 0x01, 0x01, 0x01, 0x01, 0x02, 0x01, 0x01, 0x04, 0x01, 0x61, 0x8c, 0x32, 0xd9, 0xf5, 0x05}; + if (sample.size() < 100 * 1000 * 1000) { + std::size_t offset = sample.size(); + sample.resize(100 * 1000 * 1000); + for (const std::vector e{0x04, 0x00, 0x8d, 0x00}; offset + e.size() < sample.size(); offset += e.size()) { + std::memcpy(sample.data() + offset, e.data(), e.size()); + } + } + EXPECT_TRUE(sample.size() >= 99e+6 and sample.size() < 105e+6); + nodetool::COMMAND_PING::request in{}; + EXPECT_TRUE(epee::serialization::load_t_from_binary(in, {sample.data(), sample.size()})); +} + +TEST(portable_scheme_binary_codec_large_packets, dos_2) +{ + std::vector sample{0x01, 0x11, 0x01, 0x01, 0x01, 0x01, 0x02, 0x01, 0x01, 0x04, 0x01, 0x61, 0x8c, 0xbe, 0x83, 0xd7, 0x17}; + sample.resize(100 * 1000 * 1000, {}); + EXPECT_TRUE(sample.size() >= 99e+6 and sample.size() < 105e+6); + nodetool::COMMAND_PING::request in{}; + EXPECT_TRUE(epee::serialization::load_t_from_binary(in, {sample.data(), sample.size()})); +} + +TEST(portable_scheme_binary_codec_large_packets, too_small_txs_blobs) +{ + using T = cryptonote::NOTIFY_NEW_TRANSACTIONS::request; + epee::byte_slice sample; + if (sample.empty()) { + T out{}; + out.txs.resize(100 * 1000 * 1000 / (1 + 41 - 1), std::string(41 - 1, '\x00')); + epee::serialization::store_t_to_binary(out, sample); + } + EXPECT_TRUE(sample.size() >= 99e+6 and sample.size() < 105e+6); + T in{}; + EXPECT_FALSE(epee::serialization::load_t_from_binary(in, {sample.data(), sample.size()})); +} + +TEST(portable_scheme_binary_codec_large_packets, smallest_txs_blobs) +{ + using T = cryptonote::NOTIFY_NEW_TRANSACTIONS::request; + epee::byte_slice sample; + if (sample.empty()) { + T out{}; + out.txs.resize(100 * 1000 * 1000 / (1 + 41), std::string(41, '\x00')); + epee::serialization::store_t_to_binary(out, sample); + } + EXPECT_TRUE(sample.size() >= 99e+6 and sample.size() < 105e+6); + T in{}; + EXPECT_TRUE(epee::serialization::load_t_from_binary(in, {sample.data(), sample.size()})); +} + +TEST(portable_scheme_binary_codec_large_packets, too_small_block_blobs) +{ + using T = cryptonote::NOTIFY_RESPONSE_GET_OBJECTS::request; + epee::byte_slice sample; + if (sample.empty()) { + T out{}; + std::string blob(72 - 1, '\x00'); + out.blocks.resize(100 * 1000 * 1000 / (blob.size() + 9), {}); + for (auto &e: out.blocks) { + e.block = blob; + } + epee::serialization::store_t_to_binary(out, sample); + } + EXPECT_TRUE(sample.size() >= 99e+6 and sample.size() < 105e+6); + T in{}; + EXPECT_FALSE(epee::serialization::load_t_from_binary(in, {sample.data(), sample.size()})); + +} + +TEST(portable_scheme_binary_codec_large_packets, smallest_block_blobs) +{ + using T = cryptonote::NOTIFY_RESPONSE_GET_OBJECTS::request; + epee::byte_slice sample; + if (sample.empty()) { + T out{}; + std::string blob(72, '\x00'); + out.blocks.resize(100 * 1000 * 1000 / (blob.size() + 9), {}); + for (auto &e: out.blocks) { + e.block = blob; + } + epee::serialization::store_t_to_binary(out, sample); + } + EXPECT_TRUE(sample.size() >= 99e+6 and sample.size() < 105e+6); + T in{}; + EXPECT_TRUE(epee::serialization::load_t_from_binary(in, {sample.data(), sample.size()})); + +} + +TEST(portable_scheme_binary_codec_large_packets, too_small_compact_txs_blobs) +{ + using T = cryptonote::NOTIFY_RESPONSE_GET_OBJECTS::request; + epee::byte_slice sample; + if (sample.empty()) { + T out{}; + out.blocks.resize(1); + std::string block_blob(72, '\x00'); + std::string tx_blob(41 - 1, '\x00'); + out.blocks[0].block = block_blob; + out.blocks[0].txs.resize(100 * 1000 * 1000 / (1 + tx_blob.size()), tx_blob); + epee::serialization::store_t_to_binary(out, sample); + } + EXPECT_TRUE(sample.size() >= 99e+6 and sample.size() < 105e+6); + T in{}; + EXPECT_FALSE(epee::serialization::load_t_from_binary(in, {sample.data(), sample.size()})); +} + +TEST(portable_scheme_binary_codec_large_packets, smallest_compact_txs_blobs) +{ + using T = cryptonote::NOTIFY_RESPONSE_GET_OBJECTS::request; + epee::byte_slice sample; + if (sample.empty()) { + T out{}; + out.blocks.resize(1); + std::string block_blob(72, '\x00'); + std::string tx_blob(41, '\x00'); + out.blocks[0].block = block_blob; + out.blocks[0].txs.resize(100 * 1000 * 1000 / (1 + tx_blob.size()), tx_blob); + epee::serialization::store_t_to_binary(out, sample); + } + EXPECT_TRUE(sample.size() >= 99e+6 and sample.size() < 105e+6); + T in{}; + EXPECT_TRUE(epee::serialization::load_t_from_binary(in, {sample.data(), sample.size()})); +}