Skip to content

Commit a448249

Browse files
developer-at-bcndeveloper-at-bcn
committed
Add the batch of release v3.4.4 commits
1 parent 5a8a0a3 commit a448249

21 files changed

+372
-121
lines changed

Diff for: ‎README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# Bytecoin
22

3-
[![Build Status](https://dev.azure.com/bcndev/bytecoin/_apis/build/status/bytecoin-daemons?branchName=releases/3.4.3)](https://dev.azure.com/bcndev/bytecoin/_build/latest?definitionId=1&branchName=releases/3.4.3)
3+
[![Build Status](https://dev.azure.com/bcndev/bytecoin/_apis/build/status/bytecoin-daemons?branchName=releases/3.4.4)](https://dev.azure.com/bcndev/bytecoin/_build/latest?definitionId=1&branchName=releases/3.4.4)
44

55
## About
66

Diff for: ‎ReleaseNotes.md

+5
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
## Release Notes
22

3+
### v3.4.4 (Amethyst)
4+
5+
- `walletd` can now sync most of the blockchain from static files (tremendously increasing efficiency for public nodes), reverting to RPC only for the (small) part of blockchain after last hard checkpoint.
6+
- Fixed bug when during wallet sync some transactions from memory pool were requested and processed more than once
7+
38
### v3.4.3 (Amethyst)
49

510
- In `get_transfers` `walletd`'s method error is returned if `from_height` is larger than `to_height` or top block height.

Diff for: ‎azure-pipelines.yml

+4-2
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,8 @@ jobs:
5454
displayName: Clone daemons code
5555
5656
- script: |
57-
wget -c 'https://dl.bintray.com/boostorg/release/1.69.0/source/boost_1_69_0.tar.gz'
57+
wget -c 'https://sourceforge.net/projects/boost/files/boost/1.69.0/boost_1_69_0.tar.gz'
58+
# wget -c 'https://dl.bintray.com/boostorg/release/1.69.0/source/boost_1_69_0.tar.gz'
5859
mkdir boost && tar -xzf ./boost_1_69_0.tar.gz --directory boost --strip-components=1
5960
displayName: Fetch boost
6061
@@ -200,7 +201,8 @@ jobs:
200201
displayName: Clone daemons code
201202
202203
- bash: |
203-
curl -L -O https://dl.bintray.com/boostorg/release/1.69.0/source/boost_1_69_0.zip
204+
curl -L -O https://sourceforge.net/projects/boost/files/boost/1.69.0/boost_1_69_0.zip
205+
# curl -L -O https://dl.bintray.com/boostorg/release/1.69.0/source/boost_1_69_0.zip
204206
7z x boost_1_69_0.zip
205207
mv boost_1_69_0 boost
206208
displayName: Fetch boost

Diff for: ‎src/Core/Config.cpp

+2-1
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ Config::Config(common::CommandLine &cmd)
6161
, multicast_port(P2P_DEFAULT_PORT)
6262
, multicast_period(net == "main" ? 0 : 60.0f) // No multicast in main net due to anonymity
6363
, secrets_via_api(cmd.get_bool("--secrets-via-api"))
64+
, bytecoind_cors_asterisk(cmd.get_bool("--CORS")) // Experimental, for public nodes
6465
, bytecoind_bind_port(RPC_DEFAULT_PORT)
6566
, bytecoind_bind_ip("127.0.0.1") // Less attack vectors from outside for ordinary uses
6667
, bytecoind_remote_ip("127.0.0.1")
@@ -149,7 +150,7 @@ Config::Config(common::CommandLine &cmd)
149150
std::vector<NetworkAddress> exclusive_nodes_list;
150151
parse_peer_and_add_to_container(cmd, exclusive_nodes_list, "--exclusive-node-address");
151152
parse_peer_and_add_to_container(
152-
cmd, exclusive_nodes_list, "--exclusive-node-address", "Use --exclusive-node-address instead");
153+
cmd, exclusive_nodes_list, "--add-exclusive-node", "Use --exclusive-node-address instead");
153154
if (!priority_nodes.empty() && !exclusive_nodes_list.empty())
154155
throw ConfigError("Priority nodes and exclusive nodes cannot be used together");
155156
if (!exclusive_nodes_list.empty()) {

Diff for: ‎src/Core/Config.hpp

+1
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ class Config { // Consensus does not depend on those parameters
5151

5252
std::string bytecoind_authorization;
5353
std::string bytecoind_authorization_private;
54+
bool bytecoind_cors_asterisk = false;
5455
bool good_bytecoind_auth(const std::string &auth) const;
5556
bool good_bytecoind_auth_private(const std::string &auth) const;
5657
uint16_t bytecoind_bind_port;

Diff for: ‎src/Core/Node.cpp

+111-2
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
#include "common/Base58.hpp"
1313
#include "common/JsonValue.hpp"
1414
#include "common/StringTools.hpp"
15-
#include "http/Client.hpp"
1615
#include "http/Server.hpp"
1716
#include "platform/PathTools.hpp"
1817
#include "platform/PreventSleep.hpp"
@@ -165,6 +164,7 @@ void Node::advance_long_poll() {
165164
last_http_response.r.http_version_major = lit->original_request.r.http_version_major;
166165
last_http_response.r.http_version_minor = lit->original_request.r.http_version_minor;
167166
last_http_response.r.keep_alive = lit->original_request.r.keep_alive;
167+
fill_cors(lit->original_request, last_http_response);
168168
if (method_status) {
169169
last_http_response.set_body(json_rpc::create_response_body(resp, jid));
170170
} else {
@@ -181,7 +181,7 @@ void Node::advance_long_poll() {
181181
last_http_response.set_body(json_rpc::create_error_response_body(json_err, jid));
182182
}
183183
}
184-
lit->original_who->write(std::move(last_http_response));
184+
http::Server::write(lit->original_who, std::move(last_http_response));
185185
lit = m_long_poll_http_clients.erase(lit);
186186
}
187187
}
@@ -196,7 +196,26 @@ static const std::string beautiful_index_start =
196196
static const std::string beautiful_index_finish = " </td></tr></table></body></html>";
197197
static const std::string robots_txt = "User-agent: *\r\nDisallow: /";
198198

199+
void Node::fill_cors(const http::RequestBody & req, http::ResponseBody & res) {
200+
if (req.r.origin.empty() || !m_config.bytecoind_cors_asterisk)
201+
return;
202+
res.r.headers.push_back({"Access-Control-Allow-Origin", "*"});
203+
res.r.headers.push_back({"Access-Control-Allow-Methods", "GET, POST, OPTIONS"});
204+
res.r.headers.push_back({"Access-Control-Allow-Headers",
205+
"DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range"});
206+
if (req.r.method == "OPTIONS")
207+
return;
208+
res.r.headers.push_back({"Access-Control-Expose-Headers", "Content-Length,Content-Range"});
209+
}
210+
199211
bool Node::on_api_http_request(http::Client *who, http::RequestBody &&request, http::ResponseBody &response) {
212+
fill_cors(request, response);
213+
if (request.r.method == "OPTIONS") {
214+
response.r.headers.push_back({"Content-Type", "text/plain"});
215+
response.r.headers.push_back({"Access-Control-Max-Age", "172800"}); // 2 days
216+
response.r.status = 200;
217+
return true;
218+
}
200219
if (request.r.uri == "/robots.txt") {
201220
response.r.add_headers_nocache();
202221
response.r.headers.push_back({"Content-Type", "text/plain; charset=UTF-8"});
@@ -234,6 +253,40 @@ bool Node::on_api_http_request(http::Client *who, http::RequestBody &&request, h
234253
response.r.status = 200;
235254
return true;
236255
}
256+
// We emulate nginx here, will remove later
257+
if (m_config.bytecoind_cors_asterisk && common::starts_with(request.r.uri, api::cnd::SyncBlocks::url_prefix())) {
258+
Height height = 0;
259+
if (api::cnd::SyncBlocks::parse_filename(
260+
request.r.uri.substr(api::cnd::SyncBlocks::url_prefix().size()), &height)) {
261+
if (height > m_block_chain.get_currency().last_hard_checkpoint().height) {
262+
response.r.headers.push_back({"Content-Type", "text/html; charset=UTF-8"});
263+
response.r.status = 404;
264+
response.set_body(
265+
"<html><body>404 Not Found - static blocks can be queried only up to last hard checkpoint</body></html>");
266+
return true;
267+
}
268+
response.r.headers.push_back({"Content-Type", "application/octet-stream"});
269+
response.r.headers.push_back({"Cache-Control", "max-age=2628000, public"}); // month
270+
response.r.status = 200;
271+
if (height % 10 != 0) {
272+
// Emulate static link
273+
response.set_body(common::to_string(height - height % 10));
274+
return true;
275+
}
276+
api::cnd::SyncBlocks::ResponseCompact res;
277+
size_t total_size = 0;
278+
Hash hash;
279+
while (m_block_chain.get_chain(height + static_cast<Height>(res.blocks.size()), &hash)) {
280+
res.blocks.push_back(m_block_chain.fill_sync_block_compact(hash));
281+
total_size += res.blocks.back().header.block_size;
282+
if (total_size >= 1024 * 1024 || height + static_cast<Height>(res.blocks.size()) >
283+
m_block_chain.get_currency().last_hard_checkpoint().height)
284+
break;
285+
}
286+
response.set_body(json_rpc::create_binary_response_body(res, common::JsonValue(nullptr)));
287+
return true;
288+
}
289+
}
237290
response.r.status = 404;
238291
response.r.headers.push_back({"Content-Type", "text/html; charset=UTF-8"});
239292
response.set_body("<html><body>404 Not Found</body></html>");
@@ -952,3 +1005,59 @@ bool Node::on_submitblock(http::Client *, http::RequestBody &&http_request, json
9521005
res.depth = api::HeightOrDepth(res.block_header.height) - api::HeightOrDepth(m_block_chain.get_tip_height()) - 1;
9531006
return true;
9541007
}
1008+
1009+
void Node::export_static_sync_blocks(const BlockChainState &block_chain, const std::string &folder) {
1010+
if (block_chain.get_tip_height() < block_chain.get_currency().last_hard_checkpoint().height)
1011+
throw std::runtime_error("Daemon must be synced at least to last hard checkpoint");
1012+
if (!platform::folder_exists(folder))
1013+
throw std::runtime_error("Folder for static sync_blocks must exist " + folder);
1014+
api::cnd::SyncBlocks::Request req;
1015+
req.max_size = 1024 * 1024;
1016+
req.max_count = 1000;
1017+
req.need_redundant_data = false;
1018+
std::vector<Hash> all_chain(block_chain.get_currency().last_hard_checkpoint().height + 1);
1019+
std::cout << "Reading chain, can take a minute or two..." << std::endl;
1020+
for (Height i = 0; i != block_chain.get_currency().last_hard_checkpoint().height + 1; ++i)
1021+
invariant(block_chain.get_chain(i, &all_chain.at(i)), "");
1022+
Height ha = 0;
1023+
api::cnd::SyncBlocks::ResponseCompact res;
1024+
size_t max_total_count = 0;
1025+
size_t min_total_count = std::numeric_limits<size_t>::max();
1026+
size_t sum_total_size_real = 0;
1027+
size_t sum_total_size_link = 0;
1028+
while (ha <= block_chain.get_currency().last_hard_checkpoint().height) {
1029+
res.blocks.clear();
1030+
size_t total_size = 0;
1031+
while (ha + res.blocks.size() <= block_chain.get_currency().last_hard_checkpoint().height) {
1032+
res.blocks.push_back(block_chain.fill_sync_block_compact(all_chain.at(ha + res.blocks.size())));
1033+
total_size += seria::binary_size(res.blocks.back());
1034+
if (total_size >= req.max_size)
1035+
break;
1036+
}
1037+
invariant(!res.blocks.empty(), "");
1038+
auto ba = json_rpc::create_binary_response_body(res, common::JsonValue(nullptr));
1039+
std::cout << "Saving chunk start=" << ha << ", count=" << res.blocks.size() << ", size=" << ba.size()
1040+
<< std::endl;
1041+
sum_total_size_real += ba.size();
1042+
for (size_t d = 0; d != res.blocks.size(); ++d) {
1043+
if (d == 1) {
1044+
ba = common::to_string(ha);
1045+
sum_total_size_link += ba.size() * (res.blocks.size() - 1);
1046+
}
1047+
std::string subfolder;
1048+
auto file_name = api::cnd::SyncBlocks::get_filename(Height(ha + d), &subfolder);
1049+
invariant(
1050+
platform::create_folders_if_necessary(folder + api::cnd::SyncBlocks::url_prefix() + "/" + subfolder),
1051+
"");
1052+
invariant(platform::save_file(
1053+
folder + api::cnd::SyncBlocks::url_prefix() + "/" + file_name, ba.data(), ba.size()),
1054+
"");
1055+
}
1056+
ha += res.blocks.size();
1057+
max_total_count = std::max(max_total_count, res.blocks.size());
1058+
min_total_count = std::min(min_total_count, res.blocks.size());
1059+
}
1060+
std::cout << "min_total_count=" << min_total_count << " max_total_count=" << max_total_count << std::endl;
1061+
std::cout << "sum_total_size_real=" << sum_total_size_real << " sum_total_size_link=" << sum_total_size_link
1062+
<< std::endl;
1063+
}

Diff for: ‎src/Core/Node.hpp

+3
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,8 @@ class Node {
8585
BlockChainState &m_block_chain;
8686
const Config &m_config;
8787

88+
static void export_static_sync_blocks(const BlockChainState &block_chain, const std::string &folder);
89+
8890
protected:
8991
std::unique_ptr<http::Server> m_api;
9092
std::unique_ptr<platform::PreventSleep> m_prevent_sleep;
@@ -199,6 +201,7 @@ class Node {
199201
void broadcast(P2PProtocolBytecoin *exclude, const BinaryArray &data);
200202
void broadcast(P2PProtocolBytecoin *exclude, const BinaryArray &data_v1, const BinaryArray &data_v4);
201203

204+
void fill_cors(const http::RequestBody & req, http::ResponseBody & res);
202205
bool on_api_http_request(http::Client *, http::RequestBody &&, http::ResponseBody &);
203206
void on_api_http_disconnect(http::Client *);
204207

Diff for: ‎src/Core/WalletNode.cpp

+27-16
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,17 @@
77
#include "TransactionBuilder.hpp"
88
#include "TransactionExtra.hpp"
99
#include "common/Math.hpp"
10-
#include "http/Client.hpp"
1110
#include "http/Server.hpp"
1211
#include "platform/Time.hpp"
1312
#include "seria/BinaryInputStream.hpp"
1413
#include "seria/BinaryOutputStream.hpp"
1514
#include "seria/KVBinaryInputStream.hpp"
1615
#include "seria/KVBinaryOutputStream.hpp"
1716

17+
#ifndef __EMSCRIPTEN__
18+
#include "Node.hpp"
19+
#endif
20+
1821
using namespace cn;
1922

2023
const WalletNode::HandlersMap WalletNode::m_jsonrpc_handlers = {
@@ -52,8 +55,10 @@ bool WalletNode::on_api_http_request(http::Client *who, http::RequestBody &&requ
5255
if (method_found)
5356
return result;
5457
}
58+
#ifndef __EMSCRIPTEN__
5559
if (m_inproc_node)
5660
return m_inproc_node->on_json_rpc(who, std::move(request), response);
61+
#endif
5762
m_log(logging::INFO) << "http_request node tunneling url=" << request.r.uri
5863
<< " start of body=" << request.body.substr(0, 200);
5964
http::RequestBody original_request;
@@ -68,15 +73,15 @@ bool WalletNode::on_api_http_request(http::Client *who, http::RequestBody &&requ
6873
send_response.r.http_version_minor = wc.original_request.r.http_version_minor;
6974
send_response.r.keep_alive = wc.original_request.r.keep_alive;
7075
// bytecoind never sends connection-close, so we are safe to retain all headers
71-
wc.original_who->write(std::move(send_response));
76+
http::Server::write(wc.original_who, std::move(send_response));
7277
},
7378
[](const WaitingClient &wc, std::string err) {
7479
http::ResponseBody send_response;
7580
send_response.r.http_version_major = wc.original_request.r.http_version_major;
7681
send_response.r.http_version_minor = wc.original_request.r.http_version_minor;
7782
send_response.r.keep_alive = wc.original_request.r.keep_alive;
7883
send_response.r.status = 504; // TODO -test this code path
79-
wc.original_who->write(std::move(send_response));
84+
http::Server::write(wc.original_who, std::move(send_response));
8085
});
8186
return false;
8287
}
@@ -556,6 +561,7 @@ bool WalletNode::on_create_transaction(http::Client *who, http::RequestBody &&ra
556561
// We ask excess output for the case of collision with our output
557562
// We ask minimum anonymity, though less than requested might be returned
558563
ra_request.amounts = selector.get_ra_amounts();
564+
#ifndef __EMSCRIPTEN__
559565
if (m_inproc_node) {
560566
api::cnd::GetRandomOutputs::Response ra_response;
561567
m_inproc_node->on_get_random_outputs(
@@ -585,7 +591,7 @@ bool WalletNode::on_create_transaction(http::Client *who, http::RequestBody &&ra
585591
response.transaction = ptx;
586592
return true;
587593
}
588-
594+
#endif
589595
http::RequestBody new_request =
590596
json_rpc::create_request(api::cnd::url(), api::cnd::GetRandomOutputs::method(), ra_request);
591597
new_request.r.basic_authorization = m_config.bytecoind_authorization;
@@ -632,7 +638,7 @@ bool WalletNode::on_create_transaction(http::Client *who, http::RequestBody &&ra
632638
last_response.transaction.size = last_response.binary_transaction.size();
633639
last_http_response.set_body(json_rpc::create_response_body(last_response, wc.original_jsonrpc_id));
634640
}
635-
wc.original_who->write(std::move(last_http_response));
641+
http::Server::write(wc.original_who, std::move(last_http_response));
636642
},
637643
[=](const WaitingClient &wc, std::string err) mutable {
638644
m_log(logging::INFO) << "got error to get_random_outputs from " CRYPTONOTE_NAME "d, " << err;
@@ -641,7 +647,7 @@ bool WalletNode::on_create_transaction(http::Client *who, http::RequestBody &&ra
641647
last_http_response.r.status = 200;
642648
last_http_response.set_body(json_rpc::create_error_response_body(
643649
json_rpc::Error(json_rpc::INTERNAL_ERROR, err), wc.original_jsonrpc_id));
644-
wc.original_who->write(std::move(last_http_response));
650+
http::Server::write(wc.original_who, std::move(last_http_response));
645651
});
646652
return false;
647653
}
@@ -692,7 +698,7 @@ bool WalletNode::on_create_sendproof(http::Client *who, http::RequestBody &&raw_
692698
last_http_response.r.headers.push_back({"Content-Type", "application/json; charset=utf-8"});
693699
last_http_response.r.status = 200;
694700
last_http_response.set_body(json_rpc::create_response_body(last_response, wc.original_jsonrpc_id));
695-
wc.original_who->write(std::move(last_http_response));
701+
http::Server::write(wc.original_who, std::move(last_http_response));
696702
},
697703
[=](const WaitingClient &wc, std::string err) mutable {
698704
m_log(logging::INFO) << "got error to get_raw_transaction from " CRYPTONOTE_NAME "d, " << err;
@@ -701,7 +707,7 @@ bool WalletNode::on_create_sendproof(http::Client *who, http::RequestBody &&raw_
701707
last_http_response.r.status = 200;
702708
last_http_response.set_body(json_rpc::create_error_response_body(
703709
json_rpc::Error(json_rpc::INTERNAL_ERROR, err), wc.original_jsonrpc_id));
704-
wc.original_who->write(std::move(last_http_response));
710+
http::Server::write(wc.original_who, std::move(last_http_response));
705711
});
706712
return false;
707713
}
@@ -711,11 +717,13 @@ bool WalletNode::on_send_transaction(http::Client *who, http::RequestBody &&raw_
711717
api::cnd::SendTransaction::Response &response) {
712718
m_wallet_state.add_to_payment_queue(request.binary_transaction, true);
713719
advance_long_poll();
720+
#ifndef __EMSCRIPTEN__
714721
if (m_inproc_node) {
715722
m_inproc_node->on_send_transaction(
716723
nullptr, std::move(raw_request), std::move(raw_js_request), std::move(request), response);
717724
return true;
718725
}
726+
#endif
719727
http::RequestBody new_request;
720728
new_request.set_body(std::move(raw_request.body)); // We save on copying body here
721729
new_request.r.set_firstline("POST", api::cnd::url(), 1, 1);
@@ -726,15 +734,15 @@ bool WalletNode::on_send_transaction(http::Client *who, http::RequestBody &&raw_
726734
resp.r.http_version_major = wc2.original_request.r.http_version_major;
727735
resp.r.http_version_minor = wc2.original_request.r.http_version_minor;
728736
resp.r.keep_alive = wc2.original_request.r.keep_alive;
729-
wc2.original_who->write(std::move(resp));
737+
http::Server::write(wc2.original_who, std::move(resp));
730738
},
731739
[](const WaitingClient &wc2, std::string err) {
732740
http::ResponseBody resp(wc2.original_request.r);
733741
resp.r.headers.push_back({"Content-Type", "application/json; charset=utf-8"});
734742
resp.r.status = 200;
735743
resp.set_body(json_rpc::create_error_response_body(
736744
json_rpc::Error(json_rpc::INTERNAL_ERROR, err), wc2.original_jsonrpc_id));
737-
wc2.original_who->write(std::move(resp));
745+
http::Server::write(wc2.original_who, std::move(resp));
738746
});
739747
return false;
740748
}
@@ -819,15 +827,18 @@ void WalletNode::advance_long_poll() {
819827

820828
for (auto lit = m_long_poll_http_clients.begin(); lit != m_long_poll_http_clients.end();)
821829
if (resp.ready_for_longpoll(lit->original_get_status)) {
830+
LongPollClient cli = std::move(*lit);
831+
lit = m_long_poll_http_clients.erase(lit);
832+
// We erase first, because on some platforms on_api_http_disconnect will be called
833+
// synchronously in Server::write, and will also attempt to erase from m_long_poll_http_clients
822834
http::ResponseBody last_http_response;
823835
last_http_response.r.headers.push_back({"Content-Type", "application/json; charset=utf-8"});
824836
last_http_response.r.status = 200;
825-
last_http_response.r.http_version_major = lit->original_request.r.http_version_major;
826-
last_http_response.r.http_version_minor = lit->original_request.r.http_version_minor;
827-
last_http_response.r.keep_alive = lit->original_request.r.keep_alive;
828-
last_http_response.set_body(json_rpc::create_response_body(resp, lit->original_jsonrpc_id));
829-
lit->original_who->write(std::move(last_http_response));
830-
lit = m_long_poll_http_clients.erase(lit);
837+
last_http_response.r.http_version_major = cli.original_request.r.http_version_major;
838+
last_http_response.r.http_version_minor = cli.original_request.r.http_version_minor;
839+
last_http_response.r.keep_alive = cli.original_request.r.keep_alive;
840+
last_http_response.set_body(json_rpc::create_response_body(resp, cli.original_jsonrpc_id));
841+
http::Server::write(cli.original_who, std::move(last_http_response));
831842
} else
832843
++lit;
833844
}

0 commit comments

Comments
 (0)