bugfixes + decreased archive slice size

yieldloop · Apr 8, 2020 · 8be3fc9 · 8be3fc9
1 parent 148a5e0
commit 8be3fc9
Show file tree

Hide file tree

Showing 11 changed files with 290 additions and 102 deletions.
diff --git a/crypto/Ed25519.cpp b/crypto/Ed25519.cpp
@@ -22,7 +22,7 @@
 
 #include <openssl/opensslv.h>
 
-#if OPENSSL_VERSION_NUMBER >= 0x10101000L && OPENSSL_VERSION_NUMBER != 0x20000000L
+#if OPENSSL_VERSION_NUMBER >= 0x10101000L && OPENSSL_VERSION_NUMBER != 0x20000000L || defined(OPENSSL_IS_BORINGSSL)
 
 #include "td/utils/base64.h"
 #include "td/utils/BigNum.h"
@@ -57,7 +57,7 @@ SecureString Ed25519::PrivateKey::as_octet_string() const {
   return octet_string_.copy();
 }
 
-#if OPENSSL_VERSION_NUMBER >= 0x10101000L && OPENSSL_VERSION_NUMBER != 0x20000000L
+#if OPENSSL_VERSION_NUMBER >= 0x10101000L && OPENSSL_VERSION_NUMBER != 0x20000000L || defined(OPENSSL_IS_BORINGSSL)
 
 namespace detail {
 

diff --git a/crypto/block/dump-block.cpp b/crypto/block/dump-block.cpp
@@ -31,10 +31,12 @@
 #include "block-db.h"
 #include "block-auto.h"
 #include "block-parse.h"
+#include "mc-config.h"
 #include "vm/cp0.h"
 #include <getopt.h>
 
 using td::Ref;
+using namespace std::literals::string_literals;
 
 int verbosity;
 
@@ -46,6 +48,12 @@ struct IntError {
   }
 };
 
+void throw_err(td::Status err) {
+  if (err.is_error()) {
+    throw IntError{err.to_string()};
+  }
+}
+
 td::Ref<vm::Cell> load_boc(std::string filename) {
   std::cerr << "loading bag-of-cell file " << filename << std::endl;
   auto bytes_res = block::load_binary_file(filename);
@@ -63,6 +71,8 @@ td::Ref<vm::Cell> load_boc(std::string filename) {
   return boc.get_root_cell();
 }
 
+std::vector<Ref<vm::Cell>> loaded_boc;
+
 void test1() {
   block::ShardId id{ton::masterchainId}, id2{ton::basechainId, 0x11efULL << 48};
   std::cout << '[' << id << "][" << id2 << ']' << std::endl;
@@ -190,6 +200,49 @@ void test2(vm::CellSlice& cs) {
             << block::gen::HashmapE{32, block::gen::t_uint16}.validate_upto(1024, cs) << std::endl;
 }
 
+td::Status test_vset() {
+  if (loaded_boc.size() != 2) {
+    return td::Status::Error(
+        "must have exactly two boc files (with a masterchain Block and with ConfigParams) for vset compute test");
+  }
+  std::cerr << "running test_vset()\n";
+  TRY_RESULT(config, block::Config::unpack_config(vm::load_cell_slice_ref(loaded_boc[1])));
+  std::cerr << "config unpacked\n";
+  auto cv_root = config->get_config_param(34);
+  if (cv_root.is_null()) {
+    return td::Status::Error("no config parameter 34");
+  }
+  std::cerr << "config param #34 obtained\n";
+  TRY_RESULT(cur_validators, block::Config::unpack_validator_set(std::move(cv_root)));
+  // auto vconf = config->get_catchain_validators_config();
+  std::cerr << "validator set unpacked\n";
+  std::cerr << "unpacking ShardHashes\n";
+  block::ShardConfig shards;
+  if (!shards.unpack(vm::load_cell_slice_ref(loaded_boc[0]))) {
+    return td::Status::Error("cannot unpack ShardConfig");
+  }
+  std::cerr << "ShardHashes initialized\n";
+  ton::ShardIdFull shard{0, 0x6e80000000000000};
+  ton::CatchainSeqno cc_seqno = std::max(48763, 48763) + 1 + 1;
+  ton::UnixTime now = 1586169666;
+  cc_seqno = shards.get_shard_cc_seqno(shard);
+  std::cerr << "shard=" << shard.to_str() << " cc_seqno=" << cc_seqno << " time=" << now << std::endl;
+  if (cc_seqno == ~0U) {
+    return td::Status::Error("cannot compute cc_seqno for shard "s + shard.to_str());
+  }
+  auto nodes = config->compute_validator_set(shard, *cur_validators, now, cc_seqno);
+  if (nodes.empty()) {
+    return td::Status::Error(PSTRING() << "compute_validator_set() for " << shard.to_str() << "," << now << ","
+                                       << cc_seqno << " returned empty list");
+  }
+  for (auto& x : nodes) {
+    std::cout << "weight=" << x.weight << " key=" << x.key.as_bits256().to_hex() << " addr=" << x.addr.to_hex()
+              << std::endl;
+  }
+  // ...
+  return td::Status::OK();
+}
+
 void usage() {
   std::cout << "usage: dump-block [-t<typename>][-S][<boc-file>]\n\tor dump-block -h\n\tDumps specified blockchain "
                "block or state "
@@ -202,8 +255,11 @@ int main(int argc, char* const argv[]) {
   int new_verbosity_level = VERBOSITY_NAME(INFO);
   const char* tname = nullptr;
   const tlb::TLB* type = &block::gen::t_Block;
+  bool vset_compute_test = false;
+  bool store_loaded = false;
+  int dump = 3;
   auto zerostate = std::make_unique<block::ZerostateInfo>();
-  while ((i = getopt(argc, argv, "CSt:hv:")) != -1) {
+  while ((i = getopt(argc, argv, "CSt:hqv:")) != -1) {
     switch (i) {
       case 'C':
         type = &block::gen::t_VmCont;
@@ -218,6 +274,12 @@ int main(int argc, char* const argv[]) {
       case 'v':
         new_verbosity_level = VERBOSITY_NAME(FATAL) + (verbosity = td::to_integer<int>(td::Slice(optarg)));
         break;
+      case 'q':
+        type = &block::gen::t_ShardHashes;
+        vset_compute_test = true;
+        store_loaded = true;
+        dump = 0;
+        break;
       case 'h':
         usage();
         std::exit(2);
@@ -228,17 +290,22 @@ int main(int argc, char* const argv[]) {
   }
   SET_VERBOSITY_LEVEL(new_verbosity_level);
   try {
-    bool done = false;
+    int loaded = 0;
     while (optind < argc) {
       auto boc = load_boc(argv[optind++]);
       if (boc.is_null()) {
-        std::cerr << "(invalid boc)" << std::endl;
+        std::cerr << "(invalid boc in file" << argv[optind - 1] << ")" << std::endl;
         std::exit(2);
       } else {
-        done = true;
-        vm::CellSlice cs{vm::NoVm(), boc};
-        cs.print_rec(std::cout);
-        std::cout << std::endl;
+        if (store_loaded) {
+          loaded_boc.push_back(boc);
+        }
+        ++loaded;
+        if (dump & 1) {
+          vm::CellSlice cs{vm::NoVm(), boc};
+          cs.print_rec(std::cout);
+          std::cout << std::endl;
+        }
         if (!type) {
           tlb::TypenameLookup dict(block::gen::register_simple_types);
           type = dict.lookup(tname);
@@ -247,20 +314,31 @@ int main(int argc, char* const argv[]) {
             std::exit(3);
           }
         }
-        type->print_ref(std::cout, boc);
-        std::cout << std::endl;
+        if (dump & 2) {
+          type->print_ref(std::cout, boc);
+          std::cout << std::endl;
+        }
         bool ok = type->validate_ref(1048576, boc);
         std::cout << "(" << (ok ? "" : "in") << "valid " << *type << ")" << std::endl;
+        if (vset_compute_test) {
+          if (!ok || loaded > 2) {
+            std::cerr << "fatal: validity check failed\n";
+            exit(3);
+          }
+          type = &block::gen::t_ConfigParams;
+        }
       }
     }
-    if (!done) {
+    if (vset_compute_test) {
+      throw_err(test_vset());
+    } else if (!loaded) {
       test1();
     }
   } catch (IntError& err) {
-    std::cerr << "caught internal error " << err.err_msg << std::endl;
+    std::cerr << "internal error: " << err.err_msg << std::endl;
     return 1;
   } catch (vm::VmError& err) {
-    std::cerr << "caught vm error " << err.get_msg() << std::endl;
+    std::cerr << "vm error: " << err.get_msg() << std::endl;
     return 1;
   }
   return 0;

diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp
@@ -980,8 +980,11 @@ bool TestNode::do_parse_line() {
            (parse_block_id_ext(domain, blkid) ? get_word_to(domain) : (blkid = mc_last_id_).is_valid()) &&
            (seekeoln() || parse_int16(cat)) && seekeoln() &&
            dns_resolve_start(workchain, addr, blkid, domain, cat, step);
-  } else if (word == "allshards") {
-    return eoln() ? get_all_shards() : (parse_block_id_ext(blkid) && seekeoln() && get_all_shards(false, blkid));
+  } else if (word == "allshards" || word == "allshardssave") {
+    std::string filename;
+    return (word.size() <= 9 || get_word_to(filename)) &&
+           (seekeoln() ? get_all_shards(filename)
+                       : (parse_block_id_ext(blkid) && seekeoln() && get_all_shards(filename, false, blkid)));
   } else if (word == "saveconfig") {
     blkid = mc_last_id_;
     std::string filename;
@@ -2105,7 +2108,7 @@ void TestNode::got_block_transactions(ton::BlockIdExt blkid, int mode, unsigned
   out << (incomplete ? "(block transaction list incomplete)" : "(end of block transaction list)") << std::endl;
 }
 
-bool TestNode::get_all_shards(bool use_last, ton::BlockIdExt blkid) {
+bool TestNode::get_all_shards(std::string filename, bool use_last, ton::BlockIdExt blkid) {
   if (use_last) {
     blkid = mc_last_id_;
   }
@@ -2122,7 +2125,7 @@ bool TestNode::get_all_shards(bool use_last, ton::BlockIdExt blkid) {
   auto b = ton::serialize_tl_object(
       ton::create_tl_object<ton::lite_api::liteServer_getAllShardsInfo>(ton::create_tl_lite_block_id(blkid)), true);
   LOG(INFO) << "requesting recent shard configuration";
-  return envelope_send_query(std::move(b), [Self = actor_id(this)](td::Result<td::BufferSlice> R)->void {
+  return envelope_send_query(std::move(b), [ Self = actor_id(this), filename ](td::Result<td::BufferSlice> R)->void {
     if (R.is_error()) {
       return;
     }
@@ -2132,12 +2135,12 @@ bool TestNode::get_all_shards(bool use_last, ton::BlockIdExt blkid) {
     } else {
       auto f = F.move_as_ok();
       td::actor::send_closure_later(Self, &TestNode::got_all_shards, ton::create_block_id(f->id_), std::move(f->proof_),
-                                    std::move(f->data_));
+                                    std::move(f->data_), filename);
     }
   });
 }
 
-void TestNode::got_all_shards(ton::BlockIdExt blk, td::BufferSlice proof, td::BufferSlice data) {
+void TestNode::got_all_shards(ton::BlockIdExt blk, td::BufferSlice proof, td::BufferSlice data, std::string filename) {
   LOG(INFO) << "got shard configuration with respect to block " << blk.to_str();
   if (data.empty()) {
     td::TerminalIO::out() << "shard configuration is empty" << '\n';
@@ -2171,6 +2174,15 @@ void TestNode::got_all_shards(ton::BlockIdExt blk, td::BufferSlice proof, td::Bu
         }
       }
     }
+    if (!filename.empty()) {
+      auto res1 = td::write_file(filename, data.as_slice());
+      if (res1.is_error()) {
+        LOG(ERROR) << "cannot write shard configuration to file `" << filename << "` : " << res1.move_as_error();
+      } else {
+        out << "saved shard configuration (ShardHashes) to file `" << filename << "` (" << data.size() << " bytes)"
+            << std::endl;
+      }
+    }
   }
   show_new_blkids();
 }

diff --git a/lite-client/lite-client.h b/lite-client/lite-client.h
@@ -142,8 +142,8 @@ class TestNode : public td::actor::Actor {
                           std::string domain, std::string qdomain, int cat, int mode, int used_bits,
                           Ref<vm::Cell> value);
   bool show_dns_record(std::ostream& os, int cat, Ref<vm::Cell> value, bool raw_dump);
-  bool get_all_shards(bool use_last = true, ton::BlockIdExt blkid = {});
-  void got_all_shards(ton::BlockIdExt blk, td::BufferSlice proof, td::BufferSlice data);
+  bool get_all_shards(std::string filename = "", bool use_last = true, ton::BlockIdExt blkid = {});
+  void got_all_shards(ton::BlockIdExt blk, td::BufferSlice proof, td::BufferSlice data, std::string filename);
   bool get_config_params(ton::BlockIdExt blkid, td::Promise<td::Unit> do_after, int mode = 0, std::string filename = "",
                          std::vector<int> params = {});
   void got_config_params(ton::BlockIdExt req_blkid, ton::BlockIdExt blkid, td::BufferSlice state_proof,

diff --git a/tdutils/td/utils/BigNum.cpp b/tdutils/td/utils/BigNum.cpp
@@ -98,7 +98,9 @@ BigNum BigNum::from_binary(Slice str) {
 }
 
 BigNum BigNum::from_le_binary(Slice str) {
-#if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER) && !defined(OPENSSL_IS_BORINGSSL)
+#if defined(OPENSSL_IS_BORINGSSL)
+  return BigNum(make_unique<Impl>(BN_le2bn(str.ubegin(), narrow_cast<int>(str.size()), nullptr)));
+#elif OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER)
   return BigNum(make_unique<Impl>(BN_lebin2bn(str.ubegin(), narrow_cast<int>(str.size()), nullptr)));
 #else
   LOG(FATAL) << "Unsupported from_le_binary";
@@ -221,15 +223,19 @@ string BigNum::to_binary(int exact_size) const {
 }
 
 string BigNum::to_le_binary(int exact_size) const {
-#if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER) && !defined(OPENSSL_IS_BORINGSSL)
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER) || defined(OPENSSL_IS_BORINGSSL)
   int num_size = get_num_bytes();
   if (exact_size == -1) {
     exact_size = num_size;
   } else {
     CHECK(exact_size >= num_size);
   }
   string res(exact_size, '\0');
+#if defined(OPENSSL_IS_BORINGSSL)
+  BN_bn2le_padded(MutableSlice(res).ubegin(), exact_size, impl_->big_num);
+#else
   BN_bn2lebinpad(impl_->big_num, MutableSlice(res).ubegin(), exact_size);
+#endif
   return res;
 #else
   LOG(FATAL) << "Unsupported to_le_binary";

diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp
@@ -540,7 +540,7 @@ void ArchiveManager::load_package(PackageId id) {
     }
   }
 
-  desc.file = td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, prefix);
+  desc.file = td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, db_root_);
 
   get_file_map(id).emplace(id, std::move(desc));
 }
@@ -574,7 +574,7 @@ ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull shard
   FileDescription desc{id, false};
   td::mkdir(db_root_ + id.path()).ensure();
   std::string prefix = PSTRING() << db_root_ << id.path() << id.name();
-  desc.file = td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, prefix);
+  desc.file = td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, db_root_);
   if (!id.temp) {
     update_desc(desc, shard, seqno, ts, lt);
   }

diff --git a/validator/db/archive-manager.hpp b/validator/db/archive-manager.hpp
@@ -24,32 +24,6 @@ namespace ton {
 
 namespace validator {
 
-struct PackageId {
-  td::uint32 id;
-  bool key;
-  bool temp;
-
-  explicit PackageId(td::uint32 id, bool key, bool temp) : id(id), key(key), temp(temp) {
-  }
-
-  bool operator<(const PackageId &with) const {
-    return id < with.id;
-  }
-  bool operator==(const PackageId &with) const {
-    return id == with.id;
-  }
-
-  std::string path() const;
-  std::string name() const;
-
-  bool is_empty() {
-    return id == std::numeric_limits<td::uint32>::max();
-  }
-  static PackageId empty(bool key, bool temp) {
-    return PackageId(std::numeric_limits<td::uint32>::max(), key, temp);
-  }
-};
-
 class RootDb;
 
 class ArchiveManager : public td::actor::Actor {