diff --git a/src/consensus/chain.cpp b/src/consensus/chain.cpp index 06c83978..35321eb6 100644 --- a/src/consensus/chain.cpp +++ b/src/consensus/chain.cpp @@ -138,8 +138,9 @@ std::vector Chain::GetSortedSubgraph(const ConstBlockPtr& pblock) return result; } -void Chain::CheckTxPartition(Vertex& b, float ms_hashrate) { - if (b.minerChainHeight <= GetParams().sortitionThreshold) { +void Chain::CheckTxPartition(Vertex& b) { + auto msLinkHeight = GetVertex(b.cblock->GetMilestoneHash())->height; + if (msLinkHeight <= GetParams().sortitionThreshold) { if (b.cblock->IsRegistration()) { if (b.cblock->GetTransactionSize() > 1) { memset(&b.validity[1], Vertex::Validity::INVALID, b.validity.size() - 1); @@ -162,17 +163,18 @@ void Chain::CheckTxPartition(Vertex& b, float ms_hashrate) { // Construct a cumulator for the block if it is not cached Cumulator cum; - ConstBlockPtr cursor = b.cblock; + Vertex blk_cursor = b; VertexPtr previous; - while (!cum.Full()) { - previous = GetVertex(cursor->GetPrevHash()); + while (previous->height > (previous->height - GetCumulatorCapacity()) && previous->height > 0) { + previous = GetVertex(blk_cursor.cblock->GetPrevHash()); if (!previous) { // should not happen - throw std::logic_error("Cannot find " + std::to_string(cursor->GetPrevHash()) + " in cumulatorMap."); + throw std::logic_error("Cannot find " + std::to_string(blk_cursor.cblock->GetPrevHash()) + + " in cumulatorMap."); } - cum.Add(previous->cblock, false); - cursor = previous->cblock; + cum.Add(*previous, *this, false); + blk_cursor = *previous; } cumulatorMap_.emplace(b.cblock->GetPrevHash(), cum); @@ -182,7 +184,7 @@ void Chain::CheckTxPartition(Vertex& b, float ms_hashrate) { Cumulator& cum = nodeHandler.mapped(); // Allowed distance - auto allowed = CalculateAllowedDist(cum, ms_hashrate); + auto allowed = CalculateAllowedDist(cum, msLinkHeight); // Distances of the transaction hashes and previous block hash const auto& txns = b.cblock->GetTransactions(); @@ -201,11 +203,20 @@ void Chain::CheckTxPartition(Vertex& b, float ms_hashrate) { } // Update key for the cumulator - cum.Add(b.cblock, true); + cum.Add(b, *this, true); nodeHandler.key() = b.cblock->GetHash(); cumulatorMap_.insert(std::move(nodeHandler)); } +const Cumulator* Chain::GetCumulator(const uint256& h) const { + auto it = cumulatorMap_.find(h); + if (it == cumulatorMap_.end()) { + return nullptr; + } + + return &it->second; +} + VertexPtr Chain::Verify(const ConstBlockPtr& pblock) { auto height = GetChainHead()->height + 1; @@ -216,6 +227,7 @@ VertexPtr Chain::Verify(const ConstBlockPtr& pblock) { std::vector vtcs; std::vector wvtcs; + std::unordered_set cumulators; RegChange regChange; TXOC txoc; vtcs.reserve(blocksToValidate.size()); @@ -230,8 +242,9 @@ VertexPtr Chain::Verify(const ConstBlockPtr& pblock) { // validate each block in order for (auto& vtx : vtcs) { vtx->height = height; + + const auto& blkHash = vtx->cblock->GetHash(); if (vtx->cblock->IsFirstRegistration()) { - const auto& blkHash = vtx->cblock->GetHash(); prevRedempHashMap_.insert_or_assign(blkHash, const_cast(blkHash)); vtx->isRedeemed = Vertex::NOT_YET_REDEEMED; regChange.Create(blkHash, blkHash); @@ -261,6 +274,11 @@ VertexPtr Chain::Verify(const ConstBlockPtr& pblock) { vtx->UpdateReward(GetPrevReward(*vtx)); } verifying_.insert({vtx->cblock->GetHash(), vtx}); + + auto cum = cumulatorMap_.find(blkHash); + if (cum != cumulatorMap_.end()) { + cumulators.insert(&(cum->second)); + } } CreateNextMilestone(GetChainHead(), *vtcs.back(), std::move(wvtcs), std::move(regChange), std::move(txoc)); @@ -322,7 +340,7 @@ std::pair Chain::Validate(Vertex& vertex, RegChange& regChange) { // txns with invalid distance will have validity == INVALID, and others are left unchanged VertexPtr prevMs = DAG->GetMsVertex(vertex.cblock->GetMilestoneHash()); assert(prevMs); - CheckTxPartition(vertex, prevMs->snapshot->hashRate); + CheckTxPartition(vertex); // check utxo // txns with valid utxo will have validity == VALID, and others are left unchanged @@ -520,6 +538,16 @@ VertexPtr Chain::GetMsVertexCache(const uint256& msHash) const { return nullptr; } +MilestonePtr Chain::GetMsVertex(size_t height) const { + size_t leastHeightCached = GetLeastHeightCached(); + + if (height < leastHeightCached) { + return STORE->GetMilestoneAt(height)->snapshot; + } else { + return milestones_[height - leastHeightCached]; + } +} + void Chain::PopOldest(const std::vector& vtxToRemove, const TXOC& txocToRemove) { for (const auto& lvsh : vtxToRemove) { // Modify redemption status for those prev regs in DB @@ -588,77 +616,94 @@ bool Chain::IsTxFitsLedger(const ConstTxPtr& tx) const { // Cumulator //////////////////// -void Cumulator::Add(const ConstBlockPtr& block, bool ascending) { - const auto& chainwork = block->GetChainWork(); - uint32_t chainwork_comp = chainwork.GetCompact(); +size_t GetCumulatorCapacity() { + static size_t const cap = GetParams().punctualityThred + GetParams().sortitionThreshold; + return cap; +} - if (timestamps.size() < GetParams().sortitionThreshold) { - sum += chainwork; - } else { - arith_uint256 subtrahend = arith_uint256().SetCompact(chainworks.front().first); - sum += (chainwork - subtrahend); +void Cumulator::Add(const Vertex& block, const Chain& chain, bool ascending) { + auto msHeight = block.height; + assert(msHeight > 0); - // Pop the first element if the counter is already 1, - // or decrease the counter of the first element by 1 - if (chainworks.front().second == 1) { - chainworks.pop_front(); - } else { - chainworks.front().second--; + // Update queue + if (ascending) { + // Align the segmt_sizes_ + auto back_height = sizes_.back().first; + while (back_height++ < msHeight) { + // This happens when there is level set contains + // no block from this miner chain. + + if (Full()) { + sizes_.pop_front(); + } + + sizes_.emplace_back(back_height, std::make_pair(chain.GetMsVertex(back_height)->lvsSize, 0)); } - timestamps.pop_front(); - } + sizes_.back().second.second++; - if (ascending) { - if (!chainworks.empty() && chainworks.back().first == chainwork_comp) { - chainworks.back().second++; - } else { - chainworks.emplace_back(chainwork_comp, 1); - } - timestamps.emplace_back(block->GetTime()); } else { - if (!chainworks.empty() && chainworks.front().first == chainwork_comp) { - chainworks.front().second++; - } else { - chainworks.emplace_front(chainwork_comp, 1); + auto front_height = sizes_.front().first; + while (front_height-- > msHeight) { + sizes_.emplace_front(front_height, std::make_pair(chain.GetMsVertex(front_height)->lvsSize, 0)); + + // We don't check whether the queue is full here. + // It's the caller's responsibility to make sure that + // the capacity is not exceeded when adding elements + // backwards. } - timestamps.emplace_front(block->GetTime()); + + sizes_.front().second.second++; } } -arith_uint256 Cumulator::Sum() const { - return sum; -} +double Cumulator::Percentage(size_t height) const { + if (sum_cache_.size() > GetParams().punctualityThred) { + sum_cache_.erase(sizes_.back().first - GetParams().punctualityThred); + } + + auto sums_it = sum_cache_.find(height); + if (sums_it == sum_cache_.end()) { + auto cursor = sizes_.rbegin(); + while (cursor->first != height) { + cursor++; + } + + auto sums = cursor->second; + cursor++; + while (cursor->first > (height - GetParams().sortitionThreshold)) { + sums.first += cursor->second.first; + sums.second += cursor->second.second; + + cursor++; + } + + sum_cache_.emplace(height, sums); -uint32_t Cumulator::TimeSpan() const { - return timestamps.back() - timestamps.front(); + return sums.second / sums.first; + } + + return sums_it->second.first / sums_it->second.second; } bool Cumulator::Full() const { - return timestamps.size() == GetParams().sortitionThreshold; + return sizes_.size() == GetCumulatorCapacity(); } bool Cumulator::Empty() const { - return timestamps.empty(); + return sizes_.empty(); } void Cumulator::Clear() { - chainworks.clear(); - timestamps.clear(); - sum = 0; + sizes_.clear(); } std::string std::to_string(const Cumulator& cum) { std::string s; s += " Cumulator { \n"; - s += " chainworks { \n"; - for (auto& e : cum.chainworks) { - s += strprintf(" { %s, %s }\n", arith_uint256().SetCompact(e.first).GetLow64(), e.second); - } - s += " }\n"; - s += " timestamps { \n"; - for (auto& t : cum.timestamps) { - s += strprintf(" %s\n", t); + s += " sizes { \n"; + for (auto& e : cum.sizes_) { + s += strprintf(" { %s, %s, %s}\n", e.first, e.second.first, e.second.second); } s += " }\n"; s += " }"; diff --git a/src/consensus/chain.h b/src/consensus/chain.h index 8a4bd1f0..97b930b4 100644 --- a/src/consensus/chain.h +++ b/src/consensus/chain.h @@ -13,41 +13,48 @@ #include #include +class Chain; + class Cumulator; namespace std { string to_string(const Cumulator& b); } // namespace std +size_t GetCumulatorCapacity(); + class Cumulator { public: - void Add(const ConstBlockPtr& block, bool ascending); - arith_uint256 Sum() const; - uint32_t TimeSpan() const; + void Add(const Vertex& block, const Chain&, bool ascending); + double Percentage(size_t height) const; bool Full() const; bool Empty() const; void Clear(); friend std::string std::to_string(const Cumulator&); + friend struct std::hash; private: - // Elements in chainworks: - // {chainwork, counter of consecutive chainworks that are equal} - // For example, the queue of chainworks - // { 1, 1, 3, 2, 2, 2, 2, 2, 2, 2 } - // are stored as: - // { {1, 2}, {3, 1}, {2, 7} } - std::deque> chainworks; - std::deque timestamps; - arith_uint256 sum = 0; + // < + // level set height, + // < + // size of the level set, + // size of the segment of the miner chain contained in the level set + // > + // > + std::deque>> sizes_; + + // Caches the sums of the level set segments of length sortitionThreshold, + // so that each sum need only be calculated at most once at each height. + mutable std::unordered_map> sum_cache_; }; /** Hasher for unordered_map */ -template <> -struct std::hash { - size_t operator()(const Cumulator& x) const { - return x.Sum().GetCompact() ^ x.TimeSpan(); - } -}; +// template <> +// struct std::hash { +// size_t operator()(const Cumulator& x) const { +// return (size_t) x.lvs_sum_ ^ (size_t) x.miner_sum_; +//} +//}; class Chain { public: @@ -78,6 +85,7 @@ class Chain { VertexPtr GetVertexCache(const uint256&) const; VertexPtr GetVertex(const uint256&) const; VertexPtr GetMsVertexCache(const uint256&) const; + MilestonePtr GetMsVertex(size_t height) const; /** Gets a list of block to verify by the post-order DFS */ std::vector GetSortedSubgraph(const ConstBlockPtr& pblock); @@ -110,6 +118,8 @@ class Chain { milestones_.emplace_back(ms.snapshot); } + const Cumulator* GetCumulator(const uint256&) const; + /** * Off-line verification (building ledger) on a level set * performed when we add a milestone block to this chain. @@ -188,7 +198,7 @@ class Chain { std::optional ValidateRedemption(Vertex&, RegChange&); bool ValidateTx(const Transaction&, uint32_t index, TXOC&, Coin& fee); TXOC ValidateTxns(Vertex&); - void CheckTxPartition(Vertex&, float); + void CheckTxPartition(Vertex&); Coin GetPrevReward(const Vertex& vtx) const { return GetVertex(vtx.cblock->GetPrevHash())->cumulativeReward; @@ -202,9 +212,8 @@ class Chain { typedef std::unique_ptr ChainPtr; -inline double CalculateAllowedDist(const Cumulator& cum, float msHashRate) { - return cum.Sum().GetDouble() / std::max(cum.TimeSpan(), (uint32_t) 1) / msHashRate * - (GetParams().sortitionCoefficient * GetParams().maxTarget.GetDouble()); +inline double CalculateAllowedDist(const Cumulator& cum, size_t msHeight) { + return cum.Percentage(msHeight) * (GetParams().sortitionCoefficient * GetParams().maxTarget.GetDouble()); } #endif // EPIC_CHAIN_H diff --git a/src/consensus/dag_manager.cpp b/src/consensus/dag_manager.cpp index e99965c9..ba7aea5a 100644 --- a/src/consensus/dag_manager.cpp +++ b/src/consensus/dag_manager.cpp @@ -774,3 +774,7 @@ void DAGManager::UpdateStatOnLvsStored(const MilestonePtr& pms) { stat_.tStart = pms->GetLevelSet().front().lock()->cblock->GetTime(); } } + +const Cumulator* DAGManager::GetCumulator(const uint256& h) const { + return GetBestChain().GetCumulator(h); +} diff --git a/src/consensus/dag_manager.h b/src/consensus/dag_manager.h index d0cf3d7e..2a777178 100644 --- a/src/consensus/dag_manager.h +++ b/src/consensus/dag_manager.h @@ -114,6 +114,8 @@ class DAGManager { StatData GetStatData() const; + const Cumulator* GetCumulator(const uint256&) const; + /** * Blocks the main thread from going forward * until DAG completes all the tasks diff --git a/src/consensus/milestone.cpp b/src/consensus/milestone.cpp index be7f41d4..b83053d5 100644 --- a/src/consensus/milestone.cpp +++ b/src/consensus/milestone.cpp @@ -16,9 +16,9 @@ Milestone::Milestone(const std::shared_ptr& previous, RegChange&& regChange, TXOC&& txoc) : height(previous->height + 1), milestoneTarget(previous->milestoneTarget), blockTarget(previous->blockTarget), - hashRate(previous->hashRate), lastUpdateTime(previous->lastUpdateTime), nTxnsCounter_(previous->nTxnsCounter_), - nBlkCounter_(previous->nBlkCounter_), lvs_(std::move(lvs)), txoc_(std::move(txoc)), - regChange_(std::move(regChange)) { + hashRate(previous->hashRate), lvsSize(lvs.size()), lastUpdateTime(previous->lastUpdateTime), + nTxnsCounter_(previous->nTxnsCounter_), nBlkCounter_(previous->nBlkCounter_), lvs_(std::move(lvs)), + txoc_(std::move(txoc)), regChange_(std::move(regChange)) { if (previous->chainwork == 0) { previous->chainwork = UintToArith256(STORE->GetBestChainWork()); } diff --git a/src/consensus/milestone.h b/src/consensus/milestone.h index 9522def2..cfb4f31a 100644 --- a/src/consensus/milestone.h +++ b/src/consensus/milestone.h @@ -21,6 +21,7 @@ class Milestone { arith_uint256 milestoneTarget; arith_uint256 blockTarget; float hashRate; + uint32_t lvsSize; uint32_t lastUpdateTime = 0; bool stored = false; @@ -94,7 +95,7 @@ class Milestone { ADD_SERIALIZE_METHODS template inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(hashRate); + READWRITE(lvsSize); if (ser_action.ForRead()) { milestoneTarget.SetCompact(ser_readdata32(s)); blockTarget.SetCompact(ser_readdata32(s)); @@ -110,6 +111,7 @@ class Milestone { bool operator==(const Milestone& rhs) const { // clang-format off return hashRate == rhs.hashRate && + lvsSize == rhs.lvsSize && milestoneTarget.GetCompact() == rhs.milestoneTarget.GetCompact() && blockTarget.GetCompact() == rhs.blockTarget.GetCompact() && (chainwork == 0 || rhs.chainwork == 0) ? true : chainwork == rhs.chainwork && diff --git a/src/pow/miner.cpp b/src/pow/miner.cpp index 4afc4824..d1e09bbb 100644 --- a/src/pow/miner.cpp +++ b/src/pow/miner.cpp @@ -64,15 +64,6 @@ void Miner::Run() { selfChainHeads_.pop_front(); } - // Restore distanceCal_ - if (selfChainHead_ && distanceCal_.Empty()) { - auto cursor = selfChainHead_; - do { - distanceCal_.Add(cursor, false); - cursor = STORE->FindBlock(cursor->GetPrevHash()); - } while (*cursor != *GENESIS && !distanceCal_.Full()); - } - chainHead_ = DAG->GetMilestoneHead(); inspector_ = std::thread([&]() { @@ -111,6 +102,16 @@ void Miner::Run() { prevHash = GENESIS->GetHash(); b.AddTransaction(std::move(firstRegTx)); } else { + // Restore distanceCal_ + if (!distanceCal_) { + ConstBlockPtr cursor = selfChainHead_; + + while (!distanceCal_ && *cursor != *GENESIS) { + distanceCal_ = DAG->GetCumulator(cursor->GetHash()); + cursor = STORE->FindBlock(cursor->GetPrevHash()); + } + } + prevHash = selfChainHead_->GetHash(); auto max_ntx = GetParams().blockCapacity; @@ -122,22 +123,20 @@ void Miner::Run() { selfChainHead_ = nullptr; prevHash = GENESIS->GetHash(); selfChainHeads_.clear(); - distanceCal_.Clear(); + distanceCal_ = nullptr; } else { b.AddTransaction(std::move(tx)); max_ntx--; } } - if (distanceCal_.Full()) { + if (distanceCal_) { if (counter % 10 == 0) { spdlog::debug("[Miner] Hashing power percentage {}", - distanceCal_.Sum().GetDouble() / std::max(distanceCal_.TimeSpan(), (uint32_t) 1) / - std::atomic_load(&chainHead_)->snapshot->hashRate); + distanceCal_->Percentage(std::atomic_load(&chainHead_)->height)); } - auto allowed = - CalculateAllowedDist(distanceCal_, std::atomic_load(&chainHead_)->snapshot->hashRate); + auto allowed = CalculateAllowedDist(*distanceCal_, std::atomic_load(&chainHead_)->height); b.AddTransactions(MEMPOOL->ExtractTransactions(prevHash, allowed, max_ntx)); } } @@ -190,7 +189,6 @@ void Miner::Run() { PEERMAN->RelayBlock(bPtr, nullptr); } - distanceCal_.Add(bPtr, true); selfChainHead_ = bPtr; selfChainHeads_.push(bPtr->GetHash()); DAG->AddNewBlock(bPtr, nullptr); diff --git a/src/pow/miner.h b/src/pow/miner.h index b935abe7..fccfe869 100644 --- a/src/pow/miner.h +++ b/src/pow/miner.h @@ -41,9 +41,9 @@ class Miner { private: Solver* solver; - ConstBlockPtr selfChainHead_ = nullptr; - VertexPtr chainHead_ = nullptr; - Cumulator distanceCal_; + ConstBlockPtr selfChainHead_ = nullptr; + VertexPtr chainHead_ = nullptr; + const Cumulator* distanceCal_ = nullptr; CircularQueue selfChainHeads_; uint256 SelectTip(); diff --git a/test/consensus/test_chain_verification.cpp b/test/consensus/test_chain_verification.cpp index 6c8d2997..09f2d8c4 100644 --- a/test/consensus/test_chain_verification.cpp +++ b/test/consensus/test_chain_verification.cpp @@ -58,8 +58,8 @@ class TestChainVerification : public testing::Test { return c->ValidateTxns(vertex); } - bool IsValidDistance(Chain* c, Vertex& vtx, uint64_t msHashRate) { - c->CheckTxPartition(vtx, msHashRate); + bool IsValidDistance(Chain* c, Vertex& vtx) { + c->CheckTxPartition(vtx); for (size_t i = 0; i < vtx.validity.size(); ++i) { if (vtx.validity[i] == Vertex::Validity::INVALID) { return false; @@ -486,7 +486,7 @@ TEST_F(TestChainVerification, CheckPartition) { reg_inv.CalculateOptimalEncodingSize(); Vertex reg_inv_vtx{reg_inv}; reg_inv_vtx.minerChainHeight = 1; - EXPECT_FALSE(IsValidDistance(&c, reg_inv_vtx, GENESIS_VERTEX->snapshot->hashRate)); + EXPECT_FALSE(IsValidDistance(&c, reg_inv_vtx)); // Valid registration block Block reg{GetParams().version, @@ -503,7 +503,7 @@ TEST_F(TestChainVerification, CheckPartition) { Vertex reg_vtx{reg}; reg_vtx.minerChainHeight = 1; AddToHistory(&c, std::make_shared(reg_vtx)); - EXPECT_TRUE(IsValidDistance(&c, reg_vtx, GENESIS_VERTEX->snapshot->hashRate)); + EXPECT_TRUE(IsValidDistance(&c, reg_vtx)); // Malicious blocks // Block with transaction but minerChainHeight not reached sortitionThreshold @@ -521,7 +521,7 @@ TEST_F(TestChainVerification, CheckPartition) { Vertex vtx1{b1}; vtx1.minerChainHeight = 2; AddToHistory(&c, std::make_shared(vtx1)); - EXPECT_FALSE(IsValidDistance(&c, vtx1, GENESIS_VERTEX->snapshot->hashRate)); + EXPECT_FALSE(IsValidDistance(&c, vtx1)); // Block with invalid distance Block b2{GetParams().version, @@ -538,5 +538,5 @@ TEST_F(TestChainVerification, CheckPartition) { b2.CalculateOptimalEncodingSize(); Vertex vtx2{b2}; vtx2.minerChainHeight = 3; - EXPECT_FALSE(IsValidDistance(&c, vtx2, 1000000000)); + EXPECT_FALSE(IsValidDistance(&c, vtx2)); }