context: add simple rtt estimator

After b3c1c3b the only source of block time
inaccuracy left is PrepareRequest propagation. In general it's undetermined and
can vary from node to node and from round to round. But in practice it's
relatively stable and can be averaged out over some number of samples. While
backup node can't know when request was sent to it, it can measure what happens
to its request when it's primary since healthy nodes are expected to respond
immediately making request/response pretty similar to ping/pong roundtrip. This
data also can't be perfect, messages can travel different ways from A to B and
B to A on a P2P network, but in practice it's good enough and it's much better
than having no data at all.

Notice that this timer adjustment scheme uses local timer only, we trust our
local clock inevitably and we don't need to mess with any view of time that
other nodes have (including the one from the block header).

It can be attacked by malicious peer or node delaying messages, but as we
expect 2F+1 nodes to be correct one this is enough to collect proper number of
responses and move on, seriously delayed response will be outdated by the time
it comes then. And if any intermediary can delay messages to all nodes for
arbitrary time on our P2P network we can't have any reliable timing anyway.

To me it fixes as much of neo-project/neo#2918 as
only possible.

Signed-off-by: Roman Khimov <[email protected]>

CHANGELOG.md

@@ -11,6 +11,7 @@ Behaviour changes:
 Improvements:
  * timer adjustment for most of the consensus time, more accurate block
    intervals (#55)
+ * timer adjustment for network roundtrip time (#140)
 
 Bugs fixed:
  * inappropriate log on attempt to construct Commit for anti-MEV enabled WatchOnly

context.go

@@ -91,6 +91,9 @@ type Context[H Hash] struct {
 	lastBlockTime      time.Time // Wall clock time of when we started (as in PrepareRequest) creating the last block (used for timer adjustments).
 	lastBlockIndex     uint32
 	lastBlockView      byte
+
+	prepareSentTime time.Time
+	rttEstimates    rtt
 }
 
 // N returns total number of validators.
@@ -236,6 +239,7 @@ func (c *Context[H]) PreBlock() PreBlock[H] {
 
 func (c *Context[H]) reset(view byte, ts uint64) {
 	c.MyIndex = -1
+	c.prepareSentTime = time.Time{}
 	c.lastBlockTimestamp = ts
 
 	if view == 0 {

dbft.go

@@ -153,6 +153,7 @@ func (d *DBFT[H]) initializeConsensus(view byte, ts uint64) {
 		var ts = d.Timer.Now()
 		var diff = ts.Sub(d.lastBlockTime)
 		timeout -= diff
+		timeout -= d.rttEstimates.avg / 2
 		timeout = max(0, timeout)
 	}
 	d.changeTimer(timeout)
@@ -482,6 +483,10 @@ func (d *DBFT[H]) onPrepareResponse(msg ConsensusPayload[H]) {
 		}
 	}
 
+	if d.IsPrimary() && !d.prepareSentTime.IsZero() && !d.recovering {
+		d.rttEstimates.addTime(time.Since(d.prepareSentTime))
+	}
+
 	d.extendTimer(2)
 
 	if !d.Context.WatchOnly() && !d.CommitSent() && (!d.isAntiMEVExtensionEnabled() || !d.PreCommitSent()) && d.RequestSentOrReceived() {

rtt.go

@@ -0,0 +1,26 @@
+package dbft
+
+import (
+	"time"
+)
+
+const rttLength = 7 * 10 // 10 rounds with 7 nodes
+
+type rtt struct {
+	times [rttLength]time.Duration
+	idx   int
+	avg   time.Duration
+}
+
+func (r *rtt) addTime(t time.Duration) {
+	var old = r.times[r.idx]
+
+	if old != 0 {
+		t = min(t, 2*old) // Too long delays should be normalized, we don't want to overshoot.
+	}
+
+	r.avg = r.avg + (t-old)/time.Duration(len(r.times))
+	r.avg = max(0, r.avg) // Can't be less than zero.
+	r.times[r.idx] = t
+	r.idx = (r.idx + 1) % len(r.times)
+}

send.go

@@ -27,6 +27,8 @@ func (d *DBFT[H]) sendPrepareRequest() {
 	d.PreparationPayloads[d.MyIndex] = msg
 	d.broadcast(msg)
 
+	d.prepareSentTime = d.Timer.Now()
+
 	delay := d.SecondsPerBlock << (d.ViewNumber + 1)
 	if d.ViewNumber == 0 {
 		delay -= d.SecondsPerBlock