Merge branch 'origin/develop' into parallel_block_download

Update into latest devp2p

Author: rairyx

pyethapp/synchronizer.py

@@ -30,18 +30,19 @@ def __init__(self, header=None):
 class SyncTask(object):
 
     """
-    synchronizes a the chain starting from a given blockhash
-    blockchain hash is fetched from a single peer (which led to the unknown blockhash)
-    blocks are fetched from the best peers 
-
-    with missing block:
-        fetch headers
-            until known block
-    for headers 
-        fetch block bodies
-            for each block body
-                construct block
-                chainservice.add_blocks() # blocks if queue is full
+    Block header syncing 
+    When syncing with the original peer, from the latest block head of the
+    chain, divide the missing blocks into N sections, each section is made of
+    128  blockheader batches, each batch contains 192 headers, downloading a
+    skeleton of first header of each header batch from the original peer, for
+    each available idle peer, download header batch in parallel, for
+    each batch, match the first header and last header against respected
+    skeleton headers, verify header order and save the downloaded batch into a
+    header cache and deliver the partially downloaded headers to a queue for
+    body downloads scheduling in ascending order.
+    When header section downloading is complete, move the starting header
+    position to the start of next section, if the downloading is interrupted, restart downloading
+    from the head of best block of the current chain`
     """
     initial_blockheaders_per_request = 32
     max_blockheaders_per_request = 192
@@ -359,8 +360,17 @@ def receive_blockheaders(self, proto, blockheaders):
             self.requests[proto].set(blockheaders)
 
 
-class SyncBody(object):
 
+class SyncBody(object):
+    """
+    Handles body syncing
+    For each available peer, fetch block bodies in parallel from the task queue
+    in batches (128), for each body fetch response, match it against headers in
+    the body fetch task queue, if it matches, put the downloaded body in a body
+    result cache, delete the corresponding task from task  queue, import the
+    block bodies from block cache into the chain, remove the imported bodies
+    from body cache
+    """ 
     max_blocks_per_request = 128
     max_blocks_process= 2048 
     blocks_request_timeout = 19.
@@ -412,7 +422,12 @@ def run(self):
             print(traceback.format_exc())
             self.exit(success=False)
 
-
+      
+    #Body fetch scheduler
+    #Body fetch scheduler reads from downloaded header queue, dividing headers
+    #into batches(2048 or less), for each header batch adding the headers to the
+    #task queue, each queue item contains a task of 128 body fetches, activate
+    #body fetcher
     def schedule_block_fetch(self):
         batch_header = []
         log_st.debug('start sheduleing blocks')
@@ -650,7 +665,6 @@ def deliver_blocks(self, proto, bodies):
            self.body_cache = self.body_cache[nimp:]+[None for b in body_result]
            self.body_cache_offset += nimp
            log_body_st.debug('body cache offset', offset=self.body_cache_offset)
-        log_body_st.debug('body cache', bodycache=self.body_cache)
         return result.block
 
 

requirements.txt

@@ -11,9 +11,9 @@ werkzeug
 ipython>=5.4.0,<6.0.0
 statistics
 requests
-rlp>=0.5.1,<0.6.0
-devp2p>=0.8.0
-ethereum>=1.5.1
+rlp>=0.6.0,<0.7.0
+devp2p>=0.9.3
+ethereum>=2.0.4
 pbkdf2
 scrypt
 pexpect

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.5.0
+current_version = 1.5.1a0
 commit = True
 tag = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)((?P<release>[a-z]+)(?P<num>\d+))?

setup.py

@@ -58,7 +58,7 @@ def run_tests(self):
 
 # *IMPORTANT*: Don't manually change the version here. Use the 'bump2version' utility.
 # see: https://github.com/ethereum/pyethapp/wiki/Development:-Versions-and-Releases
-version = '1.5.0'
+version = '1.5.1a0'
 
 setup(
     name='pyethapp',