Add iterate map

Author: kianenigma

Diff for: .gitignore

@@ -1,3 +1,5 @@
+.vscode
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

Diff for: README.md

@@ -1,15 +1,15 @@
 # Python Substrate Interface
- 
+
 [![Travis CI Build Status](https://api.travis-ci.org/polkascan/py-substrate-interface.svg?branch=master)](https://travis-ci.org/polkascan/py-substrate-interface)
-[![Latest Version](https://img.shields.io/pypi/v/substrate-interface.svg)](https://pypi.org/project/substrate-interface/) 
+[![Latest Version](https://img.shields.io/pypi/v/substrate-interface.svg)](https://pypi.org/project/substrate-interface/)
 [![Supported Python versions](https://img.shields.io/pypi/pyversions/substrate-interface.svg)](https://pypi.org/project/substrate-interface/)
 [![License](https://img.shields.io/pypi/l/substrate-interface.svg)](https://github.com/polkascan/py-substrate-interface/blob/master/LICENSE)
 
 Python Substrate Interface Library
 
 ## Description
 This library specializes in interfacing with a Substrate node, providing additional convenience methods to deal with
-SCALE encoding/decoding (the default output and input format of the Substrate JSONRPC), metadata parsing, type registry 
+SCALE encoding/decoding (the default output and input format of the Substrate JSONRPC), metadata parsing, type registry
 management and versioning of types.
 
 ## Documentation
@@ -33,11 +33,11 @@ substrate = SubstrateInterface(
     type_registry_preset='kusama'
 )
 
-substrate.get_chain_head() 
+substrate.get_chain_head()
 ```
 Note on support for wss, this is still quite limited at the moment as connections are not reused yet. Until support is
 improved it is prefered to use http endpoints (e.g. http://127.0.0.1:9933)
-   
+
 ### Get extrinsics for a certain block
 
 ```python
@@ -74,8 +74,8 @@ for extrinsic in result['block']['extrinsics']:
 
 
 ### Make a storage call
-The modules and storage functions are provided in the metadata (see `substrate.get_metadata_storage_functions()`), 
-parameters will be automatically converted to SCALE-bytes (also including decoding of SS58 addresses).   
+The modules and storage functions are provided in the metadata (see `substrate.get_metadata_storage_functions()`),
+parameters will be automatically converted to SCALE-bytes (also including decoding of SS58 addresses).
 
 ```python
 balance_info = substrate.get_runtime_state(
@@ -107,6 +107,17 @@ if balance_info:
     ))
 ```
 
+Or get all the key pairs of a map:
+
+```python
+# Get all the stash and controller bondings.
+all_bonded_stash_ctrls = substrate.iterate_map(
+    module='Staking',
+    storage_function='Bonded',
+    block_hash=block_hash
+)
+```
+
 ### Create and send signed extrinsics
 
 The following code snippet illustrates how to create a call, wrap it in an signed extrinsic and send it to the network:
@@ -157,9 +168,9 @@ if keypair.verify("Test123", signature):
 ### Metadata and type versioning
 
 Py-substrate-interface makes it also possible to easily interprete changed types and historic runtimes. As an example
-we create an (not very useful) historic call of a module that has been removed later on: retrieval of historic metadata and 
-apply the correct version of types in the type registry is all done automatically. Because parsing of metadata and 
-type registry is quite heavy, the result will be cached per runtime id. In the future there could be support for 
+we create an (not very useful) historic call of a module that has been removed later on: retrieval of historic metadata and
+apply the correct version of types in the type registry is all done automatically. Because parsing of metadata and
+type registry is quite heavy, the result will be cached per runtime id. In the future there could be support for
 caching backends like Redis to make this cache more persistent.
 
 Create an unsigned extrinsic of a module that was removed by providing block hash:

Diff for: substrateinterface/__init__.py

@@ -250,7 +250,10 @@ async def ws_request(ws_payload):
                 This method doesn't return but updates the `ws_result` object variable with the result
                 """
                 async with websockets.connect(
-                        self.url
+                        self.url,
+                        max_size=2**32,
+                        read_limit=2**32,
+                        write_limit=2**32,
                 ) as websocket:
                     await websocket.send(json.dumps(ws_payload))
 
@@ -729,6 +732,60 @@ def init_runtime(self, block_hash=None, block_id=None):
                 self.debug_message('Stored metadata for {} in Redis'.format(self.runtime_version))
                 self.cache_region.set('METADATA_{}'.format(self.runtime_version), self.metadata_decoder)
 
+    def iterate_map(self, module, storage_function, block_hash=None):
+        """
+        iterates over all key-pairs localted at the given module and storage_function. The storage
+        item must be a map.
+
+        Parameters
+        ----------
+        module: The module name in the metadata, e.g. Balances or Account.
+        storage_function: The storage function name, e.g. FreeBalance or AccountNonce.
+        block_hash: Optional block hash, when left to None the chain tip will be used.
+
+        Returns
+        -------
+        A two dimensional list of key-value pairs, both decoded into the given type, e.g.
+        [[k1, v1], [k2, v2], ...]
+        """
+        self.init_runtime(block_hash=block_hash)
+
+        key_type = None
+        value_type = None
+        concat_hash_len = None
+        for metadata_module in self.metadata_decoder.metadata.modules:
+            if metadata_module.name == module:
+                if metadata_module.storage:
+                    for storage_item in metadata_module.storage.items:
+                        if storage_item.name == storage_function:
+                            if 'MapType' in storage_item.type:
+                                key_type = storage_item.type['MapType']['key']
+                                value_type = storage_item.type['MapType']['value']
+                                if storage_item.type['MapType']['hasher'] == "Blake2_128Concat":
+                                    concat_hash_len = 32
+                                elif storage_item.type['MapType']['hasher'] == "Twox64Concat":
+                                    concat_hash_len = 16
+                                else:
+                                    raise ValueError('Unsupported hash type')
+                            else:
+                                raise ValueError('Given storage is not a map')
+
+
+        prefix = self.generate_storage_hash(module, storage_function)
+        prefix_len = len(prefix)
+        pairs = self.rpc_request(method="state_getPairs", params=[prefix, block_hash]).get('result')
+
+        # convert keys to the portion that needs to be decoded.
+        pairs = map(lambda kp: ["0x" + kp[0][prefix_len + concat_hash_len:], kp[1]], pairs)
+
+        # decode both of them
+        pairs = map(
+            lambda kp: [self.decode_scale(key_type, kp[0]), self.decode_scale(value_type, kp[1])],
+            list(pairs)
+        )
+
+        return list(pairs)
+
     def get_runtime_state(self, module, storage_function, params=None, block_hash=None):
         """
         Retrieves the storage entry for given module, function and optional parameters at given block hash