diff --git a/commune/client/client.py b/commune/client/client.py index 0db9780a..76e6d66c 100644 --- a/commune/client/client.py +++ b/commune/client/client.py @@ -7,17 +7,6 @@ import aiohttp import json - -from aiohttp.streams import StreamReader - -# Define a custom StreamReader with a higher limit -class CustomStreamReader(StreamReader): - def __init__(self, *args, **kwargs): - # You can adjust the limit here to a value that fits your needs - # This example sets it to 1MB - super().__init__(*args, limit=1024*1024, **kwargs) - - class Client(c.Module): def __init__( diff --git a/commune/client/http/client_http.py b/commune/client/http/client_http.py index 0db9780a..7c8c1888 100755 --- a/commune/client/http/client_http.py +++ b/commune/client/http/client_http.py @@ -171,6 +171,7 @@ def all_history(cls, key=None, history_path='history'): + @classmethod def rm_key_history(cls, key=None, history_path='history'): key = c.get_key(key) diff --git a/commune/key/key.py b/commune/key/key.py index d34a948b..83122be3 100644 --- a/commune/key/key.py +++ b/commune/key/key.py @@ -467,11 +467,11 @@ def rm_key(cls, key=None): assert c.exists(key2path[key]) == False, 'key not deleted' return {'deleted':[key]} + @property def crypto_type_name(self): return self.crypto_type2name(self.crypto_type) - @classmethod def rm_keys(cls, rm_keys, verbose:bool=False): @@ -488,6 +488,7 @@ def rm_keys(cls, rm_keys, verbose:bool=False): @classmethod def rm_all_keys(cls): return cls.rm_keys(cls.keys()) + crypto_types = ['ED25519', 'SR25519', 'ECDSA'] @classmethod @@ -522,15 +523,6 @@ def resolve_crypto_type(cls, crypto_type): assert crypto_type in list(KeypairType.__dict__.values()), f'crypto_type {crypto_type} not supported' return crypto_type - @classmethod - def gen_n(cls, n=10, **kwargs): - keys = [] - for i in range(n): - keys.append(cls.gen(**kwargs)) - - return keys - - @classmethod def gen(cls, mnemonic:str = None, @@ -557,11 +549,8 @@ def gen(cls, else: mnemonic = cls.generate_mnemonic() key = cls.create_from_mnemonic(mnemonic, crypto_type=crypto_type) - if json: return key.to_json() - - return key diff --git a/commune/key/key_docs.md b/commune/key/key_docs.md deleted file mode 100644 index 44a2c665..00000000 --- a/commune/key/key_docs.md +++ /dev/null @@ -1,97 +0,0 @@ -```markdown -# Key Management - -In this tutorial, we'll explore the usage of the `commune` Python package for managing keys, balances, stakes, and key statistics. - -## Getting Started - -First, you need to import the `commune` module: - -```python -import commune as c -``` - -## Listing Keys - -To start, let's list all the available keys using the `keys()` function: - -```python -c.keys() -``` - -## Adding and Removing Keys - -You can add and remove keys with the following steps: - -### Adding a New Key - -To add a new key, use the `add_key()` function: - -```python -c.add_key('fam') -``` - -### Removing a Key - -To remove a key, use the `rm_key()` function: - -```python -c.rm_key('demo') # Replace 'demo' with the key you want to remove -``` - -## Saving and Loading Keys - -You can save and load keys for future use: - -### Saving Keys - -To save the keys, use the `save_keys()` function: - -```python -c.save_keys() -``` - -### Loading Keys - -To load the saved keys, use the `load_keys()` function: - -```python -c.load_keys() -``` - -## Retrieving Balances and Stakes - -You can retrieve balance and stake information for a specific key: - -### Balance - -To get the balance for a key, use the `get_balance()` function: - -```python -c.get_balance('fam') # Replace 'fam' with the key name -``` - -### Stake - -To get the stake for a key, use the `get_stake()` function: - -```python -c.get_stake('fam') # Replace 'fam' with the key name -``` - -## Retrieving Key Statistics - -You can also retrieve key statistics for each subnet associated with a key: - -```python -c.key_stats('fam') # Replace 'fam' with the key name -``` - -## Conclusion - -In this tutorial, you learned how to use the `commune` Python package to manage keys, balances, stakes, and key statistics. Feel free to explore further and adapt these concepts to your specific use cases. - -Remember to refer to the official documentation of the `commune` package for more detailed information and additional features. -``` - -Feel free to adjust the formatting or wording to better match your preferred style or context! \ No newline at end of file diff --git a/commune/module/module.py b/commune/module/module.py index 3fdb7eb3..0ba8e4ac 100755 --- a/commune/module/module.py +++ b/commune/module/module.py @@ -2988,9 +2988,7 @@ def serve(cls, else: port = c.free_port() - # NOTE REMOVE THIS FROM THE KWARGS REMOTE - if remote: # GET THE LOCAL KWARGS FOR SENDING TO THE REMOTE @@ -3033,7 +3031,7 @@ def serve(cls, tag = None self.tag = tag - self.key = server_name + address = c.get_address(server_name, network=network) if address != None and ':' in address: @@ -3053,7 +3051,7 @@ def serve(cls, setattr(self, 'whitelist', whitelist) setattr(self, 'blacklist', blacklist) - c.module(f'server.{mode}')(module=self, + c.module(f'server')(module=self, name=server_name, port=port, network=network, diff --git a/commune/server/server.py b/commune/server/server.py index ff3c2f2d..6cf06f1a 100644 --- a/commune/server/server.py +++ b/commune/server/server.py @@ -55,11 +55,10 @@ def __init__( self.name = name - self.schema = {} if hasattr(module, 'schema'): self.schema = module.schema() else: - self.schema = c.get_schema(module) + self.schema = c.schema(module) module.ip = self.ip module.port = self.port @@ -78,6 +77,8 @@ def set_key(self, key): self.key = c.get_key(self.name) if isinstance(self.key, str): self.key = c.get_key(self.key) + self.module.key = self.key + c.print(f'🔑 Key: {self.key} 🔑\033') def set_address(self,ip='0.0.0.0', port:int=None): @@ -276,13 +277,23 @@ def generator_wrapper(self, generator): @classmethod - def test(cls): + def test_serving(cls): module_name = 'storage::test' module = c.serve(module_name, wait_for_server=True) module = c.connect(module_name) module.put("hey",1) c.kill(module_name) + @classmethod + def test_serving_with_different_key(cls): + module_name = 'storage::test' + module = c.serve(module_name, wait_for_server=True) + module = c.connect(module_name) + module.put("hey",1) + c.kill(module_name) + + + # HISTORY diff --git a/commune/server/ucall/__init__.py b/commune/server/ucall/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/commune/server/ucall/client/__init__.py b/commune/server/ucall/client/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/commune/server/ucall/client/ucall_client.py b/commune/server/ucall/client/ucall_client.py deleted file mode 100755 index bc0e94bc..00000000 --- a/commune/server/ucall/client/ucall_client.py +++ /dev/null @@ -1,117 +0,0 @@ - - -from typing import Tuple, List, Union -import sys -import os -import asyncio -import requests -from functools import partial -import commune as c -import aiohttp -import json -from ucall.client import Client -from aiohttp.streams import StreamReader - - -class ClientUcall(c.Module): - - def __init__( - self, - ip: str ='0.0.0.0', - port: int = 50053 , - network: bool = None, - key : str = None, - loop: 'asyncio.EventLoop' = None, - use_http: bool = True - ): - self.loop = c.get_event_loop() if loop == None else loop - - - - self.ip = ip = ip if ip else c.default_ip - self.port = port = port if port else c.free_port() - self.address = f"{self.ip}:{self.port}" - self.use_http = use_http - - self.client = Client(uri = self.ip, port = self.port, use_http = use_http) - - self.set_client(ip =ip, port = port, use_http=use_http) - self.serializer = c.serializer() - self.key = c.get_key(key) - self.my_ip = c.ip() - self.network = c.resolve_network(network) - self.start_timestamp = c.timestamp() - - def age(self): - return self.start_timestamp - c.timestamp() - - - async def async_forward(self, - fn: str, - args: list = None, - kwargs: dict = None, - timeout: int = 10, - headers : dict ={'Content-Type': 'application/json'}): - - args = args if args else [] - kwargs = kwargs if kwargs else {} - - request_data = { - "args": args, - "kwargs": kwargs, - "ip": self.my_ip, - "timestamp": c.timestamp(), - } - - # serialize this into a json string - request_data = self.serializer.serialize( request_data) - - # sign the request - params = self.key.sign(request_data, return_json=True) - - client = self.client.remote_call(fn=fn, input=params) - - - return response - - - def process_output(self, result): - ## handles - if isinstance(result, str): - result = json.loads(result) - if 'data' in result: - result = self.serializer.deserialize(result['data']) - return result['data'] - else: - return result - - def forward(self,*args,return_future:bool=False, timeout:str=4, **kwargs): - forward_future = asyncio.wait_for(self.async_forward(*args, **kwargs), timeout=timeout) - if return_future: - return forward_future - else: - return self.loop.run_until_complete(forward_future) - - - __call__ = forward - - def __str__ ( self ): - return "Client({})".format(self.address) - def __repr__ ( self ): - return self.__str__() - def __exit__ ( self ): - self.__del__() - - - def test_module(self): - module = Client(ip='0.0.0.0', port=8091) - import torch - data = { - 'bro': torch.ones(10,10), - 'fam': torch.zeros(10,10) - } - - def virtual(self): - return c.virtual_client(module = self) - - diff --git a/commune/server/ucall/server_ucall.py b/commune/server/ucall/server_ucall.py deleted file mode 100755 index 603a4afb..00000000 --- a/commune/server/ucall/server_ucall.py +++ /dev/null @@ -1,242 +0,0 @@ - -from typing import Dict, List, Optional, Union -import commune as c -import torch -import traceback -import json - - - - -class ServerUcall(c.Module): - def __init__( - self, - module: Union[c.Module, object], - name: str = None, - network:str = 'local', - port: Optional[int] = None, - sse: bool = False, - chunk_size: int = 42_000, - max_request_staleness: int = 60, - verbose: bool = False, - timeout: int = 256, - access_module: str = 'server.access', - public: bool = False, - serializer: str = 'serializer', - new_event_loop:bool = True - ) -> 'Server': - - self.serializer = c.module(serializer)() - self.ip = c.default_ip # default to '0.0.0.0' - self.port = int(port) if port != None else c.free_port() - self.address = f"{self.ip}:{self.port}" - self.max_request_staleness = max_request_staleness - self.network = network - self.verbose = verbose - self.sse = sse - assert self.sse == False, f"SSE not implemented yet" - self.chunk_size = chunk_size - self.timeout = timeout - self.public = public - self.module = module - if new_event_loop: - c.new_event_loop() - - # name - if name == None: - if hasattr(self.module, 'server_name'): - name = self.module.server_name - else: - name = self.module.__class__.__name__ - self.name = name - - - self.key = module.key - # register the server - module.ip = self.ip - module.port = self.port - module.address = self.address - self.access_module = c.module(access_module)(module=self.module) - - - self.set_api(ip=self.ip, port=self.port) - - - def set_api(self, ip:str = '0.0.0.0', port:int = 8888): - ip = self.ip if ip == None else ip - port = self.port if port == None else port - - - from ucall.posix import Server - # from ucall.uring import Server on 5.19+ - self.app = Server(port=self.port) - @self.app - def remote_call(fn:str, input:dict): - """ - THE ULTIMATE RPC CALL - - fn (str): the function to call - input (dict): the input to the function - data (dict): the data to pass to the function - kwargs (dict): the keyword arguments to pass to the function - args (list): the positional arguments to pass to the function - timestamp (int/float): the timestamp of the request - signature (str): the signature of the data request - - """ - input['fn'] = fn - input = self.process_input(input) - data = input['data'] - args = data.get('args',[]) - kwargs = data.get('kwargs', {}) - - input_kwargs = dict(fn=fn, - args=args, - kwargs=kwargs) - fn_name = f"{self.name}::{fn}" - c.print(f'🚀 Forwarding {input["address"]} --> {fn_name} 🚀\033', color='yellow') - - - try: - result = self.forward(**input_kwargs) - # if the result is a future, we need to wait for it to finish - except Exception as e: - result = c.detailed_error(e) - - if isinstance(result, dict) and 'error' in result: - success = False - success = True - - - if success: - c.print(f'✅ Success: {self.name}::{fn} --> {input["address"]}... ✅\033 ', color='green') - else: - c.print(f'🚨 Error: {self.name}::{fn} --> {input["address"]}... 🚨\033', color='red') - result = self.process_result(result) - return result - - self.serve() - - - - def state_dict(self) -> Dict: - return { - 'ip': self.ip, - 'port': self.port, - 'address': self.address, - } - - - - def process_input(self,input: dict) -> bool: - - """ - INPUT - { - 'data': { - 'args': [], - 'kwargs': {}, - 'timestamp': 0, - }, - } - - - """ - assert 'data' in input, f"Data not included" - - # you can verify the input with the server key class - if not self.public: - assert 'signature' in input, f"Data not signed" - assert self.key.verify(input), f"Data not signed with correct key" - input['data'] = self.serializer.deserialize(input['data']) - - if self.verbose: - # here we want to verify the data is signed with the correct key - request_staleness = c.timestamp() - input['data'].get('timestamp', 0) - # verifty the request is not too old - assert request_staleness < self.max_request_staleness, f"Request is too old, {request_staleness} > MAX_STALENESS ({self.max_request_staleness}) seconds old" - self.access_module.verify(input) - - - return input - - - def process_result(self, result): - if self.sse: - # from sse_starlette.sse import EventSourceResponse - # # for sse we want to wrap the generator in an eventsource response - # result = self.generator_wrapper(result) - # return EventSourceResponse(result) - assert False, f"SSE not implemented yet" - else: - # if we are not using sse, then we can do this with json - if c.is_generator(result): - result = list(result) - result = self.serializer.serialize({'data': result}) - result = self.key.sign(result, return_json=True) - return result - - - def generator_wrapper(self, generator): - if not c.is_generator(generator): - generator = [generator] - - for item in generator: - # we wrap the item in a json object, just like the serializer does - item = self.serializer.serialize({'data': item}) - item = self.key.sign(item, return_json=True) - item = json.dumps(item) - item_size = c.sizeof(item) - if item_size > self.chunk_size: - # if the item is too big, we need to chunk it - item_hash = c.hash(item) - chunks =[f'CHUNKSTART:{item_hash}'] + [item[i:i+self.chunk_size] for i in range(0, item_size, self.chunk_size)] + [f'CHUNKEND:{item_hash}'] - # we need to yield the chunks in a format that the eventsource response can understand - for chunk in chunks: - yield chunk - - yield item - - - def serve(self, **kwargs): - import uvicorn - - try: - c.print(f'\033🚀 Serving {self.name} on {self.address} 🚀\033') - c.register_server(name=self.name, address = self.address, network=self.network) - c.print(f'\033🚀 Registered {self.name} --> {self.ip}:{self.port} 🚀\033') - self.app.run() - except Exception as e: - c.print(e, color='red') - c.deregister_server(self.name, network=self.network) - finally: - c.deregister_server(self.name, network=self.network) - - - def forward(self, fn: str, args: List = None, kwargs: Dict = None, **extra_kwargs): - if args is None: - args = [] - if kwargs is None: - kwargs = {} - obj = getattr(self.module, fn) - if callable(obj): - response = obj(*args, **kwargs) - else: - response = obj - - return response - - - def __del__(self): - c.deregister_server(self.name) - - - @classmethod - def test(cls): - self = cls(module=c.module("module")()) - - - - @classmethod - def install(cls): - return c.cmd("pip3 install ucall") diff --git a/commune/subspace/subspace.py b/commune/subspace/subspace.py index b55a982f..c66ae91c 100644 --- a/commune/subspace/subspace.py +++ b/commune/subspace/subspace.py @@ -390,7 +390,6 @@ def query_map(self, name: str = 'StakeFrom', module='SubspaceModule', update: bool = True, max_age = None, # max age in seconds - new_connection=False, mode = 'ws', **kwargs ) -> Optional[object]: @@ -404,8 +403,7 @@ def query_map(self, name: str = 'StakeFrom', if name == 'Account': module = 'System' - network = self.resolve_network(network, new_connection=new_connection, mode=mode) - + network = self.resolve_network(network, new_connection=False, mode=mode) path = f'query/{network}/{module}.{name}' # resolving the params params = params or [] diff --git a/commune/vali/vali.py b/commune/vali/vali.py index 3554f2a1..b67d0743 100644 --- a/commune/vali/vali.py +++ b/commune/vali/vali.py @@ -385,9 +385,22 @@ def save_votes(self, votes:dict): def vote(self, async_vote:bool=False, save:bool = True, + cache_exceptions:bool=True, **kwargs): - - + + + if cache_exceptions: + try: + return self.vote(async_vote=async_vote, save=save, cache_exceptions=False, **kwargs) + except Exception as e: + return c.detailed_error(e) + + if not self.should_vote: + return {'success': False, + 'msg': 'Not voting', + 'network': self.network, + 'vote_staleness': self.vote_staleness, + 'vote_interval': self.config.vote_interval} if async_vote: return c.submit(self.vote, **kwargs) @@ -588,12 +601,8 @@ def vote_staleness(self): def vote_loop(self): while True: - try: - - c.print(self.vote()) - - except Exception as e: - c.print(c.detailed_error(e)) + r = self.vote(cache_exceptions=True) + run_info = self.run_info() c.print(run_info, color='cyan') c.sleep(self.config.sleep_interval) diff --git a/docs/managing_keys.md b/docs/key_management.md similarity index 84% rename from docs/managing_keys.md rename to docs/key_management.md index f03a6491..14c815b4 100644 --- a/docs/managing_keys.md +++ b/docs/key_management.md @@ -112,24 +112,24 @@ To get the balance for a key, use the `get_balance()` function: ```python c.get_balance('fam') # Replace 'fam' with the key name ``` +or +```bash +c get_balance fam +``` ### Stake To get the stake for a key, use the `get_stake()` function: +```bash +c get_stake fam # Replace 'fam' with the key name or the address +``` + ```python c.get_stake('fam') # Replace 'fam' with the key name ``` +### Get Registered Keys - -## Conclusion - -In this tutorial, you learned how to use the `commune` Python package to manage keys, balances, stakes, and key statistics. Feel free to explore further and adapt these concepts to your specific use cases. - -Remember to refer to the official documentation of the `commune` package for more detailed information and additional features. -``` - -Feel free to adjust the formatting or wording to better match your preferred style or context! \ No newline at end of file diff --git a/docs/module.md b/docs/module.md index 985f6874..315f66a7 100644 --- a/docs/module.md +++ b/docs/module.md @@ -13,21 +13,13 @@ In this tutorial, we'll explore how to use the `commune` library for module mana You can use the following steps to find and work with modules using the `commune` library. -### List All Modules -You can list all available modules using the following code: +## New Module Creation +To create a new module, you can use the `commune` command line tool: -```python -import commune as c -modules_list = c.modules()[:10] -c.print(modules_list) -``` - -To Serve a module -```python -c.serve('model.openai') +```bash +c new_module agi ``` - ### Searching for a Specific Module To search for a specific module, you can use the `c.modules()` function with a search query: diff --git a/docs/subspace/mining/miner.md b/docs/subspace/mining/miner.md index acfed3d7..a15c2523 100644 --- a/docs/subspace/mining/miner.md +++ b/docs/subspace/mining/miner.md @@ -10,12 +10,10 @@ To register a validator with a specific tag, use the following CLI command: INPUT ```bash -c serve model.openai::whatdp +c serve model.openai::whadup # c register model.openai::whadup subnet=commune ``` - - ```python c.module('model.openai').register(tag='whadup', subnet=commune) ```