diff --git a/commune/__init__.py b/commune/__init__.py index 69ff1ae47..db1c79a5e 100755 --- a/commune/__init__.py +++ b/commune/__init__.py @@ -1,13 +1,13 @@ -from .module import Module -from .vali import Vali -from .server import Server -from .client import Client -from .key import Key +from .module import Module # the module module +from .vali import Vali # the vali module +from .server import Server # the server module +from .client import Client # the client module +from .key import Key # the key module # set the module functions as globalsw -IDONTGIVEAFUCKWHATYOUCALLTHIS = c = Block = Lego = M = Agent = Module # alias c.Module as c.Block, c.Lego, c.M +M = IDONTGIVEAFUCKWHATYOUCALLTHIS = c = Block = Lego = M = Agent = Module # alias c.Module as c.Block, c.Lego, c.M c.add_to_globals(globals()) -key = c.get_key # override key function with file key in commune/key.py +key = c.get_key # override key function with file key in commune/key.py TODO: remove this line with a better solution diff --git a/commune/module.py b/commune/module.py index b9fff937f..2308d090e 100755 --- a/commune/module.py +++ b/commune/module.py @@ -526,6 +526,7 @@ def add_routes(cls, routes:dict=None, verbose=False, add_utils=True): This allows you to call the function as if it were a method of the current module. for example """ + routes = routes or cls.get_routes() t0 = time.time() # WARNING : THE PLACE HOLDERS MUST NOT INTERFERE WITH THE KWARGS OTHERWISE IT WILL CAUSE A BUG IF THE KWARGS ARE THE SAME AS THE PLACEHOLDERS # THE PLACEHOLDERS ARE NAMED AS module_ph and fn_ph AND WILL UNLIKELY INTERFERE WITH THE KWARGS @@ -545,8 +546,7 @@ def fn(*args, **kwargs): else: return fn_obj return fn(*args, **kwargs) - - for module, fns in cls.get_routes().items(): + for module, fns in routes.items(): for fn in fns: if not hasattr(cls, fn): fn_obj = partial(fn_generator, route=module + '.' + fn) @@ -846,8 +846,9 @@ def resolve_path(cls, path:str = None, extension:Optional[str]=None): def abspath(cls, path:str): return os.path.abspath(os.path.expanduser(path)) + @classmethod - def put_text(cls, path:str, text:str, key=None, bits_per_character=8) -> None: + def put_text(cls, path:str, text:str, key=None) -> None: # Get the absolute path of the file path = cls.resolve_path(path) dirpath = os.path.dirname(path) @@ -861,13 +862,10 @@ def put_text(cls, path:str, text:str, key=None, bits_per_character=8) -> None: with open(path, 'w') as file: file.write(text) # get size - text_size = len(text)*bits_per_character - - return {'success': True, 'path': f'{path}', 'size': text_size} + return {'success': True, 'path': f'{path}', 'size': len(text)*8} @classmethod def ls(cls, path:str = '', - recursive:bool = False, search = None, return_full_path:bool = True): """ @@ -1860,7 +1858,7 @@ def get_tree(cls, path, depth = 10, max_age=60, update=False, **kwargs): tree_cache_path = 'tree/'+os.path.abspath(path).replace('/', '_') tree = c.get(tree_cache_path, None, max_age=max_age, update=update) if tree == None: - c.print(f'TREE(max_age={max_age}, depth={depth}, pat={path})', color='green') + c.print(f'TREE(max_age={max_age}, depth={depth}, path={path})', color='green') class_paths = cls.find_classes(path, depth=depth) simple_paths = [cls.objectpath2name(p) for p in class_paths] tree = dict(zip(simple_paths, class_paths)) @@ -1894,7 +1892,6 @@ def module(cls, path:str = 'module', try: module = c.import_object(path) except Exception as e: - tree = c.tree(update=1) if trials == 0: raise ValueError(f'Error in module {og_path} {e}') return c.module(path, cache=cache, verbose=verbose, tree=tree, trials=trials-1) @@ -1916,6 +1913,7 @@ def module(cls, path:str = 'module', module.params = lambda *args, **kwargs : c.params(module) module.key = c.get_key(module.module_name(), create_if_not_exists=True) module.fn2code = lambda *args, **kwargs : c.fn2code(module) + module.help = lambda *args, **kwargs : c.help(*args, module=module, **kwargs) c.print(f'Module({og_path}->{path})({latency}s)', verbose=verbose) return module @@ -2093,9 +2091,9 @@ def set_api_key(self, api_key:str, cache:bool = True): self.add_api_key(api_key) assert isinstance(api_key, str) - def add_api_key(self, api_key:str, path=None): + def add_api_key(self, api_key:str): assert isinstance(api_key, str) - path = self.resolve_path(path or 'api_keys') + path = self.resolve_path('api_keys') api_keys = self.get(path, []) api_keys.append(api_key) api_keys = list(set(api_keys)) @@ -2104,8 +2102,7 @@ def add_api_key(self, api_key:str, path=None): def set_api_keys(self, api_keys:str): api_keys = list(set(api_keys)) - self.put('api_keys', api_keys) - return {'api_keys': api_keys} + return self.put('api_keys', api_keys) def rm_api_key(self, api_key:str): assert isinstance(api_key, str) @@ -2115,12 +2112,11 @@ def rm_api_key(self, api_key:str): api_keys.pop(i) break path = self.resolve_path('api_keys') - self.put(path, api_keys) - return {'api_keys': api_keys} + return self.put(path, api_keys) def get_api_key(self, module=None): if module != None: - self = self.module(module) + self = c.module(module) api_keys = self.api_keys() if len(api_keys) == 0: raise @@ -2155,12 +2151,7 @@ def remote_fn(cls, if 'remote' in kwargs: kwargs['remote'] = False assert fn != None, 'fn must be specified for pm2 launch' - kwargs = { - 'module': module, - 'fn': fn, - 'args': args, - 'kwargs': kwargs - } + kwargs = {'module': module, 'fn': fn, 'args': args, 'kwargs': kwargs} name = name or module if refresh: c.kill(name) @@ -2184,6 +2175,8 @@ def resolve_extension( filename:str, extension = '.py') -> str: return filename + extension def help(self, *text, module=None, global_context=f'{rootpath}/docs', **kwargs): + if self.module_name() == 'module': + return c.module('docs')().help(*text) text = ' '.join(map(str, text)) if global_context != None: text = text + str(c.file2text(global_context)) @@ -2239,7 +2232,6 @@ def fn2module(cls, path=None): fn2module[f] = m return fn2module - def install(self, path ): path = path + '/requirements.txt' @@ -2251,9 +2243,195 @@ def epoch(self, *args, **kwargs): return c.run_epoch(*args, **kwargs) -c.routes = c.get_routes() +c.routes = { + "vali": [ + "run_epoch", + "setup_vali", + "from_module" + ], + "py": [ + "envs", + "env2cmd", + "create_env", + "env2path" + ], + "cli": [ + "parse_args" + ], + "streamlit": [ + "set_page_config", + "load_style", + "st_load_css" + ], + "docker": [ + "containers", + "dlogs", + "images" + ], + "client": [ + "call", + "call_search", + "connect" + ], + "repo": [ + "is_repo", + "repos" + ], + "serializer": [ + "serialize", + "deserialize", + "serializer_map", + ], + "key": [ + "rename_key", + "ss58_encode", + "ss58_decode", + "key2mem", + "key_info_map", + "key_info", + "valid_ss58_address", + "valid_h160_address", + "add_key", + "from_password", + "str2key", + "pwd2key", + "getmem", + "mem", + "mems", + "switch_key", + "module_info", + "rename_kefy", + "mv_key", + "add_keys", + "key_exists", + "ls_keys", + "rm_key", + "key_encrypted", + "encrypt_key", + "get_keys", + "rm_keys", + "key2address", + "key_addresses", + "address2key", + "is_key", + "new_key", + "save_keys", + "load_key", + "load_keys", + "get_signer", + "encrypt_file", + "decrypt_file", + "get_key_for_address", + "resolve_key_address", + "ticket" + ], + "remote": [ + "host2ssh" + ], + "network": [ + "networks", + "register_server", + "deregister_server", + "server_exists", + "add_server", + "has_server", + "add_servers", + "rm_servers", + "rm_server", + "namespace", + "namespace", + "infos", + "get_address", + "servers", + "name2address" + ], + "app": [ + "start_app", + "app", + "apps", + "app2info", + "kill_app" + ], + "user": [ + "role2users", + "is_user", + "get_user", + "update_user", + "get_role", + "refresh_users", + "user_exists", + "is_admin", + "admins", + "add_admin", + "rm_admin", + "num_roles", + "rm_user" + ], + "server": [ + "serve", + "wait_for_server", + "endpoint", + "is_endpoint", + "fleet", + "processes", + "kill", + "kill_many", + "kill_all", + "kill_all_processes", + "logs" + ], + + "subspace": [ + "transfer_stake", + "stake_trnsfer", + "switch", + "switchnet", + "subnet", + "update_module", + "subnet_params_map", + "staketo", + "network", + "get_staketo", + "stakefrom", + "get_stakefrom", + "switch_network", + "key2balance", + "subnets", + "send", + "my_keys", + "key2value", + "transfer", + "multistake", + "stake", + "unstake", + "register", + "subnet_params", + "global_params", + "balance", + "get_balance", + "get_stak", + "get_stake_to", + "get_stake_from", + "my_stake_to", + "netuid2subnet", + "subnet2netuid", + "is_registered", + "update_subnet", + "my_subnets", + "my_netuids", + "register_subnet", + "registered_subnets", + "registered_netuids" + ], + "model.openrouter": [ + "generate", + "models" + ], + "chat": ["ask", "models", "pricing", "model2info"] +} c.add_routes() Module = c # Module is alias of c Module.run(__name__) + diff --git a/commune/modules/agent/agent.py b/commune/modules/agent/agent.py index 0eddd0514..d736236ae 100644 --- a/commune/modules/agent/agent.py +++ b/commune/modules/agent/agent.py @@ -30,13 +30,50 @@ def args2text(self, args): def find_text(self, *args, **kwargs): text = [c.get_text(f) for f in self.find_files(*args, **kwargs)] - size = self.get_size(text) return size def get_size(self, x): return len(str(x)) - + + + def modules(self, + query='', + output_format="DICT(data:list[str])" , + path='./', + n=5, + model='sonnet'): + front_anchor = f"<{self.anchor}>" + back_anchor = f"" + context = c.modules() + prompt = f""" + QUERY + {query} + INSTRUCTION + get the top {n} files that match the query + instead of using the full {os.path.expanduser('~')}, use ~ + CONTEXT + {context} + OUTPUT + (JSON ONLY AND ONLY RESPOND WITH THE FOLLOWING INCLUDING THE ANCHORS SO WE CAN PARSE) + {front_anchor}{output_format}{back_anchor} + """ + output = '' + for ch in c.ask(prompt, model=model): + print(ch, end='') + output += ch + if ch == front_anchor: + break + if '```json' in output: + output = output.split('```json')[1].split('```')[0] + elif front_anchor in output: + output = output.split(front_anchor)[1].split(back_anchor)[0] + else: + output = output + output = json.loads(output) + assert len(output) > 0 + return output + def find_files(self, query='', @@ -93,4 +130,5 @@ def batch_context(self, path='./', batch_size=20000): def tools(self): return c.fns("fn") - \ No newline at end of file + + diff --git a/commune/modules/sandbox.py b/commune/modules/sandbox.py index c8bb3c64d..faf867a75 100644 --- a/commune/modules/sandbox.py +++ b/commune/modules/sandbox.py @@ -1,3 +1,3 @@ import commune as c -c.ask('find a file with openai ', c.files()) \ No newline at end of file +c.print(c.modules()) \ No newline at end of file diff --git a/commune/network/utils.py b/commune/modules/selector/selector.py similarity index 100% rename from commune/network/utils.py rename to commune/modules/selector/selector.py diff --git a/commune/network/network.py b/commune/network/network.py index bee0235ad..bd33a98b2 100644 --- a/commune/network/network.py +++ b/commune/network/network.py @@ -3,6 +3,7 @@ import os class Network(c.Module): + min_stake = 0 blocktime = block_time = 8 n = 100 tempo = 60 diff --git a/commune/network/subspace/subspace.py b/commune/network/subspace/subspace.py index 094e4b6fe..51f3c972b 100644 --- a/commune/network/subspace/subspace.py +++ b/commune/network/subspace/subspace.py @@ -29,9 +29,11 @@ class Subspace(c.Module): + min_stake = 50000 tempo = 60 - blocktime =block_time = 8 + blocktime = block_time = 8 blocks_per_day = 24*60*60/block_time + url_map = { "main": [ "api.communeai.net" @@ -898,9 +900,9 @@ def compose_call( Raises: ChainTransactionError: If the transaction fails. """ - c.print(f'Calling module={module} fn={fn} network={self.network} url={self.url})') - c.print(f'params: {params}') - + c.print(f'SUBSPACE({module}/{fn} network={self.network} url={self.url})') + c.print('PARAMS --> ',params) + key = self.resolve_key(key) if key is None and not unsigned: @@ -1084,7 +1086,7 @@ def to_nanos(self, amount): return amount * 10**9 - def top_miners( self, subnet, amount=4, key='module'): + def send_my_modules( self, amount=1, subnet=0, key='module'): destinations = self.my_keys(subnet) amounts = [amount] * len(destinations) return self.transfer_multiple(key=key, destinations=destinations,amounts=amounts) @@ -1182,7 +1184,7 @@ def update_modules( self, subnet: str, timeout: int=60) -> ExtrinsicReceipt: modules = self.my_modules(subnet) futures = [] for m in modules: - if m['ready']: + if m['serving']: continue print(f'Updating {m["name"]}') futures += [c.submit(self.update_module, dict(name=m['name'], subnet=subnet), timeout=timeout)] @@ -1264,8 +1266,6 @@ def register( subnet: The network subnet to register the module in. If None, a default value is used. """ - - key = c.get_key(key) if address == None: namespace = c.namespace() @@ -1285,7 +1285,16 @@ def register( response = self.compose_call("register", params=params, key=key, wait_for_finalization=wait_for_finalization) return response - def deregister(self, key: Keypair, subnet: int) -> ExtrinsicReceipt: + def dereg(self, key: Keypair, subnet: int=0): + return self.deregister(key=key, subnet=subnet) + + def dereg_many(self, *key: Keypair, subnet: int = 0): + futures = [c.submit(self.deregister, dict(key=k, subnet=subnet)) for k in key ] + results = [] + for f in c.as_completed(futures): + results += [f.result()] + return results + def deregister(self, key: Keypair, subnet: int=0) -> ExtrinsicReceipt: """ Deregisters a module from the network. @@ -1307,6 +1316,9 @@ def deregister(self, key: Keypair, subnet: int) -> ExtrinsicReceipt: return response + def reg(self, key: Keypair, subnet: int=0): + return self.register(key=key, subnet=subnet) + def register_subnet(self, name: str, metadata: str | None = None, key: Keypair=None) -> ExtrinsicReceipt: """ Registers a new subnet in the network. @@ -1329,6 +1341,8 @@ def register_subnet(self, name: str, metadata: str | None = None, key: Keypair= } response = self.compose_call("register_subnet", params=params, key=key) return response + + regnet = register_subnet def set_weights( self, @@ -1962,10 +1976,8 @@ def max_allowed_weights( """ Retrieves a mapping of maximum allowed weights for the network. """ - return self.query_map("MaxAllowedWeights", extract_value=extract_value) - def legit_whitelist( self, extract_value: bool = False ) -> dict[Ss58Address, int]: @@ -1974,7 +1986,6 @@ def legit_whitelist( """ return self.query_map( "LegitWhitelist", module="GovernanceModule", extract_value=extract_value) - def subnet_names(self, extract_value: bool = False, max_age=60, update=False) -> dict[int, str]: """ Retrieves a mapping of subnet names within the network. @@ -1983,7 +1994,6 @@ def subnet_names(self, extract_value: bool = False, max_age=60, update=False) -> return {int(k):v for k,v in subnet_names.items()} - def subnet_map(self, max_age=10, update=False) -> dict[int, str]: """ Retrieves a mapping of subnet names within the network. @@ -2029,21 +2039,16 @@ def subnets(self): return self.subnet_names() def get_balances( - self, addresses=None, extract_value: bool = False, block_hash: str | None = None + self, key_addresses=None, extract_value: bool = False, block_hash: str | None = None ) -> dict[str, dict[str, int | dict[str, int | float]]]: """ Retrieves a mapping of account balances within the network. """ key2address = c.key2address() - addresses = addresses or list(key2address.values()) - addresses = [key2address.get(a, a) for a in addresses] + key_addresses = key_addresses or list(key2address.values()) + key_addresses = [key2address.get(a, a) for a in key_addresses] with self.get_conn(init=True) as substrate: - balances = substrate.query_multi( - [ - substrate.create_storage_key(pallet='System', storage_function='Account', params=[address]) for address in addresses if not address.startswith('0x') - ] - ) - + balances = substrate.query_multi( [substrate.create_storage_key(pallet='System', storage_function='Account', params=[ka]) for ka in key_addresses if not ka.startswith('0x')]) return len(balances) def names( @@ -2474,20 +2479,19 @@ def my_modules(self, subnet=0, max_age=60, features=['name', 'key', 'address', ' my_keys += [k] modules = self.get_modules(my_keys, subnet=subnet) for i,m in enumerate(modules): - ready = m['name'] in namespace - m['ready'] = ready - local_key_alias = address2key[m['key']] + serving = m['name'] in namespace + m['serving'] = serving m['name'] = address2key[m['key']] modules[i] = m - - features += ['ready'] - + features += ['serving'] modules = [{f:m[f] for f in features} for m in modules] return modules + + def my_valis(self, subnet=0): + return [m for m in self.my_modules(subnet) if m['stake'] > self.min_stake] def my_keys(self, subnet=0): - key2address = c.key2address() - return [key2address[k] for k in self.my_modules(subnet)] + return [m['key'] for m in self.my_modules(subnet)] def all_modules(self, diff --git a/commune/serializer/pandas.py b/commune/serializer/pandas.py index 93820002c..3b60712d8 100644 --- a/commune/serializer/pandas.py +++ b/commune/serializer/pandas.py @@ -1,15 +1,16 @@ import json +import pandas as pd class PandasSerializer: - def serialize(self, data: 'pd.DataFrame') -> 'DataBlock': + def serialize(self, data: pd.DataFrame) -> 'DataBlock': data = data.to_json() if isinstance(data, bytes): data = data.decode('utf-8') return data - def deserialize(self, data: bytes) -> 'pd.DataFrame': - import pandas as pd + def deserialize(self, data: bytes) -> pd.DataFrame: + data = pd.DataFrame.from_dict(json.loads(data)) print(data) return data diff --git a/commune/server.py b/commune/server.py index c4444573f..98cf057d6 100644 --- a/commune/server.py +++ b/commune/server.py @@ -181,10 +181,13 @@ def forward(self, fn:str, request: Request, catch_exception:bool=True) -> dict: if c.is_generator(result): output = [] def generator_wrapper(generator): - for item in generator: - output_item = self.serializer.serialize(item) - output += [output_item] - yield output_item + try: + for item in generator: + output_item = self.serializer.serialize(item) + yield output_item + except Exception as e: + c.print(e) + yield str(c.detailed_error(e)) result = EventSourceResponse(generator_wrapper(result)) else: output = self.serializer.serialize(result) diff --git a/commune/utils/misc.py b/commune/utils/misc.py index 15c0f557d..8bf8292a4 100644 --- a/commune/utils/misc.py +++ b/commune/utils/misc.py @@ -703,7 +703,6 @@ def file2text(path = './', 'node_modules'], relative=True, **kwargs): path = os.path.abspath(os.path.expanduser(path)) - print(path, 'FAMMM') file2text = {} for file in get_glob(path, recursive=True): if os.path.isdir(file): diff --git a/commune/vali.py b/commune/vali.py index c9c42fcf8..dfd6aa2aa 100644 --- a/commune/vali.py +++ b/commune/vali.py @@ -6,52 +6,71 @@ class Vali(c.Module): endpoints = ['score', 'scoreboard'] - voting_networks = ['bittensor', 'commune', 'subspace'] + voting_networks = ['bittensor', 'subspace'] networks = ['local'] + voting_networks + epoch_time = 0 + vote_time = 0 + vote_staleness = 0 + epochs = 0 def __init__(self, network= 'local', # for local subspace:test or test # for testnet subspace:main or main # for mainnet subnet : Optional[Union[str, int]] = None, # (OPTIONAL) the name of the subnetwork search : Optional[str] = None, # (OPTIONAL) the search string for the network - verbose : bool = True, # the verbose mode for the worker # EPOCH batch_size : int = 128, # the batch size of the most parallel tasks max_workers : Optional[int]= None , # the number of parallel workers in the executor score : Union['callable', int]= None, # score function - path= None, # the storage path for the module eval, if not null then the module eval is stored in this directory - min_score= 0, # the minimum weight of the scoreboard - run_loop= True, # This is the key that we need to change to false - test= False, # the test mode for the validator - tempo = None , - timeout= 3, # timeout per evaluation of the module - update=False, # update during the first epoch - key = None, + key : str = None, + path : str= None, # the storage path for the module eval, if not null then the module eval is stored in this directory + tempo : int = None , + timeout : int = 3, # timeout per evaluation of the module + update : bool =False, # update during the first epoch + run_loop : bool = True, # This is the key that we need to change to false **kwargs): + self.set_executor(max_workers=max_workers, batch_size=batch_size, timeout=timeout) + self.set_network(network=network, subnet=subnet, tempo=tempo, search=search, path=path, score=score, update=update) + self.set_key(key) + if run_loop: + c.thread(self.run_loop) + init_vali = __init__ + + def set_executor(self, max_workers:int, batch_size:int, timeout:int): + self.timeout = timeout or 3 + self.max_workers = max_workers or c.cpu_count() * 5 + self.batch_size = batch_size or 128 + self.executor = c.module('executor')(max_workers=self.max_workers, maxsize=self.batch_size) + return {'success': True, 'msg': 'Executor set', 'max_workers': self.max_workers, 'batch_size': self.batch_size, 'timeout': self.timeout} + + def set_network(self, network:str, + subnet:str=None, + tempo:int=60, + search:str=None, + path:str=None, + score = None, + update=False): + + if not network in self.networks and '/' not in network: network = f'subspace/{network}' - self.epochs = 0 - self.max_workers = max_workers or c.cpu_count() * 5 - self.batch_size = batch_size - self.min_score = min_score - self.timeout = timeout - self.test = test - self.verbose = verbose - self.search = search - if callable(score): - setattr(self, 'score', score ) - self.set_key(key) [network, subnet] = network.split('/') if '/' in network else [network, subnet] self.subnet = subnet self.network = network + self.network_module = c.module(self.network)() self.tempo = tempo - self.path = self.resolve_path( path or f'{network}/{subnet}') + self.search = search + self.path = os.path.abspath(path or self.resolve_path(f'{network}/{subnet}')) + self.set_score(score) self.sync(update=update) - if run_loop: - c.thread(self.run_loop) - init_vali = __init__ - + def score(self, module): return int('name' in module.info()) + + def set_score(self, score): + if callable(score): + setattr(self, 'score', score ) + assert callable(self.score), f'SCORE NOT SET {self.score}' + return {'success': True, 'msg': 'Score function set'} @property def is_voting_network(self): @@ -63,101 +82,88 @@ def run_loop(self): self.epoch() except Exception as e: c.print('XXXXXXXXXX EPOCH ERROR ----> XXXXXXXXXX ',c.detailed_error(e), color='red') - - epoch2results = {} - epoch_time = 0 @property def nex_epoch(self): return int(self.epoch_time + self.tempo - c.time()) + + futures = [] + results = [] def epoch(self): + if len(self.futures) > 0: + print('Cancelling futures from previous epoch') + [f.cancel() for f in self.futures] + self.futures = [] + self.results = [] next_epoch = self.nex_epoch progress = c.tqdm(total=next_epoch, desc='Next Epoch') for _ in range(next_epoch): progress.update(1) c.sleep(1) self.sync() - executor = c.module('executor')(max_workers=self.max_workers, maxsize=self.batch_size) - c.print(f'Epoch(network={self.network} epoch={self.epochs} n={self.n})', color='yellow') - futures = [] - results = [] + c.print(f'EPOCH(network={self.network} epoch={self.epochs} n={self.n})', color='yellow') progress = c.tqdm(total=self.n, desc='Evaluating Modules') # return self.modules n = len(self.modules) for i, module in enumerate(self.modules): module["i"] = i - c.print(f'EVAL(i={i}/{n}, name={module["name"]} key={module["key"]})', color='yellow') - if len(futures) < self.batch_size: - futures.append(executor.submit(self.score_module, [module], timeout=self.timeout)) + c.print(f'EVAL(i={i}/{n} key={module["key"]} name={module["name"]})', color='yellow') + if len(self.futures) < self.batch_size: + self.futures.append(self.executor.submit(self.score_module, [module], timeout=self.timeout)) else: - results.append(self.next_result(futures)) + self.results.append(self.next_result()) progress.update(1) - while len(futures) > 0: - results.append(self.next_result(futures)) - results = [r for r in results if r.get('score', 0) > self.min_score] + while len(self.futures) > 0: + self.results.append(self.next_result()) + self.results = [r for r in self.results if r.get('score', 0) > 0] self.epochs += 1 self.epoch_time = c.time() c.print(self.vote()) - return results + return self.results def sync(self, update = False): - - network_path = self.path + '/state' - max_age = self.tempo or 60 - state = c.get(network_path, max_age=max_age, update=update) - # RESOLVE THE VOTING NETWORKS - network_module = c.module(self.network)() - modules = network_module.modules(subnet=self.subnet, max_age=max_age) - params = network_module.params(subnet=self.subnet, max_age=max_age) - self.tempo = self.tempo or (params['tempo'] * network_module.block_time)//2 - self.params = params - state = {'time': c.time(), "params": params, 'modules': modules} + max_age = 0 if update else (self.tempo or 60) + self.modules = self.network_module.modules(subnet=self.subnet, max_age=max_age) + self.params = self.network_module.params(subnet=self.subnet, max_age=max_age) + self.tempo = self.tempo or (self.params['tempo'] * self.network_module.block_time)//2 if self.search != None: - modules = [m for m in modules if self.search in m['name']] - self.network_module = network_module - self.n = len(modules) - self.modules = modules - self.network_info = {'n': self.n, 'network': self.network + '/' + str(self.subnet) if self.network != 'local' else self.network, 'params': params} + self.modules = [m for m in self.modules if self.search in m['name']] + self.n = len(self.modules) + self.network_info = {'n': self.n, 'network': self.network , 'subnet': self.subnet, 'params': self.params} c.print(f' self.min_score: + if module['score'] > 0: module_path = self.path +'/'+ module['key'] c.put_json(module_path, module) return module - - def votes(self, **kwargs): - votes = {'keys': [], 'weights': []} - for module in self.scoreboard().to_records(): - if module['score'] > 0: - votes['keys'] += [module['key']] - votes['weights'] += [module['score'].item()] - return votes @property def votes_path(self): return self.path + f'/votes' - def vote(self, update=False, submit_async=True, **kwargs): + def vote(self, submit_async=True, **kwargs): + if not self.is_voting_network : - return {'msg': f'NETWORK NOT VOTING NETWORK ({self.network}) out of ({self.voting_networks})', 'success': False,} - if not hasattr(self, 'vote_time'): - self.vote_time = 0 - vote_staleness = c.time() - self.vote_time - if not update: - if vote_staleness < self.tempo: - return {'success': False, 'msg': f'Vote is too soon {vote_staleness}'} - votes =self.votes() - return self.network_module.vote(modules=votes['keys'], - weights=votes['weights'], - key=self.key, - subnet=self.subnet) + return {'success': False, 'msg': f'NOT VOTING NETWORK({self.network})'} + self.vote_staleness = c.time() - self.vote_time + if self.vote_staleness < self.tempo: + return {'success': False, 'msg': f'Vote is too soon {self.vote_staleness}'} + fn_obj = self.network_module.vote + params = dict(modules=[], weights=[], key=self.key, subnet=self.subnet) + if len(self.results) == 0: + return {'success': False, 'msg': 'No results to vote on'} + for m in self.results: + params['modules'].append(m['key']) + params['weights'].append(m['score']) + if submit_async: + return c.submit(fn_obj, params) + else: + return self.network_module.vote(**params) set_weights = vote @@ -181,7 +187,7 @@ def scoreboard(self, # chunk the jobs into batches for path in self.module_paths(): r = self.get(path, {}, max_age=max_age) - if isinstance(r, dict) and 'key' and r.get('score', 0) > self.min_score : + if isinstance(r, dict) and 'key' and r.get('score', 0) > 0 : df += [{k: r.get(k, None) for k in keys}] else : self.rm(path) @@ -206,31 +212,23 @@ def module_paths(self): paths = self.ls(self.path) return paths - def refresh_scoreboard(self): - path = self.path - c.rm(path) - return {'success': True, 'msg': 'Leaderboard removed', 'path': path} - @classmethod def run_epoch(cls, network='local', run_loop=False, **kwargs): return cls(network=network, run_loop=run_loop, **kwargs).epoch() - - - - def next_result(self, futures, features=['score', 'name', 'key', 'i']): + def next_result(self, features=['score', 'name', 'key', 'i']): try: - for future in c.as_completed(futures, timeout=self.timeout): - futures.remove(future) + for future in c.as_completed(self.futures, timeout=self.timeout): + self.futures.remove(future) result = future.result() if all([f in result for f in features]): - c.print(f'RESULT(score={result["score"]}, name={result["name"]} key={result["key"]})', color='green') + c.print(f'RESULT(score={result["score"]} key={result["key"]} name={result["name"]} )', color='green') return result except Exception as e: result = c.detailed_error(e) - c.print(f'ERROR({result})', color='red') - + c.print(f'ERROR({result})', color='red') return result + @staticmethod def test( n=1, tag = 'vali_test_net', miner='module', vali='vali', path = '/tmp/commune/vali_test',network='local'): test_miners = [f'{miner}::{tag}_{i}' for i in range(n)] @@ -247,4 +245,7 @@ def test( n=1, tag = 'vali_test_net', miner='module', vali='vali', path = '/t c.print(c.kill(miner)) return {'success': True, 'msg': 'subnet test passed'} - + def refresh_scoreboard(self): + path = self.path + c.rm(path) + return {'success': True, 'msg': 'Leaderboard removed', 'path': path} \ No newline at end of file diff --git a/docs/3_cli_basics.md b/docs/3_cli_basics.md index c6c15439a..e0b8a3c93 100644 --- a/docs/3_cli_basics.md +++ b/docs/3_cli_basics.md @@ -1,13 +1,21 @@ -# The Pythonic CLI +# CLI BASICS We have a pythonic cli for commune, which is a wrapper around the `c.Module` library. This is a simple way to interact with the commune library. This does not need to be formated like argparse, and is more like a pythonic cli, where you can test out the functions and modules. +```bash +c {module_name}/{function_name} *args **kwargs +# positional arguments and keyword arguments are accepted as if you were calling a function in python +``` +so -```bash -c {module_name}/{function_name} *args **kwargs +module = c.module("module") +module.ls("./") +is the same as +```bash +c module/ls ./ ``` ## Pythoni @@ -24,8 +32,9 @@ c ls ./ # ``` is the same as ```bash -c module/ls ./ +c module/ls ./ # calls the ls fn=ls module=module ``` + and ```python import commune as c @@ -157,4 +166,14 @@ c.module("demo").serve(tag="latest") # Misc -c run-epoch is c run_epoch # all - is _ \ No newline at end of file +c run-epoch is c run_epoch # all - is _ + + +to serve a module + + +c serve "model.openai" + +is the same as +c.serve("model.openai") + diff --git a/docs/docs.py b/docs/docs.py index db4bcab75..4fd9335e0 100644 --- a/docs/docs.py +++ b/docs/docs.py @@ -1,16 +1,16 @@ import commune as c +import os class Docs(c.Module): - def forward(self, *question, + def ask(self, *question, model='anthropic/claude-3.5-sonnet', **kwargs ) -> int: question = ' '.join(list(map(str, question))) - context = c.file2text(self.dirpath()) + context = c.file2text(os.path.dirname(__file__) ) prompt = f""" QUESTION: {question} CONTEXT: {context} """ - return c.module('chat')().generate(prompt, model=model,stream=1, **kwargs) - ask = forward \ No newline at end of file + return c.ask(prompt, model=model, **kwargs) \ No newline at end of file diff --git a/scripts/start.sh b/scripts/start.sh new file mode 100755 index 000000000..a0bee6afb --- /dev/null +++ b/scripts/start.sh @@ -0,0 +1,22 @@ + +PWD=$(pwd) +NAME=$(basename $PWD); +REPO_PATH=$(pwd) ; DOCKER_REPO_PATH=/$NAME +CACHE_PATH=~/.$NAME ; DOCKER_CACHE_PATH=/root/.$NAME +SHM_SIZE=4g + +CONTAINER_PARAMS=" -d \ + --name $NAME \ + --shm-size $SHM_SIZE \ + -v $REPO_PATH:$DOCKER_REPO_PATH \ + -v $CACHE_PATH:$DOCKER_CACHE_PATH \ + --network=host \ + --restart unless-stopped \ + --privileged \ + $NAME" + +CONTAINER_EXISTS=$(docker ps -q -f name=$NAME) +if [ $CONTAINER_EXISTS ]; then + ./scripts/stop.sh +fi +eval docker run $CONTAINER_PARAMS \ No newline at end of file diff --git a/setup.py b/setup.py index 9ecad2179..b6e6397a6 100644 --- a/setup.py +++ b/setup.py @@ -3,9 +3,7 @@ from os import path from io import open from pkg_resources import parse_requirements - libname = 'commune' - here = path.abspath(path.dirname(__file__)) with open(f'{here}/README.md', encoding='utf-8') as f: @@ -17,7 +15,7 @@ setup( name=libname, version='0.0.1', - description='A simple CLI tool to help you manage your projects and tasks and connecting all of them together in a simple way', + description='A package for building and deploying modules', long_description=long_description, long_description_content_type='text/markdown', url='https://github.com/commune-ai/commune', @@ -29,7 +27,7 @@ install_requires=install_requires, entry_points={ 'console_scripts': [ - 'c=commune.cli:main' + f'{libname[0]}={libname}.cli:main' ], }, classifiers=[ diff --git a/tests/test_serializer.py b/tests/test_serializer.py deleted file mode 100644 index b2cc290bc..000000000 --- a/tests/test_serializer.py +++ /dev/null @@ -1,31 +0,0 @@ - -import commune as c -def test(): - self = c.module('serializer')() - import torch, time - data_list = [ - torch.ones(1000), - torch.zeros(1000), - torch.rand(1000), - [1,2,3,4,5], - {'a':1, 'b':2, 'c':3}, - 'hello world', - c.df([{'name': 'joe', 'fam': 1}]), - 1, - 1.0, - True, - False, - None - - ] - for data in data_list: - t1 = time.time() - ser_data = self.serialize(data) - des_data = self.deserialize(ser_data) - des_ser_data = self.serialize(des_data) - t2 = time.time() - - latency = t2 - t1 - emoji = '✅' if str(des_ser_data) == str(ser_data) else '❌' - print(type(data),emoji) - return {'msg': 'PASSED test_serialize_deserialize'} diff --git a/tests/test_server.py b/tests/test_server.py index 8d9afd9be..a8e3fa1f2 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -2,6 +2,36 @@ import commune as c +def test(): + self = c.module('serializer')() + import torch, time + data_list = [ + torch.ones(1000), + torch.zeros(1000), + torch.rand(1000), + [1,2,3,4,5], + {'a':1, 'b':2, 'c':3}, + 'hello world', + c.df([{'name': 'joe', 'fam': 1}]), + 1, + 1.0, + True, + False, + None + + ] + for data in data_list: + t1 = time.time() + ser_data = self.serialize(data) + des_data = self.deserialize(ser_data) + des_ser_data = self.serialize(des_data) + t2 = time.time() + + latency = t2 - t1 + emoji = '✅' if str(des_ser_data) == str(ser_data) else '❌' + print(type(data),emoji) + return {'msg': 'PASSED test_serialize_deserialize'} + def test_basics() -> dict: servers = c.servers()