From a955cc5901b4b678110dcb24830e607195723268 Mon Sep 17 00:00:00 2001 From: latentvector Date: Thu, 30 May 2024 09:42:50 -0500 Subject: [PATCH] new validation tests --- commune/module/module.py | 49 +++++++++++++----------------- commune/namespace.py | 24 ++++----------- commune/tests/test_commune.py | 1 + commune/vali/vali.py | 56 ++++++++++++++++++++++++++++++----- subnet/subnet.py | 24 --------------- 5 files changed, 77 insertions(+), 77 deletions(-) diff --git a/commune/module/module.py b/commune/module/module.py index 5f712ce44..86dbd7155 100755 --- a/commune/module/module.py +++ b/commune/module/module.py @@ -2822,33 +2822,22 @@ def kill_prefix(cls, prefix:str, **kwargs): @classmethod - def kill_many(cls, search:str, network='local', parallel=True, timeout=10, n=None, **kwargs): - servers = c.servers(network=network) - servers = [s for s in servers if search in s] - + def kill_many(cls, servers, search:str = None, network='local', parallel=True, timeout=10, **kwargs): - n = n or len(servers) - - if n > 0 and n < 1: - servers = servers[:int(len(servers)*n)] - elif n > 1: - servers = servers[:n] - - assert len(servers) > 0, f'No servers found with search {search}' - if parallel: - futures = [] - for s in servers: - future = c.submit(c.kill, kwargs={'module':s, **kwargs}, mode='thread', return_future = True, timeout=timeout) - futures.append(future) - results = c.wait(futures, timeout=timeout) - else: - results = [] - for s in servers: - results.append(c.kill(s, **kwargs)) + servers = c.servers(network=network) + servers = [s for s in servers if search in s] + futures = [] + for s in servers: + future = c.submit(c.kill, kwargs={'module':s, **kwargs}, imeout=timeout) + futures.append(future) + results = [] + for r in c.as_completed(futures, timeout=timeout): + results += [r.result()] + return results - return {'success':True, 'message':f'Killed servers with prefix {search}', 'results': results} + return delete = kill_server = kill def destroy(self): @@ -5022,7 +5011,7 @@ def is_coroutine(cls, future): def as_completed(cls , futures:list, timeout:int=10, **kwargs): return concurrent.futures.as_completed(futures, timeout=timeout) @classmethod - def wait(cls, futures:list, timeout:int = 30, generator:bool=False, return_dict:bool = True) -> list: + def wait(cls, futures:list, timeout:int = None, generator:bool=False, return_dict:bool = True) -> list: import concurrent.futures @@ -5039,10 +5028,12 @@ def wait(cls, futures:list, timeout:int = 30, generator:bool=False, return_dict: future2idx = {future:i for i,future in enumerate(futures)} - if timeout == None and hasattr(futures[0], 'timeout'): - timeout = futures[0].timeout - - + if timeout == None: + if hasattr(futures[0], 'timeout'): + timeout = futures[0].timeout + else: + timeout = 30 + if generator: def get_results(futures): try: @@ -5070,6 +5061,8 @@ def get_results(futures): unfinished_futures = [future for future in futures if future in future2idx] c.print(f'Error: {e}, {len(unfinished_futures)} unfinished futures with timeout {timeout} seconds') return results + + return get_results(futures) diff --git a/commune/namespace.py b/commune/namespace.py index c6970b0f7..fc13eb7d0 100644 --- a/commune/namespace.py +++ b/commune/namespace.py @@ -17,7 +17,6 @@ class Namespace(c.Module): def namespace(cls, search=None, network:str = 'local', update:bool = False, - public:bool = False, netuid=None, max_age:int = None, **kwargs) -> dict: @@ -26,8 +25,8 @@ def namespace(cls, search=None, network = f'subspace.{netuid}' path = network - - namespace = cls.get(path, None, max_age=max_age) + namespace = cls.get(path, {}, max_age=max_age) + if 'subspace' in network: if '.' in network: @@ -36,32 +35,21 @@ def namespace(cls, search=None, netuid = netuid or 0 if c.is_int(netuid): netuid = int(netuid) - c.print(f'Getting namespace {network} with netuid {netuid}') - - # NOTE: WE INTENTIONALLY DEFER TO THE MAX_AGE OF BEING NONE HERE AND DEFER TO SUBSPACE'S MAX_AGE namespace = c.module(network)().namespace(search=search, update=update, netuid=netuid, **kwargs) elif network == 'local': - if update or namespace == None: + if update or len(namespace) == 0: namespace = cls.build_namespace(network=network) + - namespace = {} if namespace == None else namespace - + namespace = {k:v for k,v in namespace.items() if 'Error' not in k} if search != None: namespace = {k:v for k,v in namespace.items() if search in k} - - namespace = {k:v for k,v in namespace.items() if 'Error' not in k} - - if public: - namespace = {k:v.replace(c.default_ip, c.ip()) for k,v in namespace.items()} if network == 'local': - to_local_ip = lambda x: '0.0.0.0:' + x.split(':')[-1] - namespace = {k:to_local_ip(v) for k,v in namespace.items() } - - namespace = {k:v for k,v in sorted(namespace.items(), key=lambda x: x[0])} + namespace = {k: '0.0.0.0:' + v.split(':')[-1] for k,v in namespace.items() } namespace = dict(sorted(namespace.items(), key=lambda x: x[0])) diff --git a/commune/tests/test_commune.py b/commune/tests/test_commune.py index ef070f5f2..21f022f8b 100644 --- a/commune/tests/test_commune.py +++ b/commune/tests/test_commune.py @@ -9,6 +9,7 @@ def test_server(): c.module('server').test() def test_subnet(): c.module('subnet').test() + diff --git a/commune/vali/vali.py b/commune/vali/vali.py index 963396bc5..6fd9022ba 100644 --- a/commune/vali/vali.py +++ b/commune/vali/vali.py @@ -1,6 +1,8 @@ import commune as c import os +import pandas as pd + from typing import * class Vali(c.Module): @@ -8,7 +10,6 @@ class Vali(c.Module): voting_networks: ['subspace', 'bittensor'] score_fns = ['score_module', 'score'] # the score functions whitelist = ['eval_module', 'score_module', 'eval', 'leaderboard'] - futures = [] def __init__(self, config:dict=None, @@ -39,7 +40,7 @@ def init_vali(self, config=None, module=None, score_fn=None, **kwargs): self.config = config self.init_metrics() self.set_score_fn(score_fn) - + self.futures = [] c.thread(self.run_loop) init = init_vali @@ -455,7 +456,7 @@ def vote_info(self): return info - def calculate_votes(self, df=None): + def calculate_votes(self, df=None, **kwargs): network = self.config.network keys = ['name', 'w', 'staleness','latency', 'ss58_address'] leaderboard = df or self.leaderboard(network=network, @@ -464,7 +465,7 @@ def calculate_votes(self, df=None): c.print(leaderboard) assert len(leaderboard) > 0 votes = {'keys' : [],'weights' : [],'uids': [], 'timestamp' : c.time() } - key2uid = self.subspace.key2uid() if hasattr(self, 'subspace') else {} + key2uid = self.subspace.key2uid(**kwargs) if hasattr(self, 'subspace') else {} for info in leaderboard: ## valid modules have a weight greater than 0 and a valid ss58_address if 'ss58_address' in info and info['w'] >= 0: @@ -509,7 +510,7 @@ def leaderboard(self, max_age = None, network = None, ascending = True, - sort_by = ['w'], + by = 'w', to_dict = False, n = None, page = None, @@ -529,11 +530,16 @@ def leaderboard(self, else : # removing the path as it is not a valid module and is too old self.rm(path) + df = c.df(df) + if len(df) == 0: return c.df(df) - - df = df.sort_values(by=sort_by, ascending=ascending) + + + if isinstance(by, str): + by = [by] + df = df.sort_values(by=by, ascending=ascending) if n != None: if page != None: @@ -625,5 +631,41 @@ def check_peers(cls): result = c.call(peer+'/run_info') c.print(result) + @classmethod + def test(cls, n=3, sleep_time=5, + miner='miner', + vali='vali', + network='local'): + + test_miners = [f'{miner}::test_{i}' for i in range(n)] + test_vali = f'{vali}::test' + modules = test_miners + [test_vali] + for m in modules: + c.kill(m) + + for m in test_miners + [test_vali]: + if m == test_vali: + c.print(c.serve(m, kwargs={'network': network, 'search': test_miners[0].split('::')[0]})) + else: + c.print(c.serve(m)) + while not c.server_exists(test_vali): + c.sleep(1) + c.print(f'Waiting for {test_vali} to start') + c.print(c.get_namespace()) + + c.print(f'Sleeping for {sleep_time} seconds') + c.print(c.call(test_vali+'/refresh_leaderboard')) + c.sleep(sleep_time) + + leaderboard = c.call(test_vali+'/leaderboard') + assert isinstance(leaderboard, pd.DataFrame), leaderboard + assert len(leaderboard) >= n, leaderboard + c.print(c.call(test_vali+'/refresh_leaderboard')) + + c.print(leaderboard) + + for miner in test_miners + [test_vali]: + c.print(c.kill(miner)) + return {'success': True, 'msg': 'subnet test passed'} Vali.run(__name__) diff --git a/subnet/subnet.py b/subnet/subnet.py index 0d5c16c4f..ed4ffff9d 100644 --- a/subnet/subnet.py +++ b/subnet/subnet.py @@ -12,27 +12,3 @@ def score_module(self, module): else: return 0 - - @classmethod - def test(cls, n=3, sleep_time=4): - test_miners = [f'subnet.miner::test_{i}' for i in range(n)] - for miner in test_miners: - c.print(c.serve(miner)) - - test_vali = 'subnet.vali::test' - c.serve(test_vali, kwargs={'network': 'local', 'search': 'miner::test_'}) - - c.print(f'Sleeping for {sleep_time} seconds') - c.sleep(sleep_time) - - leaderboard = c.call(test_vali+'/leaderboard') - assert isinstance(leaderboard, pd.DataFrame), leaderboard - assert len(leaderboard) == n, leaderboard - c.print(c.call(test_vali+'/refresh_leaderboard')) - - c.print(leaderboard) - - c.serve('subnet.miner::test') - for miner in test_miners + [test_vali]: - c.print(c.kill(miner)) - return {'success': True, 'msg': 'subnet test passed'} \ No newline at end of file