diff --git a/README.md b/README.md index 5aff9a320..27220905f 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,29 @@ This repo is on the cutting edge of experimentation, so there may be some hiccup ## Module Filesystem -The `module.py` file serves as an anchor, organizing future modules in what we call a module filesystem. For example, you can store a dataset module in `{PWD}/dataset/text`, which will have a path of `dataset.text`. The current limitation is to have a config where the name of the config is that of the Python object. +The `module.py` file serves as an anchor, organizing future modules in what we call a module filesystem. The module filesystem is a mix of the commune's core modules with your local modules with respect to your PWD (parent working directory). This allows you to create a module in your local directory and have it be recognized by the commune. For instance if I have a file called `model.py` in my local directory, with the following code: + +```python + +import commune as c +class Example(c.Module): + def __init__(self): + pass + + def predict(self, x): + return x + 1 +``` + +I can call it from the commune cli by running the following command: + +```bash +c model/predict x=1 +``` +or +```python +c.call('model/predict', x=1) +``` + ## Subspace @@ -166,3 +188,17 @@ c register my_module_path tag=1 - Where can i find futher documentation? This repository folder, [Doc](https://github.com/commune-ai/commune/tree/main/docs). - Can I install on Windows? Yes, [Guide](https://github.com/OmnipotentLabs/communeaisetup). - Can I contribute? Absolutely! We are open to all contributions. Please feel free to submit a pull request. + + +## Testing + +We use pytest for testing. To run the tests, simply run the following command: + +Make sure you install pytest +```bash +pip install pytest +``` + +```bash +pytest commune +``` \ No newline at end of file diff --git a/api.py b/api.py new file mode 100644 index 000000000..45d3e630a --- /dev/null +++ b/api.py @@ -0,0 +1,254 @@ +import commune as c +from flask import Flask, jsonify, request +import random +from datetime import datetime, timedelta +import pandas as pd +import numpy as np +from flask_cors import CORS + +app = Flask(__name__) + + +CORS(app, origins='*') + +# Function to generate dummy data for ID, email, and date +def generate_dummy_data(): + unique_id = '#' + ''.join(random.choices('0123456789', k=7)) + + email = ''.join(random.choices('abcdefghijklmnopqrstuvwxyz', k=10)) + '@example.com' + + start_date = datetime.now() - timedelta(days=30) + end_date = datetime.now() + random_date = start_date + (end_date - start_date) * random.random() + formatted_date = random_date.strftime('%d %b, %Y') + + return unique_id, email, formatted_date + +# Endpoint to get the list of modules +@app.route('/modules', methods=['GET']) +def get_modules(): + modules = c.modules() + + # Generate dummy data for each module + formatted_modules = [] + for module in modules: + unique_id, email, formatted_date = generate_dummy_data() + formatted_module = { + "checkbox": "", + "id": unique_id, + "Name": module, + "Email": email, + "Date": formatted_date, + "status": "completed", + "trash": "" + } + formatted_modules.append(formatted_module) + + return jsonify(formatted_modules) + +# Endpoint to get metadata for a particular module +@app.route('/modules/metadata', methods=['GET']) +def get_module_metadata(): + # Retrieve module name from query parameter + module_name = request.args.get('module_name') + + if not module_name: + return jsonify({'error': 'Module name not provided in query parameter'}), 400 + + + module_metadata = {} + + try: + module = c.module(module_name) + # module_metadata['code'] = module.code() + module_metadata['config'] = module.config() + module_metadata['functions'] = module.fns() + module_metadata['schema'] = module.schema() + except Exception as e: + return jsonify({'error': str(e)}), 404 + + return jsonify(module_metadata) + +@app.route('/modules/trees', methods=['GET']) +def list_modules(): + module_name = request.args.get('module_name', None) + module = c.module() + + try: + if module_name: + modules_tree = module.tree(module_name) + else: + modules_tree = module.tree() + return jsonify(modules_tree) + except Exception as e: + return jsonify({'error': str(e)}), 500 + + +@app.route('/modules/keys', methods=['GET']) +def module_keys(): + module_name = request.args.get('module_name', None) + + try: + if module_name: + keys = c.module(module_name).keys() + else: + keys = c.keys() + return jsonify(keys) + except Exception as e: + return jsonify({'error': str(e)}), 500 + + +@app.route('/modules/active_thread_count', methods=['GET']) +def module_active_thread_count(): + module_name = request.args.get('module_name') + if not module_name: + return jsonify({'error': 'Module name not provided in query parameter'}), 400 + + try: + active_thread_count = c.module(module_name).active_thread_count() + return jsonify({"active_thread_count": active_thread_count}) + except Exception as e: + return jsonify({'error': str(e)}), 404 + + +@app.route('/modules/users', methods=['GET']) +def module_users(): # may need to be checked + module_name = request.args.get('module_name') + if not module_name: + return jsonify({'error': 'Module name not provided in query parameter'}), 400 + + try: + users = c.module(module_name).users() + return jsonify(users) + except Exception as e: + return jsonify({'error': str(e)}), 404 + +@app.route('/modules/namespaces', methods=['GET']) +def namespaces(): + module_name = request.args.get('module_name', None) + module = c.module() + try: + if module_name: + namespaces_list = module.namespace(module_name) + else: + namespaces_list = module.namespace() + return jsonify(namespaces_list) + except Exception as e: + return jsonify({'error': str(e)}), 500 + +# returns list of modules being ACTIVELY served +@app.route('/modules/servers', methods=['GET']) +def list_servers(): + module_name = request.args.get('module_name') + + try: + if module_name: + served_modules = c.servers(module_name) + else: + served_modules = c.servers() + return jsonify(served_modules) + except Exception as e: + return jsonify({'error': str(e)}), 500 + + +@app.route('/modules/info', methods=['GET']) +def get_module_info(): + module_name = request.args.get('module_name') + + if not module_name: + return jsonify({'error': 'Module name not provided in query parameter'}), 400 + + try: + module = c.module(module_name) # Adjusted this line + module_info = module.info({}) + return jsonify(module_info) + except Exception as e: + return jsonify({'error': str(e)}), 404 + + +@app.route('/modules/stats', methods=['GET']) +def module_stats(): + module_name = request.args.get('module_name') + module = c.module() # Assuming c.module() is defined elsewhere and works correctly. + + if not module_name: + return jsonify({'error': 'Module name not provided in query parameter'}), 400 + + try: + if module_name: + modules_tree = module.stats(module_name) + else: + modules_tree = module.stats() + + # Check if the result is a DataFrame and convert + if isinstance(modules_tree, pd.DataFrame): + # Convert DataFrame to a dictionary in a JSON-friendly format + modules_tree = modules_tree.to_dict(orient='records') # 'records' format means list of dicts + + return jsonify(modules_tree) + except Exception as e: + return jsonify({'error': str(e)}), 500 + +@app.route('/info', methods=['GET']) +def infos(): + module_name = request.args.get('module_name') + module = c.module() + infos = {} + if not module_name: + infos['total_module'] = len(c.modules()) + infos['total_user'] = random.randrange(start=1,stop=1000) + infos['module_pending'] = random.randrange(start=1,stop=100) + infos['review_request'] = random.randrange(start=1,stop=100) + + return jsonify(infos) + else : + return jsonify({"error":""}), 500 + +@app.route('/modules/history', methods=['GET']) +def history(): + module_name = request.args.get('module_name') + history_array = np.random.randint(60,101, size=7) + + + if not module_name: + history = {'history': history_array.tolist()} + + return jsonify(history) + else : + return jsonify({"error":""}), 500 + +@app.route('/servers/analytics', methods=['GET']) +def server_analytics(): + module_name = request.args.get('module_name') + history_array = np.random.randint(60,101, size=7) + servers = c.servers() + if not module_name: + server_anal = {'history': history_array.tolist()} + + return jsonify(server_anal) + else : + return jsonify({"error":""}), 500 + + +# TODO: add routes to consume PM2 data + + +if __name__ == '__main__': + app.run(host='0.0.0.0', port=5000, debug=True) + + +# c tree # list of modules with their location on the filesystem + +# c modules.fintest info # info abt a module + +# c modules.fintest keys # keys associated with a module + +# c modules.fintest stats # stats abt a module + +# c modules.fintest users + +# c modules.fintest active_thread_count + +# c namespace model.openai.free or c namespace + +# c servers & TODO: c server history (c servers gives list of modules being actively served) \ No newline at end of file diff --git a/commune/cli.py b/commune/cli.py index d6dcef746..8f35b98b0 100644 --- a/commune/cli.py +++ b/commune/cli.py @@ -70,8 +70,8 @@ def get_output(self, args): if callable(fn): output = fn(*args, **kwargs) - return output + @classmethod diff --git a/commune/module/module.py b/commune/module/module.py index 45ceeb34d..8b55e9112 100755 --- a/commune/module/module.py +++ b/commune/module/module.py @@ -365,6 +365,7 @@ def fn_code(cls,fn:str, @classmethod def sandbox(cls): c.cmd(f'python3 {c.libpath}/sandbox.py', verbose=True) + return sand = sandbox @classmethod @@ -3969,7 +3970,7 @@ def test_fns(cls, *args, **kwargs): return [f for f in cls.functions(*args, **kwargs) if f.startswith('test_')] @classmethod - def test(cls, module=None, timeout=60): + def test(cls, module=None, timeout=60, trials=3): module = module or cls.module_path() if c.module_exists(module + '.test'): c.print('FOUND TEST MODULE', color='yellow') @@ -3983,16 +3984,13 @@ def test(cls, module=None, timeout=60): future2fn[f] = fn fn2result = {} for f in c.as_completed(future2fn, timeout=timeout): - fn = future2fn[f] result = f.result() c.print(f'{fn} result: {result}') + assert result['success'], f'{fn} failed, {result}' fn2result[fn] = result return fn2result - - - ### TIME LAND ### @classmethod diff --git a/commune/os.py b/commune/os.py index 01fd626b7..b013651a8 100644 --- a/commune/os.py +++ b/commune/os.py @@ -574,6 +574,11 @@ def mv(cls, path1, path2): assert os.path.exists(path2), path2 assert not os.path.exists(path1), path1 return {'success': True, 'msg': f'Moved {path1} to {path2}'} + + @classmethod + def sys_path(cls): + import sys + return sys.path \ No newline at end of file diff --git a/commune/server/server.py b/commune/server/server.py index 7055a00b6..bd73a1b7b 100644 --- a/commune/server/server.py +++ b/commune/server/server.py @@ -332,6 +332,9 @@ def resolve_function_access(module): def wait_for_server(self, timeout=10): return c.wait_for_server(self.name, timeout=timeout) + + + diff --git a/commune/template.py b/commune/template/template.py similarity index 99% rename from commune/template.py rename to commune/template/template.py index d62152fb4..f21b5a71c 100644 --- a/commune/template.py +++ b/commune/template/template.py @@ -8,8 +8,6 @@ def __init__(self, obj=None, ): self.set_fn(obj) else: self.set_module(obj) - - def set_module(self, module): if module == None: return None diff --git a/commune/tests/test_commune.py b/commune/tests/test_commune.py index f7d199add..ef070f5f2 100644 --- a/commune/tests/test_commune.py +++ b/commune/tests/test_commune.py @@ -1,16 +1,14 @@ import pytest import commune as c - - def test_key(): c.module('key').test() - def test_namespace(): c.module('namespace').test() - def test_server(): c.module('server').test() - def test_subnet(): c.module('subnet').test() + + + diff --git a/commune/tree/tree.py b/commune/tree/tree.py index 53e61be7f..84842ba88 100644 --- a/commune/tree/tree.py +++ b/commune/tree/tree.py @@ -60,8 +60,9 @@ def tree(cls, tree = None, max_age = None, **kwargs ) -> List[str]: - tree = cls.resolve_tree(tree) + tree_path = cls.resolve_path(tree) module_tree = {} + tree = cls.resolve_tree(tree) path = cls.resolve_path(f'{tree}/tree') max_age = 0 if update else max_age module_tree = c.get(path, {}, max_age=max_age) diff --git a/example_model.py b/example_model.py new file mode 100644 index 000000000..d8ce87c4a --- /dev/null +++ b/example_model.py @@ -0,0 +1,10 @@ +import commune as c + +# after installing commune call me +# c example_model/predict a=10 b=10 +# c example_model/predict 10 10 + +class Example(c.Module): + def predict(self, a: int = 10, b: int = 10): + return a + b + \ No newline at end of file diff --git a/modules/streamlit/plot.py b/modules/streamlit/plot.py index ac211076f..bab7e26b7 100644 --- a/modules/streamlit/plot.py +++ b/modules/streamlit/plot.py @@ -38,7 +38,7 @@ def run(self, data, plots=None, default_plot ='histogram', title=None ): plot = plots[0] form = st.form(F'Params for {plot}') with form: - fig = getattr(self, 'st_plot_'+ plot)(df) + fig = getattr(self, 'plot_'+ plot)(df) form.form_submit_button("Render") else: raise NotImplementedError(f'Broooooo, hold on, you can only use the following {supported_types}') @@ -83,7 +83,7 @@ def plot_scatter2D(self, df=None): - def st_plot_scatter3D(self, df=None): + def plot_scatter3D(self, df=None): df = df if isinstance(df, pd.DataFrame) else self.df column_options = list(df.columns) @@ -107,7 +107,7 @@ def st_plot_scatter3D(self, df=None): return fig - def st_plot_box(self, df=None): + def plot_box(self, df=None): df = df if isinstance(df, pd.DataFrame) else self.df @@ -134,7 +134,7 @@ def st_plot_box(self, df=None): fig.update_layout(width=self.width, height=self.height, font_size=20) return fig - def st_plot_bar(self, df=None): + def plot_bar(self, df=None): df = df if isinstance(df, pd.DataFrame) else self.df column_options = list(df.columns) @@ -162,7 +162,7 @@ def st_plot_bar(self, df=None): - def st_plot_histogram(self, df=None): + def plot_histogram(self, df=None): df = df if isinstance(df, pd.DataFrame) else self.df column_options = list(df.columns) @@ -189,7 +189,7 @@ def st_plot_histogram(self, df=None): return fig - def st_plot_heatmap(cls, df=None): + def plot_heatmap(cls, df=None): df = df if isinstance(df, pd.DataFrame) else self.df column_options = list(df.columns) diff --git a/modules/streamlit/streamlit.py b/modules/streamlit/streamlit.py index 5e2dd65b0..a0f651c24 100755 --- a/modules/streamlit/streamlit.py +++ b/modules/streamlit/streamlit.py @@ -22,7 +22,7 @@ class StreamlitModule(c.Module): def streamlit_functions(self): return [fn for fn in dir(self) if fn.startswith('st_')] - + def run(self, data, plots=None, default_plot ='histogram', title=None ): self.cols= st.columns([1,3]) plots = plots or self.plot_options() diff --git a/sandbox.py b/sandbox.py new file mode 100644 index 000000000..5328ee55a --- /dev/null +++ b/sandbox.py @@ -0,0 +1 @@ +print('fam') \ No newline at end of file diff --git a/subnet/subnet.md b/subnet/subnet.md index dfe897b2a..35c1a3c3a 100644 --- a/subnet/subnet.md +++ b/subnet/subnet.md @@ -1,63 +1,65 @@ -Subnet Tutorial +# Subnet Tutorial -In this tutorial, you will learn how to deploy a subnet on the network and perform various tasks related to staking, registration, and validation. +In this tutorial, you'll learn how to deploy a subnet on the network and perform various tasks related to staking, registration, and validation. -What is a subnet, you ask? A subnet is a group of validators that validate miners on the network. +### What is a Subnet? -The subnet we are going to do is called "subnet", but you can name it whatever you want, by renaming the subnet/subnet.py file to {subnet_name}{subnet_name}.py. - -Each validator has a score function - -Your score function will be used to determine the weight of your vote in the network. -To define the score function, you need to define a function called "score" in your vali.py file. +A subnet is a group of validators that validate miners on the network. The subnet we'll create is called "subnet", but you can name it whatever you want by renaming the `subnet/subnet.py` file to `{subnet_name}{subnet_name}.py`. +Each validator has a score function. This function is used to determine the weight of your vote in the network. To define the score function, you need to define a function called `score` in your `vali.py` file. ## Important Validator Functions -score_module +### score_module The validator runs a thread executor over the namespace of the subnet, where miners are chosen to be validated. The validator will then run the score function on each miner in the namespace, and the miner with the highest score will be chosen to be validated. -filter_module - -If you want to filter the namespace of the subnet, you can define a function called "filter_module" in your vali.py file. This filter function will be used to filter the namespace of the subnet, and only the miners that pass the filter will be validated. - - -Important Miner Functions - -forward - -The forward function is used to forward the miner to the subnet. The miner will be sent to the subnet, and the subnet will validate the miner. This can be a different funciton if the validator calls a different function to validate the miner. It is important to understand the function and inputs/outputs of the function being called on the validator. +### filter_module +If you want to filter the namespace of the subnet, you can define a function called `filter_module` in your `vali.py` file. This filter function will be used to filter the namespace of the subnet, and only the miners that pass the filter will be validated. -def filter_module(module_name:str): +```python +def filter_module(module_name: str): if module_name.startswith("miner"): return True return False +``` + +## Important Miner Functions -Start a Subnet +### forward -Serve the validator on the local network +The forward function is used to forward the miner to the subnet. The miner will be sent to the subnet, and the subnet will validate the miner. This can be a different function if the validator calls a different function to validate the miner. It is important to understand the function and inputs/outputs of the function being called on the validator. -c.serve("subnet.vali::test", network='local') +## Starting a Subnet -Serve a the miners on the local network (default, so you don't need to specify the network) +1. **Serve the validator on the local network:** -c.serve("subnet.miner::test_1") -c.serve("subnet.miner::test_2") + ```python + c.serve("subnet.vali::test", network='local') + ``` +2. **Serve the miners on the local network (default, so you don't need to specify the network):** -Leaderboard + ```python + c.serve("subnet.miner::test_1") + c.serve("subnet.miner::test_2") + ``` + +## Leaderboard To check the leaderboard of the subnet, use the following command: ```python - c.call("subnet.vali/leaderboard") ``` - name w staleness latency -0 subnet.miner::test_0 1.0 10.957290 0.015491 -1 subnet.miner::test_1 1.0 11.076705 0.026521 -2 subnet.miner::test_2 1.0 11.035383 0.012495 +Example output: + +| name | w | staleness | latency | +|-----------------------|-----|-----------|----------| +| subnet.miner::test_0 | 1.0 | 10.957290 | 0.015491 | +| subnet.miner::test_1 | 1.0 | 11.076705 | 0.026521 | +| subnet.miner::test_2 | 1.0 | 11.035383 | 0.012495 | +By following these steps, you can set up and manage a subnet, ensuring your validators and miners are properly configured and performing as expected. Happy staking and validating! \ No newline at end of file diff --git a/subnet/subnet.py b/subnet/subnet.py index 1a55814d1..94f137a16 100644 --- a/subnet/subnet.py +++ b/subnet/subnet.py @@ -1,4 +1,6 @@ import commune as c +import pandas as pd + class Subnet(c.Module): def score_module(self, module): @@ -12,7 +14,8 @@ def score_module(self, module): return 0 - def test(self, n=3, sleep_time=3): + @classmethod + def test(cls, n=3, sleep_time=1): test_vali = 'subnet.vali::test' test_miners = [f'subnet.miner::test_{i}' for i in range(n)] for miner in test_miners: @@ -23,6 +26,8 @@ def test(self, n=3, sleep_time=3): c.sleep(sleep_time) leaderboard = c.call(test_vali+'/leaderboard') + assert isinstance(leaderboard, pd.DataFrame), leaderboard + assert len(leaderboard) == n, leaderboard diff --git a/subnet/vali.py b/subnet/vali.py index 2837aa483..50bf87d79 100644 --- a/subnet/vali.py +++ b/subnet/vali.py @@ -1,7 +1,7 @@ import commune as c class Subnet(c.m('vali')): - def __init__(self, network='local', search='subnet', **kwargs): + def __init__(self, network='local', search='add', **kwargs): self.init_vali(locals()) def score_module(self, module): @@ -14,10 +14,10 @@ def score_module(self, module): return 0 def test(self, n=3, sleep_time=3): - test_miners = ['subnet.miner::test' for i in range(n)] + test_miners = ['subnet.miner.add::test' for i in range(n)] for miner in test_miners: c.serve(miner) - test_vali = 'subnet.vali::test' + test_vali = 'subnet.vali.add::test' for miner in test_miners: c.serve(test_vali, kwargs={'network': 'local'}) diff --git a/yo.py b/yo.py deleted file mode 100644 index 7070ff8e4..000000000 --- a/yo.py +++ /dev/null @@ -1,5 +0,0 @@ -import commune as c - -class Yo(c.Module): - def call(self, a): - return a \ No newline at end of file