Skip to content

Commit

Permalink
vali
Browse files Browse the repository at this point in the history
  • Loading branch information
latentvector committed Apr 22, 2024
1 parent 74fca47 commit 2bdab2f
Show file tree
Hide file tree
Showing 6 changed files with 144 additions and 111 deletions.
12 changes: 7 additions & 5 deletions commune/module/module.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6234,9 +6234,9 @@ def ss58_decode(*args, **kwargs):

@classmethod
def fn2str(cls,search = None, code = True, defaults = True, **kwargs):
schema = cls.schema(search=search, defaults=defaults)
fns = cls.fns(search=search)
fn2str = {}
for fn in schema.keys():
for fn in fns:
fn2str[fn] = cls.fn_code(fn)

return fn2str
Expand Down Expand Up @@ -8096,15 +8096,17 @@ def launcher_keys(cls):
return [k for k in keys if k.startswith('module::')]

@classmethod
def load_launcher_keys(cls, amount=400, **kwargs):
def load_launcher_keys(cls, amount=600, **kwargs):
launcher_keys = cls.launcher_keys()
key2address = c.key2address()
destinations = []
amounts = []
launcher2balance = c.get_balances(launcher_keys)
for k in launcher_keys:
amount_needed = amount - launcher2balance.get(k, 0)
k_address = key2address[k]
amount_needed = amount - launcher2balance.get(k_address, 0)
if amount_needed > 0:
destinations.append(k)
destinations.append(k_address)
amounts.append(amount_needed)
else:
c.print(f'{k} has enough balance --> {launcher2balance.get(k, 0)}')
Expand Down
119 changes: 68 additions & 51 deletions commune/remote/app.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,13 @@ class App(c.Module):
def __init__(self, **kwargs):
self.set_config(kwargs=kwargs)
self.remote = c.module('remote')()
c.thread(self.run_loop)


def run_loop(self):
c.new_event_loop()
while True:
st.write('Running')

@classmethod
def app(cls, module: str = None, **kwargs):
Expand Down Expand Up @@ -53,22 +60,37 @@ def manage_hosts_dashboard(self):
with st.expander('Add Host', expanded=False):
st.markdown('## Hosts')
cols = st.columns(2)

host = cols[0].text_input('Host', '0.0.0.0')
port = cols[1].number_input('Port', 22, 30000000000, 22)
user = st.text_input('User', 'root')
pwd = st.text_input('Password', type='password')

cols = st.columns(2)
user = cols[0].text_input('User', 'root')
pwd = cols[1].text_input('Password', type='password')
name = cols[0].text_input('Name', user+'@'+host)
add_host = st.button('Add Host')

if add_host:
self.remote.add_host(host=host, port=port, user=user, pwd=pwd)

with st.expander('Remove Host', expanded=False):
host_names = list(self.remote.hosts().keys())
rm_host_name = st.selectbox('Host Name', host_names)
rm_host_name = st.selectbox('Host to Remove', host_names)
rm_host = st.button('Remove Host')
if rm_host:
self.remote.rm_host(rm_host_name)

with st.expander('Rename Host', expanded=False):
host_names = list(self.remote.hosts().keys())
rm_host_name = st.selectbox('Host Name', host_names)
new_host_name = st.text_input('New Host Name')
rename_host = st.button('Rename Host')
if rename_host:
host = self.remote.hosts()[rm_host_name]
self.remote.add_host(host)

self.remote.rm_host(rm_host_name)

self.host2ssh_search()


Expand Down Expand Up @@ -111,78 +133,80 @@ def ssh_dashboard(self):
# make this a stop button red

stop_button = cols[1].button('Stop')
host2future = {}

host2stats = self.get('host2stats', {})

future2host = {}
if run_button and not stop_button:
for host in host_names:
future = c.submit(self.remote.ssh_cmd, args=[cmd], kwargs=dict(host=host, verbose=False, sudo=sudo, search=host_names, cwd=cwd), return_future=True, timeout=timeout)
host2future[host] = future
future2host[future] = host
host2stats[host] = host2stats.get(host, {'success': 0, 'error': 0 })


futures = list(host2future.values())
hosts = list(host2future.keys())
cols = st.columns(num_columns)
failed_hosts = []
col_idx = 0

errors = []
futures = list(future2host.keys())

try:
for result in c.wait(futures, timeout=timeout, generator=True, return_dict=True):
success_expander = st.expander('Results', expanded=expanded)
failed_expander = st.expander('Failed', expanded=expanded)

try:
for future in c.as_completed(futures, timeout=timeout):

host = hosts[result['idx']]
if host == None:
continue

host2future.pop(host)
result = result['result']
host = future2host.pop(future)
stats = host2stats.get(host, {'success': 0, 'error': 0})
result = future.result()
is_error = c.is_error(result)
msg = result['error'] if is_error else result.strip()
if is_error:
with failed_expander:
st.error(f'Error: {result}')

else:

# get the colkumne
col_idx = (col_idx) % len(cols)
col = cols[col_idx]
col_idx += 1
with success_expander:

msg = result if is_error else result.strip()

# get the colkumne
col_idx = (col_idx) % len(cols)
col = cols[col_idx]
col_idx += 1

stats = host2stats[host]



stats = host2stats.get(host, {'success': 0, 'error': 0})

# if the column is full, add a new column
with col:
msg = fn_code(msg)
emoji = c.emoji("cross") if is_error else c.emoji("check")
title = f'{emoji} :: {host} :: {emoji}'

if is_error:
failed_hosts += [host]
errors += [msg]
stats['error'] += 1

else:
stats['last_success'] = c.time()
stats['success'] += 1
with st.expander(title, expanded=expanded):
st.code(msg)


# if the column is full, add a new column
with col:
msg = fn_code(msg)
emoji = c.emoji("cross") if is_error else c.emoji("check")
title = f'{emoji} :: {host} :: {emoji}'


host2stats[host] = stats

if is_error:
failed_hosts += [host]
errors += [msg]
stats['error'] += 1

else:
stats['last_success'] = c.time()
stats['success'] += 1
st.write(title)
st.code(msg)

host2stats[host] = stats


except Exception as e:
pending_hosts = list(host2future.keys())
pending_hosts = list(future2host.values())
st.error(c.detailed_error(e))
st.error(f"Hosts {pending_hosts} timed out")
failed_hosts += pending_hosts

for host in pending_hosts:
stats = host2stats[host]
stats['error'] += 1
Expand All @@ -198,19 +222,12 @@ def ssh_dashboard(self):
if delete_failed:
for host in selected_failed_hosts:
self.remote.rm_host(host)

for host, error in zip(failed_hosts, errors):

st.write(f'**{host}**')
st.code(error)


def host_stats_page(self):
host2stats = self.get('host2stats', {})
host2stats = {k:v for k,v in host2stats.items() if v['success'] + v['error'] > 0}
df = pd.DataFrame(host2stats).T
st.write(df)


def ssh_params(self):

with st.expander('params', False):
Expand Down
84 changes: 49 additions & 35 deletions commune/remote/remote.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,15 @@ class Remote(c.Module):
executable_path='commune/bin/c'
@classmethod
def ssh_cmd(cls, *cmd_args,

cmd : str = None,
port = None,
user = None,
password = None,
host:str= None,
cwd:str=None,
verbose=False,
sudo=False,
stream = False,
key=None,
timeout=10,
key_policy = 'auto_add_policy',
Expand All @@ -33,24 +34,24 @@ def ssh_cmd(cls, *cmd_args,
:param command: Command to be executed on the remote machine.
:return: Command output.
"""
command = ' '.join(cmd_args).strip()


if host == None:
host = {
'host': host,
'port': port,
'user': user,
'pwd': password,
}
if port == None or user == None or password == None:
host = list(cls.hosts().values())[0]
else:
host = {
'host': host,
'port': port,
'user': user,
'pwd': password,
}
else:
host = cls.hosts().get(host, None)


host['name'] = f'{host["user"]}@{host["host"]}:{host["port"]}'


c.print(f'Running command: {command} on {host["name"]}')

# Create an Remote client instance.
client = paramiko.SSHClient()
Expand All @@ -66,72 +67,85 @@ def ssh_cmd(cls, *cmd_args,
port=host['port'],
username=host['user'],
password=host['pwd'])



# THE COMMAND

command = ' '.join(cmd_args).strip() if cmd == None else cmd
if cwd != None:
command = f'cd {cwd} && {command}'

# Execute the command on the remote server
if sudo and host['user'] != "root":
command = "sudo -S -p '' %s" % command

c.print(f'Running command: {command} on {host["name"]}')

stdin, stdout, stderr = client.exec_command(command)

try:
if sudo:
stdin.write(host['pwd'] + "\n")
stdin.flush()

stdin.write(host['pwd'] + "\n") # Send the password for sudo commands
stdin.flush() # Send the password

color = c.random_color()
# Print the output of ls command
outputs = {'error': '', 'output': ''}

for line in stdout.readlines():
if verbose:
c.print(f'[bold]{host["name"]}[/bold]', line.strip('\n'), color=color)
outputs['output'] += line

for line in stderr.readlines():
if verbose:
c.print(f'[bold]{host["name"]}[/bold]', line.strip('\n'))
outputs['error'] += line

if len(outputs['error']) == 0:
outputs = outputs['output']
def print_output():
for line in stdout.readlines():
if verbose:
c.print(f'[bold]{host["name"]}[/bold]', line.strip('\n'), color=color)
yield line

for line in stderr.readlines():
if verbose:
c.print(f'[bold]{host["name"]}[/bold]', line.strip('\n'))
yield line

if stream:
return print_output()

else:
output = ''
for line in print_output():
output += line

# stdin.close()
# stdout.close()
# stderr.close()
# client.close()
except Exception as e:
c.print(e)
return outputs

return output

@classmethod
def add_host(cls,
cmd:str = None , # in the format of
host:str = '0.0.0.0',
port:int = 22,
user:str = 'root',
pwd:str = None,
password:str = None,
name : str = None

):

hosts = cls.hosts()
host = {
'host': host,
'port': port,
'user': user,
'pwd': pwd
'host': host, # IP address of the remote machine
'port': port, # Remote port (typically 22)
'user': user, # Remote username
'pwd': pwd or password # Remote password
}

if name == None:
#
cnt = 0
name = f'{user}{cnt}'

while name in hosts:
name = f'{user}{cnt}'
cnt += 1

name = f'{user}{cnt}'
hosts[name] = host
cls.save_hosts(hosts)

Expand Down
Loading

0 comments on commit 2bdab2f

Please sign in to comment.