Merge with hummingbot/development

Author: Nicolas Baum

.coveragerc

@@ -26,6 +26,7 @@ omit =
         hummingbot/connector/exchange/paper_trade*
         hummingbot/connector/exchange/probit*
         hummingbot/connector/gateway*
+        hummingbot/connector/test_support/*
         hummingbot/core/utils/gateway_config_utils.py
         hummingbot/core/utils/kill_switch.py
         hummingbot/core/utils/wallet_setup.py
@@ -42,6 +43,9 @@ branch = true
 fail_under = 70
 precision = 2
 skip_empty = true
+exclude_lines =
+    @abstractmethod
+    @abc.abstractmethod
 
 [html]
 directory = coverage_html_report

Makefile

@@ -10,7 +10,12 @@
 .PHONY: build
 
 test:
-	coverage run -m nose --exclude-dir="test/connector" --exclude-dir="test/debug" --exclude-dir="test/mock" --exclude-dir="test/hummingbot/connector/gateway"
+	coverage run -m nose \
+ 	--exclude-dir="test/connector" \
+ 	--exclude-dir="test/debug" \
+ 	--exclude-dir="test/mock" \
+ 	--exclude-dir="test/hummingbot/connector/gateway/amm" \
+ 	--exclude-dir="test/hummingbot/connector/gateway/clob"
 
 run_coverage: test
 	coverage report

bin/docker_connection.py

@@ -1,16 +1,13 @@
-import aioprocessing
 import asyncio
-import docker
 import logging
-from multiprocessing import Process
 import types
-from typing import Callable, Dict, Any, List, Generator, Union
+from multiprocessing import Process
+from typing import Any, Callable, Dict, Generator, List, Union
 
-from hummingbot.core.gateway import (
-    docker_ipc,
-    get_gateway_container_name,
-    set_hummingbot_pipe,
-)
+import aioprocessing
+
+import docker
+from hummingbot.core.gateway import set_hummingbot_pipe
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -136,15 +133,15 @@ def fork_and_start(main_function: Callable):
         # run the main function as parent.
         main_function()
 
-        # stop the gateway container.
-        try:
-            asyncio.get_event_loop().run_until_complete(docker_ipc(
-                "stop",
-                container=get_gateway_container_name(),
-                timeout=1
-            ))
-        except Exception:
-            pass
+        # gateway removed
+        # try:
+        #     asyncio.get_event_loop().run_until_complete(docker_ipc(
+        #         "stop",
+        #         container=get_gateway_container_name(),
+        #         timeout=1
+        #     ))
+        # except Exception:
+        #     pass
     finally:
 
         # close pipes.

bin/hummingbot.py

@@ -21,7 +21,6 @@
 from hummingbot.client.ui.style import load_style
 from hummingbot.core.event.event_listener import EventListener
 from hummingbot.core.event.events import HummingbotUIEvent
-from hummingbot.core.gateway import start_existing_gateway_container
 from hummingbot.core.utils import detect_available_port
 from hummingbot.core.utils.async_utils import safe_gather
 
@@ -65,7 +64,7 @@ async def main_async(client_config_map: ClientConfigAdapter):
     start_listener: UIStartListener = UIStartListener(hb)
     hb.app.add_listener(HummingbotUIEvent.Start, start_listener)
 
-    tasks: List[Coroutine] = [hb.run(), start_existing_gateway_container(client_config_map)]
+    tasks: List[Coroutine] = [hb.run()]
     if client_config_map.debug_console:
         if not hasattr(__builtins__, "help"):
             import _sitebuiltins

bin/hummingbot_quickstart.py

@@ -30,7 +30,6 @@
 from hummingbot.client.ui import login_prompt
 from hummingbot.client.ui.style import load_style
 from hummingbot.core.event.events import HummingbotUIEvent
-from hummingbot.core.gateway import start_existing_gateway_container
 from hummingbot.core.management.console import start_management_console
 from hummingbot.core.utils.async_utils import safe_gather
 
@@ -121,7 +120,7 @@ async def quick_start(args: argparse.Namespace, secrets_manager: BaseSecretsMana
     start_listener: UIStartListener = UIStartListener(hb, is_script=is_script, is_quickstart=True)
     hb.app.add_listener(HummingbotUIEvent.Start, start_listener)
 
-    tasks: List[Coroutine] = [hb.run(), start_existing_gateway_container(client_config_map)]
+    tasks: List[Coroutine] = [hb.run()]
     if client_config_map.debug_console:
         management_port: int = detect_available_port(8211)
         tasks.append(start_management_console(locals(), host="localhost", port=management_port))

hummingbot/client/command/config_command.py

@@ -66,8 +66,6 @@
                              "pmm_script_file_path",
                              "ethereum_chain_name",
                              "gateway",
-                             "gateway_enabled",
-                             "gateway_cert_passphrase",
                              "gateway_api_host",
                              "gateway_api_port",
                              "rate_oracle_source",

hummingbot/client/command/connect_command.py

@@ -38,8 +38,7 @@ async def connect_exchange(self,  # type: HummingbotApplication
         self.app.hide_input = True
         if connector_name == "kraken":
             self.notify("Reminder: Please ensure your Kraken API Key Nonce Window is at least 10.")
-        else:
-            connector_config = ClientConfigAdapter(AllConnectorSettings.get_connector_config_keys(connector_name))
+        connector_config = ClientConfigAdapter(AllConnectorSettings.get_connector_config_keys(connector_name))
         if Security.connector_config_file_exists(connector_name):
             await Security.wait_til_decryption_done()
             api_key_config = [

hummingbot/client/command/gateway_command.py

@@ -2,7 +2,7 @@
 import asyncio
 import itertools
 import time
-from typing import TYPE_CHECKING, Any, Dict, List, Optional
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
 
 import pandas as pd
 
@@ -13,7 +13,7 @@
 from hummingbot.client.ui.completer import load_completer
 from hummingbot.client.ui.interface_utils import format_df_for_printout
 from hummingbot.connector.connector_status import get_connector_status
-from hummingbot.core.gateway import docker_ipc, get_gateway_paths
+from hummingbot.core.gateway import get_gateway_paths
 from hummingbot.core.gateway.gateway_http_client import GatewayHttpClient
 from hummingbot.core.gateway.gateway_status_monitor import GatewayStatus
 from hummingbot.core.utils.async_utils import safe_ensure_future
@@ -68,11 +68,6 @@ def gateway_config(self,
         else:
             safe_ensure_future(self._show_gateway_configuration(key), loop=self.ev_loop)
 
-    @staticmethod
-    async def check_gateway_image(docker_repo: str, docker_tag: str) -> bool:
-        image_list: List = await docker_ipc("images", name=f"{docker_repo}:{docker_tag}", quiet=True)
-        return len(image_list) > 0
-
     async def _test_connection(self):
         # test that the gateway is running
         if await self._get_gateway_instance().ping_gateway():
@@ -103,26 +98,6 @@ async def _generate_certs(
         self.notify(f"Gateway SSL certification files are created in {certs_path}.")
         self._get_gateway_instance().reload_certs(self.client_config_map)
 
-    async def _generate_gateway_confs(
-            self,       # type: HummingbotApplication
-            container_id: str, conf_path: str = "/usr/src/app/conf"
-    ):
-        try:
-            cmd: str = f"./setup/generate_conf.sh {conf_path}"
-            exec_info = await docker_ipc(method_name="exec_create",
-                                         container=container_id,
-                                         cmd=cmd,
-                                         user="hummingbot")
-
-            await docker_ipc(method_name="exec_start",
-                             exec_id=exec_info["Id"],
-                             detach=True)
-            return
-        except asyncio.CancelledError:
-            raise
-        except Exception:
-            raise
-
     async def ping_gateway_api(self, max_wait: int) -> bool:
         """
         Try to reach the gateway API for up to max_wait seconds
@@ -139,35 +114,7 @@ async def ping_gateway_api(self, max_wait: int) -> bool:
 
         return True
 
-    async def ping_gateway_docker_and_api(self, max_wait: int) -> bool:
-        """
-        Try to reach the docker and then the gateway API for up to max_wait seconds
-        """
-        now = int(time.time())
-        docker_live = await self.ping_gateway_docker()
-        while not docker_live:
-            later = int(time.time())
-            if later - now > max_wait:
-                return False
-            await asyncio.sleep(0.5)
-            docker_live = await self.ping_gateway_docker()
-
-        return await self.ping_gateway_api(max_wait)
-
-    async def ping_gateway_docker(self) -> bool:
-        try:
-            await docker_ipc("version")
-            return True
-        except Exception:
-            return False
-
     async def _gateway_status(self):
-        can_reach_docker = await self.ping_gateway_docker()
-        if not can_reach_docker:
-            self.notify("\nError: It looks like you do not have Docker installed or running. Gateway commands will not "
-                        "work without it. Please install or start Docker and restart Hummingbot.")
-            return
-
         if self._gateway_monitor.gateway_status is GatewayStatus.ONLINE:
             try:
                 status = await self._get_gateway_instance().get_gateway_status()
@@ -232,7 +179,10 @@ async def _gateway_connect(
                 available_networks: List[Dict[str, Any]] = connector_config[0]["available_networks"]
                 trading_type: str = connector_config[0]["trading_type"][0]
                 additional_spenders: List[str] = connector_config[0].get("additional_spenders", [])
-                additional_prompts: Dict[str, str] = connector_config[0].get("additional_add_wallet_prompts", {})
+                additional_prompts: Dict[str, str] = connector_config[0].get(  # These will be stored locally.
+                    "additional_add_wallet_prompts",  # If Gateway requires additional, prompts with secure info,
+                    {}  # a new attribute must be added (e.g. additional_secure_add_wallet_prompts)
+                )
 
                 # ask user to select a chain. Automatically select if there is only one.
                 chains: List[str] = [d['chain'] for d in available_networks]
@@ -286,7 +236,7 @@ async def _gateway_connect(
 
                 # if the user has no wallet, ask them to select one
                 if len(wallets) < 1 or chain == "near" or len(additional_prompts) != 0:
-                    wallet_address = await self._prompt_for_wallet_address(
+                    wallet_address, additional_prompt_values = await self._prompt_for_wallet_address(
                         chain=chain, network=network, additional_prompts=additional_prompts
                     )
 
@@ -311,9 +261,13 @@ async def _gateway_connect(
                         wallet_table: List[Dict[str, Any]] = []
                         for w in wallets:
                             balances: Dict[str, Any] = await self._get_gateway_instance().get_balances(
-                                chain, network, w, [native_token]
+                                chain, network, w, [native_token], connector
+                            )
+                            balance = (
+                                balances['balances'].get(native_token)
+                                or balances['balances']['total'].get(native_token)
                             )
-                            wallet_table.append({"balance": balances['balances'][native_token], "address": w})
+                            wallet_table.append({"balance": balance, "address": w})
 
                         wallet_df: pd.DataFrame = build_wallet_display(native_token, wallet_table)
                         self.notify(wallet_df.to_string(index=False))
@@ -332,7 +286,7 @@ async def _gateway_connect(
                     else:
                         while True:
                             try:
-                                wallet_address = await self._prompt_for_wallet_address(
+                                wallet_address, additional_prompt_values = await self._prompt_for_wallet_address(
                                     chain=chain, network=network, additional_prompts=additional_prompts
                                 )
                                 break
@@ -342,16 +296,24 @@ async def _gateway_connect(
                         # display wallet balance
                         native_token: str = native_tokens[chain]
                         balances: Dict[str, Any] = await self._get_gateway_instance().get_balances(
-                            chain, network, wallet_address, [native_token]
+                            chain, network, wallet_address, [native_token], connector
                         )
-                        wallet_table: List[Dict[str, Any]] = [{"balance": balances['balances'][native_token], "address": wallet_address}]
+                        wallet_table: List[Dict[str, Any]] = [{"balance": balances['balances'].get(native_token) or balances['balances']['total'].get(native_token), "address": wallet_address}]
                         wallet_df: pd.DataFrame = build_wallet_display(native_token, wallet_table)
                         self.notify(wallet_df.to_string(index=False))
 
                 self.app.clear_input()
 
                 # write wallets to Gateway connectors settings.
-                GatewayConnectionSetting.upsert_connector_spec(connector, chain, network, trading_type, wallet_address, additional_spenders)
+                GatewayConnectionSetting.upsert_connector_spec(
+                    connector_name=connector,
+                    chain=chain,
+                    network=network,
+                    trading_type=trading_type,
+                    wallet_address=wallet_address,
+                    additional_spenders=additional_spenders,
+                    additional_prompt_values=additional_prompt_values,
+                )
                 self.notify(f"The {connector} connector now uses wallet {wallet_address} on {chain}-{network}")
 
                 # update AllConnectorSettings and fee overrides.
@@ -369,7 +331,7 @@ async def _prompt_for_wallet_address(
         chain: str,
         network: str,
         additional_prompts: Dict[str, str],
-    ) -> Optional[str]:
+    ) -> Tuple[Optional[str], Dict[str, str]]:
         self.app.clear_input()
         self.placeholder_mode = True
         wallet_private_key = await self.app.prompt(
@@ -392,7 +354,7 @@ async def _prompt_for_wallet_address(
                 return
 
         for field, prompt in additional_prompts.items():
-            value = await self.app.prompt(prompt=prompt)
+            value = await self.app.prompt(prompt=prompt, is_password=True)
             self.app.clear_input()
             if self.app.to_stop_config:
                 return
@@ -402,7 +364,7 @@ async def _prompt_for_wallet_address(
             chain, network, wallet_private_key, **additional_prompt_values
         )
         wallet_address: str = response["address"]
-        return wallet_address
+        return wallet_address, additional_prompt_values
 
     async def _show_gateway_connector_tokens(
             self,           # type: HummingbotApplication

hummingbot/client/command/start_command.py

@@ -57,7 +57,7 @@ def start(self,  # type: HummingbotApplication
               script: Optional[str] = None,
               is_quickstart: Optional[bool] = False):
         if threading.current_thread() != threading.main_thread():
-            self.ev_loop.call_soon_threadsafe(self.start, log_level)
+            self.ev_loop.call_soon_threadsafe(self.start, log_level, script)
             return
         safe_ensure_future(self.start_check(log_level, script, is_quickstart), loop=self.ev_loop)
 

hummingbot/client/config/client_config_map.py

@@ -541,7 +541,12 @@ def validate_pmm_script_file_path(cls, v: str):
 
 
 class GatewayConfigMap(BaseClientModel):
-    gateway_api_host: str = Field(default="localhost")
+    gateway_api_host: str = Field(
+        default="localhost",
+        client_data=ClientFieldData(
+            prompt=lambda cm: "Please enter your Gateway API host",
+        ),
+    )
     gateway_api_port: str = Field(
         default="15888",
         client_data=ClientFieldData(