Add watchdog using a background thread to detect hung live execution (#…

…201)

- This does not fix any hung bugs, but will make them more apparent
- Causes and hanging could be stalled HTTPS requests, etc.
- Hopefully fix the issues of APScheduler failing to fire `run_live` timed function correctly
- Also replace `ganache` with `anvil` as `ganache` causes a lot of instability in the tests. Remove some @flaky as well.

.github/workflows/tests.yml

@@ -8,6 +8,7 @@ on:
 
 jobs:
   automated-test-suite:
+    timeout-minutes: 20
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -28,15 +29,22 @@ jobs:
         with:
           path: .venv
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
-      - name: Install Ganache
-        run: npm install -g ganache
+      - name: Install Anvil
+        run: |
+          curl -L https://foundry.paradigm.xyz | bash
+          PATH=~/.foundry/bin:$PATH
+          foundryup
+          anvil --version
       - name: Install dependencies
         if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
         # We don't install -E qstrader and run legacy tests on CI as they
         # download too much data
-        run: poetry install --no-interaction -E web-server -E execution
+        run: |          
+          poetry install --no-interaction -E web-server -E execution
       - name: Run test scripts
-        run: poetry run pytest --tb=native
+        run: |
+          PATH=~/.foundry/bin:$PATH
+          poetry run pytest --tb=native
         env:
           TRADING_STRATEGY_API_KEY: ${{ secrets.TRADING_STRATEGY_API_KEY }}
           BNB_CHAIN_JSON_RPC: ${{ secrets.BNB_CHAIN_JSON_RPC }}      

env/local-test.env

@@ -1,3 +1,5 @@
-export JSON_RPC_BINANCE="https://bsc-mainnet.nodereal.io/v1/64a9df0874fb4a93b9d0a3849de012d3"
+# export JSON_RPC_BINANCE="https://bsc-mainnet.nodereal.io/v1/64a9df0874fb4a93b9d0a3849de012d3"
+export JSON_RPC_BINANCE="https://intensive-lingering-dew.bsc.quiknode.pro/fb83ebe37b46b703d8f05c8e8fbda662aa7f9eeb/"
 export JSON_RPC_POLYGON="https://polygon-rpc.com"
 export TRADING_STRATEGY_API_KEY="secret-token:tradingstrategy-be8540bb501e2eccbdf6117ad65e0fac984ccfb3715d7a7b1046bcb8b1ebdb58"
+export BNB_CHAIN_JSON_RPC=$JSON_RPC_BINANCE

scripts/multiprocess_hang_test.py

@@ -0,0 +1,73 @@
+"""Multiprocess manager hang test.
+
+Good output (no multiprocessing.Manager):
+
+.. code-block:: text
+
+    Doing subrouting launch_and_read_process
+    poll() is None
+    Terminating
+    Got exit code -15
+    Got output Doing subrouting run_unkillable
+    This is an example output
+
+The hang with Manager:
+
+.. code-block:: text
+
+    Doing subrouting launch_and_read_process
+    poll() is None
+    Terminating
+    Got exit code -15
+
+"""
+import multiprocessing
+import subprocess
+import sys
+import time
+
+
+def launch_and_read_process():
+    proc = subprocess.Popen(
+        [
+            "python",
+            sys.argv[0],
+            "run_unkillable"
+        ],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+
+    # Give time for the process to run and print()
+    time.sleep(3)
+
+    status = proc.poll()
+    print("poll() is", status)
+
+    print("Terminating")
+    assert proc.returncode is None
+    proc.terminate()
+    exit_code = proc.wait()
+    print("Got exit code", exit_code)
+    stdout, stderr = proc.communicate()
+    print("Got output", stdout.decode("utf-8"))
+
+
+def run_unkillable():
+    # Disable manager creation to make the code run correctly
+    manager = multiprocessing.Manager()
+    d = manager.dict()
+    d["foo"] = "bar"
+    print("This is an example output", flush=True)
+    time.sleep(999)
+
+
+def main():
+    mode = sys.argv[1]
+    print("Doing subrouting", mode)
+    func = globals().get(mode)
+    func()
+
+
+if __name__ == "__main__":
+    main()

strategies/test_only/pancakeswap_v2_main_loop.py

@@ -296,7 +296,7 @@ def filter_universe(self, dataset: Dataset) -> TradingStrategyUniverse:
             # We do a bit detour here as we need to address the assets by their trading pairs first
             # https://tradingstrategy.ai/trading-view/binance/pancakeswap-v2/bnb-busd
             bnb_busd = pairs.get_pair_by_smart_contract("0x58f876857a02d6762e0101bb5c46a8c1ed44dc16")
-            bnb_busd.fee = 0.025
+            bnb_busd.fee = 25
             assert bnb_busd, "We do not have BNB-BUSD, something wrong with the dataset"
 
             # Get daily candles as Pandas DataFrame
@@ -352,7 +352,7 @@ def strategy_factory(
 
     assert isinstance(execution_model, (UniswapV2ExecutionModel, UniswapV2ExecutionModelVersion0)), f"This strategy is compatible only with UniswapV2ExecutionModel, got {execution_model}"
 
-    assert execution_model.chain_id == 1337, f"This strategy is hardcoded to ganache-cli test chain, got chain {execution_model.chain_id}"
+    assert execution_model.chain_id in (56, 1337), f"This strategy is hardcoded to ganache-cli test chain, got chain {execution_model.chain_id}"
 
     universe_model = OurUniverseModel(client, timed_task_context_manager)
 

tests/mainnet_fork/README.md

@@ -0,0 +1,4 @@
+This tests use Anvil to create a mainnet fork for testing. 
+
+They are pretty heavy to run, so thus they are in a different folder.
+

tests/ganache/test_ema_crossover.py → tests/mainnet_fork/test_ema_crossover.py

@@ -21,7 +21,9 @@
 from eth_account import Account
 
 from eth_defi.abi import get_deployed_contract
+from eth_defi.anvil import fork_network_anvil
 from eth_defi.confirmation import wait_transactions_to_complete
+from eth_defi.gas import node_default_gas_price_strategy
 from eth_typing import HexAddress, HexStr
 
 from web3 import Web3, HTTPProvider
@@ -66,31 +68,33 @@ def large_busd_holder() -> HexAddress:
 
 
 @pytest.fixture()
-def ganache_bnb_chain_fork(logger, large_busd_holder) -> str:
+def anvil_bnb_chain_fork(logger, large_busd_holder) -> str:
     """Create a testable fork of live BNB chain.
 
     :return: JSON-RPC URL for Web3
     """
 
     mainnet_rpc = os.environ["BNB_CHAIN_JSON_RPC"]
 
-    launch = fork_network(
+    # Start Ganache
+    launch = fork_network_anvil(
         mainnet_rpc,
-        block_time=1,  # Insta mining cannot be done in this test
-        evm_version="berlin",  # BSC is not yet London compatible?
-        unlocked_addresses=[large_busd_holder],  # Unlock WBNB stealing
-        quiet=True,  # Otherwise the Ganache output is millions lines of long
-    )
-    yield launch.json_rpc_url
-    # Wind down Ganache process after the test is complete
-    launch.close(verbose=True)
+        unlocked_addresses=[large_busd_holder])
+    try:
+        yield launch.json_rpc_url
+        # Wind down Ganache process after the test is complete
+    finally:
+        launch.close(log_level=logging.INFO)
 
 
 @pytest.fixture
-def web3(ganache_bnb_chain_fork: str):
+def web3(anvil_bnb_chain_fork: str):
     """Set up a local unit testing blockchain."""
     # https://web3py.readthedocs.io/en/stable/examples.html#contract-unit-tests-in-python
-    return Web3(HTTPProvider(ganache_bnb_chain_fork))
+    web3 = Web3(HTTPProvider(anvil_bnb_chain_fork, request_kwargs={"timeout": 5}))
+    web3.eth.set_gas_price_strategy(node_default_gas_price_strategy)
+    install_chain_middleware(web3)
+    return web3
 
 
 @pytest.fixture

tests/ganache/test_forked_bnb_chain_pancake.py → tests/mainnet_fork/test_forked_bnb_chain_pancake.py

@@ -19,6 +19,9 @@
 import flaky
 import pytest
 from eth_account import Account
+from eth_defi.anvil import fork_network_anvil
+from eth_defi.chain import install_chain_middleware
+from eth_defi.gas import node_default_gas_price_strategy
 from eth_typing import HexAddress, HexStr
 from hexbytes import HexBytes
 
@@ -74,33 +77,33 @@ def large_busd_holder() -> HexAddress:
 
 
 @pytest.fixture()
-def ganache_bnb_chain_fork(logger, large_busd_holder) -> str:
+def anvil_bnb_chain_fork(logger, large_busd_holder) -> str:
     """Create a testable fork of live BNB chain.
 
     :return: JSON-RPC URL for Web3
     """
 
     mainnet_rpc = os.environ["BNB_CHAIN_JSON_RPC"]
 
-    if not is_localhost_port_listening(19999):
-        # Start Ganache
-        launch = fork_network(
-            mainnet_rpc,
-            unlocked_addresses=[large_busd_holder])
+    # Start Ganache
+    launch = fork_network_anvil(
+        mainnet_rpc,
+        unlocked_addresses=[large_busd_holder])
+    try:
         yield launch.json_rpc_url
         # Wind down Ganache process after the test is complete
-        launch.close(verbose=True)
-    else:
-        logger.warning("Detected existing Ganache running - terminate with: kill -9 $(lsof -ti:19999)")
-        # Assume ganache-cli manually launched by the dev
-        yield "http://localhost:19999"
+    finally:
+        launch.close(log_level=logging.INFO)
 
 
 @pytest.fixture
-def web3(ganache_bnb_chain_fork: str):
+def web3(anvil_bnb_chain_fork: str):
     """Set up a local unit testing blockchain."""
     # https://web3py.readthedocs.io/en/stable/examples.html#contract-unit-tests-in-python
-    return Web3(HTTPProvider(ganache_bnb_chain_fork, request_kwargs={"timeout": 2}))
+    web3 = Web3(HTTPProvider(anvil_bnb_chain_fork, request_kwargs={"timeout": 5}))
+    web3.eth.set_gas_price_strategy(node_default_gas_price_strategy)
+    install_chain_middleware(web3)
+    return web3
 
 
 @pytest.fixture
@@ -245,13 +248,10 @@ def routing_model(asset_busd):
         reserve_token_address=asset_busd.address)
 
 
-# Flaky because Ganache
-@flaky.flaky(max_runs=5)
 def test_forked_pancake(
         logger: logging.Logger,
         web3: Web3,
         strategy_path: Path,
-        ganache_bnb_chain_fork,
         hot_wallet: HotWallet,
         pancakeswap_v2: UniswapV2Deployment,
         state: State,

tests/ganache/test_frozen_position_and_blacklist.py → tests/mainnet_fork/test_frozen_position_and_blacklist.py

@@ -21,7 +21,10 @@
 
 import pytest
 from eth_account import Account
+from eth_defi.anvil import fork_network_anvil
+from eth_defi.chain import install_chain_middleware
 from eth_defi.confirmation import wait_transactions_to_complete
+from eth_defi.gas import node_default_gas_price_strategy
 from eth_typing import HexAddress, HexStr
 from hexbytes import HexBytes
 
@@ -87,34 +90,33 @@ def large_busd_holder() -> HexAddress:
 
 
 @pytest.fixture()
-def ganache_bnb_chain_fork(logger, large_busd_holder) -> str:
+def anvil_bnb_chain_fork(logger, large_busd_holder) -> str:
     """Create a testable fork of live BNB chain.
 
-    Unlike other tests, we use 1 second block time,
-    because we need to test failed transaction scenarios
-    and otherwise this cannot be emulated.
-
     :return: JSON-RPC URL for Web3
     """
 
     mainnet_rpc = os.environ["BNB_CHAIN_JSON_RPC"]
 
-    assert not is_localhost_port_listening(19999), "Ganache alread running"
     # Start Ganache
-    launch = fork_network(
+    launch = fork_network_anvil(
         mainnet_rpc,
-        block_time=1,
         unlocked_addresses=[large_busd_holder])
-    yield launch.json_rpc_url
-    # Wind down Ganache process after the test is complete
-    launch.close(verbose=True)
+    try:
+        yield launch.json_rpc_url
+        # Wind down Ganache process after the test is complete
+    finally:
+        launch.close(log_level=logging.INFO)
 
 
 @pytest.fixture
-def web3(ganache_bnb_chain_fork: str):
+def web3(anvil_bnb_chain_fork: str):
     """Set up a local unit testing blockchain."""
     # https://web3py.readthedocs.io/en/stable/examples.html#contract-unit-tests-in-python
-    return Web3(HTTPProvider(ganache_bnb_chain_fork, request_kwargs={"timeout": 2}))
+    web3 = Web3(HTTPProvider(anvil_bnb_chain_fork, request_kwargs={"timeout": 5}))
+    web3.eth.set_gas_price_strategy(node_default_gas_price_strategy)
+    install_chain_middleware(web3)
+    return web3
 
 
 @pytest.fixture

tests/ganache/test_main_loop.py → tests/mainnet_fork/test_main_loop.py

@@ -68,9 +68,11 @@ def ganache_bnb_chain_fork(logger, large_busd_holder) -> str:
             mainnet_rpc,
             block_time=1,
             unlocked_addresses=[large_busd_holder])
-        yield launch.json_rpc_url
-        # Wind down Ganache process after the test is complete
-        launch.close(verbose=True)
+        try:
+            yield launch.json_rpc_url
+        finally:
+            # Wind down Ganache process after the test is complete
+            launch.close(verbose=True)
     else:
         raise AssertionError("ganache zombie detected")
 
@@ -182,7 +184,7 @@ def strategy_path() -> Path:
 
 # Flaky because of unstable Ganache
 @flaky.flaky
-def test_main_loop(
+def test_main_loop_success(
         logger: logging.Logger,
         strategy_path: Path,
         ganache_bnb_chain_fork,