Repair trade where tx generation failed (#882)

- Repair the failed portfolio state if the transaction generation failed due to a crash after `decide_trades()`, but before execution model generated blockchain transactions for broadcast

tests/legacy/test_repair_trade_missing_tx.py

@@ -0,0 +1,72 @@
+"""Check that we can repair trades that never generated blockchaint transactions due to a crash.
+
+"""
+import datetime
+import os
+from decimal import Decimal
+
+import pytest
+
+from tradeexecutor.cli.log import setup_pytest_logging
+from tradeexecutor.monkeypatch.dataclasses_json import patch_dataclasses_json
+from tradeexecutor.state.repair import repair_trades, repair_tx_not_generated
+from tradeexecutor.state.state import State
+from tradeexecutor.state.trade import TradeStatus, TradeType
+from tradeexecutor.statistics.core import calculate_statistics
+from tradeexecutor.strategy.execution_context import ExecutionMode
+from tradingstrategy.client import Client
+
+
+@pytest.fixture(scope="module")
+def logger(request):
+    """Setup test logger."""
+    return setup_pytest_logging(request, mute_requests=False)
+
+
+@pytest.fixture(scope="module")
+def state() -> State:
+    """A state dump with some failed trades we need to unwind.
+
+    Taken as a snapshot from alpha version trade execution run.
+    """
+    f = os.path.join(os.path.dirname(__file__), "trade-missing-tx.json")
+    return State.from_json(open(f, "rt").read())
+
+
+def test_repair_trade_missing_tx(
+    state: State,
+):
+    """Repair trades.
+
+    We have both buys and sells.
+
+    Failed positions are 1 (buy), 26 (buy), 36 (sell)
+    """
+
+    # Check how our positions look like
+    # before repair
+    portfolio = state.portfolio
+    pos = portfolio.get_position_by_id(4)
+    # {6: <Buy #6 0.8714827051238582225784361754 WETH at 3504.1829635670897, success phase>, 8: <Sell #8 0.871478149181352629 WETH at 3478.199651761113, planned phase>, 11: <Buy #11 0.1727611356789154584852805521 WETH at 3232.903676743047, planned phase>, 13: <Buy #13 0.1784399145897731857878380438 WETH at 3253.869387693136, planned phase>}
+    assert pos.trades[8].get_status() == TradeStatus.planned
+    assert pos.trades[8].blockchain_transactions == []
+    assert pos.trades[8].is_missing_blockchain_transactions()
+    assert len(pos.trades) == 4
+
+    assert pos.get_value() == pytest.approx(2831.946809553303)
+    assert portfolio.get_total_equity() == pytest.approx(4062.6783229397383)
+
+    repair_trades = repair_tx_not_generated(state, interactive=False)
+    assert len(repair_trades) == 7  # 7 repairs across two positions
+
+    # We went from planned -> repaird
+    pos = portfolio.get_position_by_id(4)
+    assert pos.trades[8].get_status() == TradeStatus.repaired
+    assert pos.trades[8].blockchain_transactions == []
+    assert pos.trades[8].is_repaired()
+    assert not pos.trades[8].is_missing_blockchain_transactions()
+
+    assert len(pos.trades) == 7  # 4 original + 3 repairs added
+
+    assert pos.get_value() == pytest.approx(2831.946809553303)
+    assert portfolio.get_total_equity() == pytest.approx(4062.6783229397383)

tradeexecutor/cli/commands/repair.py

@@ -12,7 +12,7 @@
 from ..bootstrap import prepare_executor_id, create_state_store, create_execution_and_sync_model, prepare_cache, create_web3_config, create_client
 from ..log import setup_logging
 from ...ethereum.rebroadcast import rebroadcast_all
-from ...state.repair import repair_trades
+from ...state.repair import repair_trades, repair_tx_not_generated
 from ...strategy.approval import UncheckedApprovalModel
 from ...strategy.bootstrap import make_factory_from_strategy_mod
 from ...strategy.description import StrategyExecutionDescription
@@ -174,7 +174,12 @@ def repair(
     routing_state, pricing_model, valuation_method = runner.setup_routing(universe)
 
     #
-    # First fix txs that have unresolved state
+    # First trades that have txs missing
+    #
+    repair_tx_not_generated(state, interactive=True)
+
+    #
+    # Second fix txs that have unresolved state
     #
 
     # Get the latest nonce from the chain

tradeexecutor/state/repair.py

@@ -17,6 +17,7 @@
 - Sell e.g. closing trade failed: position stays open, the assets are marked to be available
   for the future sell
 
+
 """
 import datetime
 import logging
@@ -131,6 +132,31 @@ def repair_trade(portfolio: Portfolio, t: TradeExecution) -> TradeExecution:
     return c
 
 
+def repair_tx_missing(portfolio: Portfolio, t: TradeExecution) -> TradeExecution:
+    """Repair a trade which failed to generate new transactions..
+
+    - Make a counter trade for bookkeeping
+
+    - Set the original trade to repaired state (instead of planned state)
+    """
+    p = portfolio.get_position_by_id(t.position_id)
+
+    c = make_counter_trade(portfolio, p, t)
+    now = datetime.datetime.utcnow()
+    t.repaired_at = t.executed_at = datetime.datetime.utcnow()
+    t.executed_quantity = 0
+    t.executed_reserve = 0
+    assert c.trade_id
+    c.repaired_trade_id = t.trade_id
+    t.add_note(f"Repaired at {now.strftime('%Y-%m-%d %H:%M')}, by #{c.trade_id}")
+    c.add_note(f"Repairing trade #{c.repaired_trade_id}")
+    assert t.get_status() == TradeStatus.repaired
+    assert t.get_value() == 0
+    assert t.get_position_quantity() == 0
+    assert t.planned_quantity != 0
+    return c
+
+
 def close_position_with_empty_trade(portfolio: Portfolio, p: TradingPosition) -> TradeExecution:
     """Make a trade that closes the position.
 
@@ -356,3 +382,81 @@ def repair_trades(
         trades_to_be_repaired,
         new_trades,
     )
+
+
+def repair_tx_not_generated(state: State, interactive=True):
+    """Repair command to fix trades that did not generate tranasctions.
+
+    - Reasons include
+
+    - Currently only manually callable from console
+
+    - Simple deletes trades that have an empty transaction list
+
+    TODO:
+
+        Change this to create repair countertrades and fix positions that way.
+
+    Example exception:
+
+    .. code-block:: text
+
+          File "/usr/src/trade-executor/tradeexecutor/ethereum/routing_model.py", line 395, in trade
+            return self.make_direct_trade(
+          File "/usr/src/trade-executor/tradeexecutor/ethereum/uniswap_v3/uniswap_v3_routing.py", line 257, in make_direct_trade
+            return super().make_direct_trade(
+          File "/usr/src/trade-executor/tradeexecutor/ethereum/routing_model.py", line 112, in make_direct_trade
+            adjusted_reserve_amount = routing_state.adjust_spend(
+          File "/usr/src/trade-executor/tradeexecutor/ethereum/routing_state.py", line 283, in adjust_spend
+            raise OutOfBalance(
+        tradeexecutor.ethereum.routing_state.OutOfBalance: Not enough tokens for <USDC at 0x2791bca1f2de4661ed88a30c99a7a9449aa84174> to perform the trade. Required: 3032399763, on-chain balance for 0x375A8Cd0A654E0eCa46F81c1E5eA5200CC6A737C is 87731979.
+
+    :return:
+        Repair trades generated.
+
+        Empty list of interactive operation was cancelled.
+
+        If empty list is returned the state must **not** be saved, as the state is already mutated.
+    """
+
+    tx_missing_trades = set()
+    portfolio = state.portfolio
+
+    for t in portfolio.get_all_trades():
+        if not t.blockchain_transactions:
+            assert t.get_status() == TradeStatus.planned, f"Trade missing tx, but status is not planned {t}"
+            tx_missing_trades.add(t)
+
+    if not tx_missing_trades:
+        if interactive:
+            print("No trades with missing blockchain transactions detected")
+        return []
+
+    if interactive:
+
+        print("Trade missing TX report")
+        print("-" * 80)
+
+        print("Trade to repair:")
+        for t in tx_missing_trades:
+            print(t)
+
+        confirm = input("Confirm delete [y/n]? ")
+        if confirm.lower() != "y":
+            raise RepairAborted()
+
+    repair_trades_generated = [repair_tx_missing(portfolio, t) for t in tx_missing_trades]
+    if interactive:
+        print("Counter-trades:")
+        for t in repair_trades_generated:
+            position = portfolio.get_position_by_id(t.position_id)
+            print("Position ", position)
+            print("Repair trade ", t.repaired_at)
+
+        confirm = input("Looks fixed [y/n]? ")
+        if confirm.lower() != "y":
+            raise RepairAborted()
+
+    return repair_trades_generated
+
+

tradeexecutor/state/trade.py

@@ -754,9 +754,29 @@ def is_test(self) -> bool:
         return TradeFlag.test_trade in self.flags
 
     def is_failed(self) -> bool:
-        """This trade was succcessfully completed."""
+        """This trade was succcessfully completed.
+
+        See also :py:meth:`is_missing_transaction_generation`.
+        """
         return (self.failed_at is not None) and (self.repaired_at is None)
 
+    def is_missing_blockchain_transactions(self) -> bool:
+        """Trade failed to generate transactions.
+
+        The system crashed between `decide_trades()` and before the execution model prepared blockchain transactions.
+        This is different from the trade execution itself failed (transaction reverted).
+
+        - Because out of balance pre-checks
+
+        - Because of some crash reason
+
+        - After the trade has been marked repaired, we return `False`.
+
+        See also :py:meth:`is_failed`.
+        """
+
+        return not (self.is_failed() or self.is_repaired()) and len(self.blockchain_transactions) == 0
+
     def is_pending(self) -> bool:
         """This trade was succcessfully completed."""
         return self.get_status() in (TradeStatus.started, TradeStatus.broadcasted)