From ef134816d9cd48eef2581d6656a28963f318f062 Mon Sep 17 00:00:00 2001
From: Mikko Ohtamaa <mikko@redinnovation.com>
Date: Sun, 26 Jan 2025 12:24:52 +0100
Subject: [PATCH] More FinRL universe creation

---
 scripts/data-downloads/prefilter-finrl.py | 23 +++++++++++++++++++----
 tests/cli/test_console.py                 |  4 ++++
 2 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/scripts/data-downloads/prefilter-finrl.py b/scripts/data-downloads/prefilter-finrl.py
index 4197d0805..4d2133297 100644
--- a/scripts/data-downloads/prefilter-finrl.py
+++ b/scripts/data-downloads/prefilter-finrl.py
@@ -29,9 +29,11 @@
 from pathlib import Path
 
 import pandas as pd
+from IPython.core.display_functions import display
 
 from tradingstrategy.chain import ChainId
 from tradingstrategy.client import Client
+from tradingstrategy.exchange import ExchangeType
 from tradingstrategy.pair import PandasPairUniverse
 from tradingstrategy.timebucket import TimeBucket
 from tradingstrategy.utils.token_filter import filter_pairs_default
@@ -66,8 +68,8 @@ def fill_missing_ohlcv(df, columns_to_zero=['open', 'high', 'low', 'close', 'vol
     filled_df = df.reindex(multi_index)
 
     # Fill specified columns with zeros where data is missing
-    for col in columns_to_zero:
-        filled_df[col] = filled_df[col].fillna(0)
+    # for col in columns_to_zero:
+    #    filled_df[col] = filled_df[col]
 
     return filled_df
 
@@ -258,12 +260,25 @@ def main():
     #         2023-01-01 16:00:00   0.0   0.0  0.0    0.0     0.0  0.0
     merged_df = filled_df
 
+    if merged_df.index.duplicated().any():
+        duplicate_rows = merged_df[merged_df.index.duplicated(keep=False)].sort_index()
+        display(duplicate_rows)
+        assert merged_df.index.duplicated().any(), f"Contains duplicates"
+
     # These are already in indexes
     merged_df = merged_df.reset_index()
 
     def _ticker(pair_id):
-        pair = pair_universe.get_pair_by_id(pair_id))
-        return f"{pair.get_ticker()}-{pair.fee}"
+        pair = pair_universe.get_pair_by_id(pair_id)
+        match pair.dex_type:
+            case ExchangeType.uniswap_v2:
+                dex_type = "v2"
+            case ExchangeType.uniswap_v3:
+                dex_type = "v3"
+            case _:
+                raise NotImplementedError(f"Unknown DEX {pair}")
+
+        return f"{pair.get_ticker()}-{dex_type}-{pair.fee}"
 
     # Add metadata to every row
     pair_universe = PandasPairUniverse(pairs_df)
diff --git a/tests/cli/test_console.py b/tests/cli/test_console.py
index 2803889b5..a5070a690 100644
--- a/tests/cli/test_console.py
+++ b/tests/cli/test_console.py
@@ -6,6 +6,10 @@
 from tradeexecutor.cli.commands.app import app
 
 pytestmark = pytest.mark.skipif(os.environ.get("JSON_RPC_ETHEREUM") is None, reason="Set JSON_RPC_ETHEREUM environment variable torun this test")
+<<<<<<< Updated upstream
+=======
+
+>>>>>>> Stashed changes
 
 
 @pytest.mark.skipif(os.environ.get("GITHUB_ACTIONS") == "true", reason="This test seems to block Github CI for some reason")