[DataCollector] add support for multi data files

tmp

Author: techfreaque

Meta/Keywords/scripting_library/data/reading/exchange_public_data.py

@@ -210,10 +210,23 @@ async def get_candles_from_name(ctx, source_name="low", time_frame=None, symbol=
 
 async def _local_candles_manager(exchange_manager, symbol, time_frame, start_timestamp, end_timestamp):
     # warning: should only be called with an exchange simulator (in backtesting)
-    ohlcv_data: list = await exchange_manager.exchange.exchange_importers[0].get_ohlcv(
-        exchange_name=exchange_manager.exchange_name,
-        symbol=symbol,
-        time_frame=commons_enums.TimeFrames(time_frame))
+    ohlcv_data: list = None
+    for importer in exchange_manager.exchange.exchange_importers:
+        if (importer.exchange_name == exchange_manager.exchange_name 
+            and symbol in importer.symbols
+            and commons_enums.TimeFrames(time_frame) in importer.time_frames
+            ):
+            ohlcv_data: list = await importer.get_ohlcv(
+                exchange_name=exchange_manager.exchange_name,
+                symbol=symbol,
+                time_frame=commons_enums.TimeFrames(time_frame),
+            )
+            break
+    if not ohlcv_data:
+        raise(
+            f"Failed to load candles data for {symbol} {time_frame} "
+            f"{exchange_manager.exchange_name} - no data available to load"
+            )
     chronological_candles = sorted(ohlcv_data, key=lambda candle: candle[0])
     full_candles_history = [
         ohlcv[-1]
@@ -237,7 +250,7 @@ async def _get_candle_manager(context, symbol, time_frame, max_history):
             return candle_manager
         start_timestamp = backtesting_api.get_backtesting_starting_time(context.exchange_manager.exchange.backtesting)
         end_timestamp = backtesting_api.get_backtesting_ending_time(context.exchange_manager.exchange.backtesting)
-        _key = symbol + time_frame + str(start_timestamp) + str(end_timestamp)
+        _key = context.exchange_manager.exchange_name + symbol + time_frame + str(start_timestamp) + str(end_timestamp)
         try:
             return run_persistence.get_shared_element(_key)
         except KeyError:

Services/Interfaces/web_interface/controllers/backtesting.py

@@ -19,6 +19,7 @@
 
 import octobot_commons.time_frame_manager as time_frame_manager
 import octobot_commons.constants as commons_constants
+import octobot_backtesting.constants as backtesting_constants
 
 import tentacles.Services.Interfaces.web_interface as web_interface
 import tentacles.Services.Interfaces.web_interface.login as login
@@ -61,10 +62,10 @@ def backtesting():
                 auto_stop = flask.request.args.get("auto_stop", False)
                 data_sources = data.get("data_sources", None)
                 if data_sources:
-                    if models.CURRENT_BOT_DATA in data_sources:
-                        data_sources = [models.CURRENT_BOT_DATA]
+                    if backtesting_constants.CONFIG_CURRENT_BOT_DATA in data_sources:
+                        data_sources = [backtesting_constants.CONFIG_CURRENT_BOT_DATA]
                 else:
-                   data_sources = [data.get("data_source", models.CURRENT_BOT_DATA) ]
+                   data_sources = [data.get("data_source", backtesting_constants.CONFIG_CURRENT_BOT_DATA) ]
                 exchange_ids = data.get("exchange_ids", None)
                 if not exchange_ids:
                     exchange_ids = [data.get("exchange_id", None)]

Services/Interfaces/web_interface/models/__init__.py

@@ -31,7 +31,6 @@
 
 
 from tentacles.Services.Interfaces.web_interface.models.backtesting import (
-    CURRENT_BOT_DATA,
     get_full_candle_history_exchange_list,
     get_other_history_exchange_list,
     get_data_files_with_description,

Services/Interfaces/web_interface/models/backtesting.py

@@ -32,7 +32,6 @@
 import octobot_tentacles_manager.api as tentacles_manager_api
 import octobot_backtesting.constants as backtesting_constants
 import octobot_backtesting.enums as backtesting_enums
-import octobot_backtesting.collectors as collectors
 import octobot_services.interfaces.util as interfaces_util
 import octobot_services.enums as services_enums
 import octobot_trading.constants as trading_constants
@@ -45,7 +44,6 @@
 
 
 STOPPING_TIMEOUT = 30
-CURRENT_BOT_DATA = "current_bot_data"
 # data collector can be really slow, let it up to 2 hours to run
 DATA_COLLECTOR_TIMEOUT = 2 * commons_constants.HOURS_TO_SECONDS
 
@@ -104,7 +102,7 @@ def start_backtesting_using_current_bot_data(data_sources, exchange_ids, source,
                                              start_timestamp=None, end_timestamp=None, trading_type=None,
                                              enable_logs=False, auto_stop=False,
                                              collector_start_callback=None, start_callback=None):
-    use_current_bot_data = data_sources == [CURRENT_BOT_DATA]
+    use_current_bot_data = data_sources == [backtesting_constants.CONFIG_CURRENT_BOT_DATA]
     files = None if use_current_bot_data else data_sources
     return _start_backtesting(files, source, reset_tentacle_config=reset_tentacle_config,
                               run_on_common_part_only=False,