scripts.train Error #55

Activity

1. asavinov commented on Mar 29, 2025

   

   asavinov

   on Mar 29, 2025

   Owner

   More actions

   Hi, probably some feature produces all nulls and hence all rows are removed. Check in the data (in matrix.csv) the values of your features and the feature definition itself. Maybe it is because of input data - although the date range is about 8 years you have only 91 records. What is your time raster: 1 minute, 1 hour, 1 day? If you share your config file then it will be easier to find the cause. You can do it in the discussions.
2. gidixi commented on Mar 30, 2025

   

   gidixi

   on Mar 30, 2025

   Author

   More actions

   Hi, good morning, I'll show you the config.jsonc file which is the standard one I find in the repository, if you need any other info let me know

   {
       "api_key": "xxxx",
       "api_secret": "xxxx",
   
       "telegram_bot_token": "xxxx",
       "telegram_chat_id": "xxxxx",
   
       "data_folder": "data",
   
       // === DOWNLOAD AND MERGE ===
   
       "symbol": "BTCUSDT",
   
       "freq": "1h", // pandas frequency
   
       "data_sources": [
           {"folder": "BTCUSDT", "file": "klines", "column_prefix": ""}
       ],
   
       // === GENERATE FEATURES ===
   
       "feature_sets": [
           {"column_prefix": "", "generator": "talib", "feature_prefix": "", "config":  {"columns": ["close"], "functions": ["SMA"], "windows": [1, 5, 10, 15, 60]}},
           {"column_prefix": "", "generator": "talib", "feature_prefix": "", "config":  {"columns": ["close"], "functions": ["LINEARREG_SLOPE"], "windows": [5, 10, 15, 60]}},
           {"column_prefix": "", "generator": "talib", "feature_prefix": "", "config":  {"columns": ["close"], "functions": ["STDDEV"], "windows": [5, 10, 15, 60]}},
           {"column_prefix": "", "generator": "common.my_feature_example:my_feature_example", "feature_prefix": "", "config":  {"columns": "close", "function": "add", "parameter": 2.0, "names": "close_add"}}
       ],
   
       // === LABELS ===
   
       "label_sets": [
           {"column_prefix": "", "generator": "highlow2", "feature_prefix": "", "config":  {"columns": ["close", "high", "low"], "function": "high", "thresholds": [2.0], "tolerance": 0.2, "horizon": 120, "names": ["high_20"]}},
           {"column_prefix": "", "generator": "highlow2", "feature_prefix": "", "config":  {"columns": ["close", "high", "low"], "function": "low", "thresholds": [2.0], "tolerance": 0.2, "horizon": 120, "names": ["low_20"]}}
       ],
   
       // === TRAIN ===
   
       "label_horizon": 120,  // Batch/offline: do not use these last rows because their labels might not be correct
       "train_length": 525600,  // Batch/offline: Uses this number of rows for training (if not additionally limited by the algorithm)
   
       "train_feature_sets": [
       {
           "generator": "train_features", "config": {
           // Use values from the attributes: train_features, labels, algorithms
       }}
       ],
   
       "train_features": [
           "close_SMA_1", "close_SMA_5", "close_SMA_10", "close_SMA_15", "close_SMA_60",
           "close_LINEARREG_SLOPE_5", "close_LINEARREG_SLOPE_10", "close_LINEARREG_SLOPE_15", "close_LINEARREG_SLOPE_60",
           "close_STDDEV_5", "close_STDDEV_10", "close_STDDEV_15", "close_STDDEV_60"
       ],
   
       "labels": ["high_20", "low_20"],
   
       "algorithms": [
           {
               "name": "lc",  // Unique name will be used as a column suffix
               "algo": "lc",  // Algorithm type is used to choose the train/predict function
               "params": {"is_scale": true, "length": 1000000},
               "train": {"penalty": "l2", "C": 1.0, "class_weight": null, "solver": "sag", "max_iter": 100}
           }
       ],
   
       "features_horizon": 2880, // Online/stream: Minimum data length for computing features. Take it from feature generator parameters
       "features_last_rows": 5, // Online/stream: Last values which are really needed and have to be computed. All older values are not needed
   
       // === GENERATE SIGNALS ===
   
       "signal_sets": [
           {
               // Combine two unsigned scores into one signed score
               "generator": "combine", "config": {
                   "columns": ["high_20_lc", "low_20_lc"],  // 2 columns: with grow score and fall score
                   "names": "trade_score",  // Output column name: positive values - buy, negative values - sell
                   "combine": "difference", // "no_combine" (or empty), "relative", "difference"
                   "coefficient": 1.0, "constant": 0.0  // Normalize
           }},
           {
               // Generate boolean buy-sell column depending on thresholds
               "generator": "threshold_rule", "config": {
                   "columns": "trade_score",
                   "names": ["buy_signal_column", "sell_signal_column"],  // Output boolean columns
                   "parameters": {
                       "buy_signal_threshold": 0.05,
                       "sell_signal_threshold": -0.05
                   }
           }}
       ],
   
       // === OUTPUTS ===
   
       "output_sets": [
           {"generator": "score_notification_model", "config": {
               "score_notification": true,
               "score_column_names": ["trade_score"],
               "notify_band_up": true,
               "notify_band_dn": true,
               "positive_bands": [
                   {"edge": 0.03, "frequency": null, "sign": "", "text": ""},
                   {"edge": 0.04, "frequency": 3, "sign": "〉", "text": "weak"},
                   {"edge": 0.05, "frequency": 2, "sign": "〉〉", "bold": false, "text": "strong"},
                   {"edge": 1.0, "frequency": 1, "sign": "〉〉〉📈", "bold": true, "text": "BUY ZONE"}
               ],
               "negative_bands": [
                   {"edge": -0.03, "frequency": null, "sign": "", "text": ""},
                   {"edge": -0.04, "frequency": 3, "sign": "〈", "text": "weak"},
                   {"edge": -0.05, "frequency": 2, "sign": "〈〈", "bold": false, "text": "strong"},
                   {"edge": -1.0, "frequency": 1, "sign": "〈〈〈📉", "bold": true, "text": "SELL ZONE"}
               ]
           }},
           {"generator": "diagram_notification_model", "config": {
               // Regularly sending historic data with prices, scores and buy-sell trade decisions
               "diagram_notification": true,
               "notification_freq": "1D",
   
               "score_column_names": "trade_score",
               "score_thresholds":[-0.05, 0.05],
   
               // 5 minutes aggregation and this number of 5 minute intervals
               "resampling_freq": "5min", "nrows": 288
           }},
           {"generator": "trader_simulation", "config": {
               "buy_signal_column":"buy_signal_column",
               "sell_signal_column": "sell_signal_column"
           }}
           //{"generator": "trader_binance", "config": {
           //    "buy_signal_column":"buy_signal_column",
           //    "sell_signal_column": "sell_signal_column"
               //"no_trades_only_data_processing": false,
               //"test_order_before_submit": false,
               //"simulate_order_execution": false,
               //"percentage_used_for_trade": 99.0, // How much should be used in orders
               //"limit_price_adjustment": 0.001, // Limit order price relative to the latest close price
           //}}
       ],
   
       // === FINDING BEST TRADE PARAMETERS ===
   
       "train_signal_model": {
           "data_start": 0,
           "data_end": null,
   
           "direction": "long",
           "topn_to_store": 10,
   
           "signal_generator": "threshold_rule",  // generator in the signal_sets section
           "buy_sell_equal": false,
           "grid": {
               "buy_signal_threshold": [0.02, 0.03, 0.04, 0.05, 0.1, 0.15],
               "sell_signal_threshold": [-0.02, -0.03, -0.04, -0.05, -0.1, -0.15]
           }
       },
   
       "rolling_predict": {
           // int, null or string with date which will be resolved using time_column and removed from source data
           "data_start": "2020-02-01 00:00:00",
           "data_end": null,
   
           // One of these 3 parameters can be null and will be computed from the other two
           "prediction_start": null,  // First row for starting predictions, for example, "2022-02-01 00:00:00"
           "prediction_size": 10080, // How many predictions, for example, 1 week 7*1440
           "prediction_steps": 4, // How many train-prediction steps
   
           "use_multiprocessing": false,
           "max_workers": 8
       }
   }
   
3. Repository owner locked and limited conversation to collaborators on Mar 30, 2025
4. asavinov

   converted this issue into a discussion #57 on Mar 30, 2025