Fix log_training_metric issue for time series models

- Modified _eval_estimator in ml.py to skip computing training metrics for TimeSeriesDataset
- Added test case test_log_training_metric_ts_models to validate the fix

Co-authored-by: thinkall <3197038+thinkall@users.noreply.github.com>

Author: Copilot

flaml/automl/ml.py

@@ -616,15 +616,18 @@ def _eval_estimator(
             logger.warning(f"ValueError {e} happened in `metric_loss_score`, set `val_loss` to `np.inf`")
         metric_for_logging = {"pred_time": pred_time}
         if log_training_metric:
-            train_pred_y = get_y_pred(estimator, X_train, eval_metric, task)
-            metric_for_logging["train_loss"] = metric_loss_score(
-                eval_metric,
-                train_pred_y,
-                y_train,
-                labels,
-                fit_kwargs.get("sample_weight"),
-                fit_kwargs.get("groups"),
-            )
+            # For time series tasks, skip computing training metrics as predicting on training data
+            # doesn't work the same way as for regular ML models (TimeSeriesDataset needs proper test_data)
+            if not isinstance(X_train, TimeSeriesDataset):
+                train_pred_y = get_y_pred(estimator, X_train, eval_metric, task)
+                metric_for_logging["train_loss"] = metric_loss_score(
+                    eval_metric,
+                    train_pred_y,
+                    y_train,
+                    labels,
+                    fit_kwargs.get("sample_weight"),
+                    fit_kwargs.get("groups"),
+                )
     else:  # customized metric function
         val_loss, metric_for_logging = eval_metric(
             X_val,

test/automl/test_forecast.py

@@ -681,6 +681,41 @@ def split_by_date(df: pd.DataFrame, dt: datetime.date):
     print("yahoo!")
 
 
+def test_log_training_metric_ts_models():
+    """Test that log_training_metric=True works with time series models (arima, sarimax, holt-winters)."""
+    import statsmodels.api as sm
+
+    # Prepare data
+    data = sm.datasets.co2.load_pandas().data["co2"].resample("MS").mean()
+    data = data.bfill().ffill().to_frame().reset_index().rename(columns={"index": "ds", "co2": "y"})
+    num_samples = data.shape[0]
+    time_horizon = 12
+    split_idx = num_samples - time_horizon
+    df = data[:split_idx]
+
+    # Test each time series model with log_training_metric=True
+    for estimator in ["arima", "sarimax", "holt-winters"]:
+        print(f"\nTesting {estimator} with log_training_metric=True")
+        automl = AutoML()
+        settings = {
+            "time_budget": 5,
+            "metric": "mape",
+            "task": "ts_forecast",
+            "eval_method": "holdout",
+            "label": "y",
+            "log_training_metric": True,  # This should not cause errors
+            "estimator_list": [estimator],
+        }
+        
+        try:
+            automl.fit(dataframe=df, **settings, period=time_horizon)
+            print(f"  ✅ {estimator} SUCCESS with log_training_metric=True")
+            assert automl.best_estimator == estimator
+        except Exception as e:
+            print(f"  ❌ {estimator} FAILED: {type(e).__name__}: {e}")
+            raise
+
+
 if __name__ == "__main__":
     # test_forecast_automl(60)
     # test_multivariate_forecast_num(5)