Fix anomaly detection validation and metrics

Author: rashidrao-pk

flaml/automl/automl.py

@@ -2513,11 +2513,7 @@ def cv_score_agg_func(val_loss_folds, log_metrics_folds):
             groups_val,
             groups,
         )
-        unlabeled_anomaly = (
-            task.is_anomaly_detection()
-            and self._y_train_all is None
-            and self._state.y_val is None
-        )
+        unlabeled_anomaly = task.is_anomaly_detection() and self._y_train_all is None and self._state.y_val is None
 
         self._search_states = {}  # key: estimator name; value: SearchState
         self._random = np.random.RandomState(RANDOM_SEED)
@@ -2603,6 +2599,12 @@ def cv_score_agg_func(val_loss_folds, log_metrics_folds):
         # Validate metric parameter before processing
         self._validate_metric_parameter(metric, allow_auto=True)
 
+        if task.is_anomaly_detection() and callable(metric):
+            raise ValueError(
+                "Callable metrics are not currently supported for anomaly_detection. "
+                "Use the built-in 'ap' or 'roc_auc' metric."
+            )
+
         metric = task.default_metric(metric)
 
         if task.is_anomaly_detection() and isinstance(metric, str) and metric not in ["ap", "roc_auc"]:
@@ -2707,6 +2709,22 @@ def is_to_reverse_metric(metric, task):
         elif max_iter is None:
             # set to a large number
             max_iter = 1000000
+
+        if task.is_anomaly_detection() and max_iter > 1:
+            from flaml.automl.ml import normalize_anomaly_labels
+
+            evaluation_labels = self._state.y_val if eval_method == "holdout" else self._y_train_all
+            if evaluation_labels is None:
+                raise ValueError(
+                    "Hyperparameter search for anomaly_detection requires labeled "
+                    "evaluation data containing both normal and anomaly samples."
+                )
+            normalized_labels = normalize_anomaly_labels(evaluation_labels)
+            if set(np.unique(normalized_labels)) != {0, 1}:
+                raise ValueError(
+                    "Hyperparameter search for anomaly_detection requires evaluation "
+                    "labels containing both normal and anomaly classes."
+                )
         self._state.retrain_final = (
             retrain_full is True
             and eval_method == "holdout"

flaml/automl/data.py

@@ -394,6 +394,11 @@ def fit_transform(self, X: Union[DataFrame, np.ndarray], y, task: Union[str, "Ta
             self._drop = drop
         if y is None:
             self.label_transformer = None
+        elif task.is_anomaly_detection() and not pd.api.types.is_numeric_dtype(y):
+            raise ValueError(
+                "Anomaly detection labels must be numeric and use either "
+                "{0, 1} with 1=anomaly or {-1, 1} with -1=anomaly."
+            )
         elif task.is_classification() or not pd.api.types.is_numeric_dtype(y) and not task.is_nlg():
             if not task.is_token_classification():
                 from sklearn.preprocessing import LabelEncoder

flaml/automl/ml.py

@@ -308,8 +308,7 @@ def normalize_anomaly_labels(y):
         return y
 
     raise ValueError(
-        "For anomaly detection with AP/ROC-AUC, labels must use "
-        "{0, 1} with 1=anomaly or {-1, 1} with -1=anomaly."
+        "For anomaly detection with AP/ROC-AUC, labels must use " "{0, 1} with 1=anomaly or {-1, 1} with -1=anomaly."
     )
 
 
@@ -636,12 +635,17 @@ def _eval_estimator(
 
         pred_time = (time.time() - pred_start) / num_val_rows
 
-        try:
-            y_val_for_metric = (
-                normalize_anomaly_labels(y_val)
-                if task.is_anomaly_detection() and eval_metric in ["ap", "roc_auc"]
-                else y_val
-            )
+        if task.is_anomaly_detection() and eval_metric in ["ap", "roc_auc"]:
+            y_val_for_metric = normalize_anomaly_labels(y_val)
+            if set(np.unique(y_val_for_metric)) != {0, 1}:
+                raise ValueError(
+                    "Anomaly detection evaluation with AP/ROC-AUC requires "
+                    "both normal and anomaly labels after normalization."
+                )
+        else:
+            y_val_for_metric = y_val
+
+        try:
             val_loss = metric_loss_score(
                 eval_metric,
                 y_processed_predict=val_pred_y,
@@ -655,7 +659,7 @@ def _eval_estimator(
             val_loss = np.inf
             logger.warning(f"ValueError {e} happened in `metric_loss_score`, set `val_loss` to `np.inf`")
         metric_for_logging = {"pred_time": pred_time}
-        if log_training_metric:
+        if log_training_metric and y_train is not None:
             # For time series forecasting, X_train may be a sampled dataset whose
             # test partition can be empty. Use the training partition from X_val
             # (which is the dataset used to define y_train above) to keep shapes

flaml/automl/task/generic_task.py

@@ -122,6 +122,14 @@ def validate_data(
             )
             assert X_train_all.size != 0, "Input data must not be empty."
 
+            if self.is_anomaly_detection():
+                for label_name, labels in (("y_train", y_train_all), ("y_val", y_val)):
+                    if labels is not None and not pd.api.types.is_numeric_dtype(labels):
+                        raise ValueError(
+                            f"{label_name} for anomaly_detection must be numeric and use either "
+                            "{0, 1} with 1=anomaly or {-1, 1} with -1=anomaly."
+                        )
+
             if y_train_all is not None:
                 assert isinstance(
                     y_train_all, (np.ndarray, pd.Series, psSeries)
@@ -272,9 +280,7 @@ def validate_data(
             state.groups = groups
 
         automl.data_size_full = (
-            len(automl._y_train_all)
-            if automl._y_train_all is not None
-            else automl._X_train_all.shape[0]
+            len(automl._y_train_all) if automl._y_train_all is not None else automl._X_train_all.shape[0]
         )
 
     @staticmethod
@@ -1342,10 +1348,7 @@ def default_estimator_list(self, estimator_list: List[str], is_spark_dataframe:
                         "Use 'isolation_forest' or a compatible custom learner."
                     )
             elif "isolation_forest" in estimator_list:
-                raise ValueError(
-                    "Built-in estimator 'isolation_forest' only supports "
-                    "the anomaly_detection task."
-                )
+                raise ValueError("Built-in estimator 'isolation_forest' only supports " "the anomaly_detection task.")
             n_estimators = len(estimator_list)
             if is_spark_dataframe:
                 # For spark dataframe, only estimators ending with '_spark' are supported

test/automl/test_anomaly_detection.py

@@ -17,6 +17,8 @@ def test_anomaly_detection_rejects_unsupported_builtin_estimator():
             estimator_list=["lgbm"],
             time_budget=1,
         )
+
+
 def test_isolation_forest_rejects_non_anomaly_task():
     automl = AutoML()
 
@@ -32,6 +34,7 @@ def test_isolation_forest_rejects_non_anomaly_task():
             time_budget=1,
         )
 
+
 def test_automl_anomaly_detection_e2e():
     X_normal, _ = make_blobs(
         n_samples=100,
@@ -181,3 +184,119 @@ def test_normalize_anomaly_labels_single_class():
         normalize_anomaly_labels(np.array([-1, 1, 1])),
         np.array([1, 0, 0]),
     )
+
+
+def test_anomaly_detection_hpo_rejects_single_class_evaluation_labels():
+    rng = np.random.RandomState(42)
+    X_train = rng.randn(80, 2)
+    X_val = rng.randn(20, 2)
+
+    with pytest.raises(
+        ValueError,
+        match="requires evaluation labels containing both normal and anomaly classes",
+    ):
+        AutoML().fit(
+            X_train=X_train,
+            y_train=np.zeros(len(X_train)),
+            X_val=X_val,
+            y_val=np.zeros(len(X_val)),
+            task="anomaly_detection",
+            estimator_list=["isolation_forest"],
+            metric="roc_auc",
+            max_iter=2,
+        )
+
+
+def test_anomaly_detection_x_only_hpo_with_training_metric_logging():
+    X_normal, _ = make_blobs(
+        n_samples=100,
+        centers=1,
+        cluster_std=0.5,
+        random_state=42,
+    )
+    rng = np.random.RandomState(42)
+    X_anomaly = rng.uniform(low=6, high=8, size=(20, 2))
+
+    X_val = np.vstack([X_normal[:20], X_anomaly])
+    y_val = np.array([0] * 20 + [1] * 20)
+
+    automl = AutoML()
+    automl.fit(
+        X_train=X_normal,
+        X_val=X_val,
+        y_val=y_val,
+        task="anomaly_detection",
+        estimator_list=["isolation_forest"],
+        metric="roc_auc",
+        max_iter=2,
+        log_training_metric=True,
+    )
+
+    assert automl.best_estimator == "isolation_forest"
+    assert automl.score(X_val, y_val) > 0.9
+
+
+@pytest.mark.parametrize(
+    "label_location",
+    ["train", "validation"],
+)
+def test_anomaly_detection_rejects_nonnumeric_labels(label_location):
+    rng = np.random.RandomState(42)
+    X_train = rng.randn(40, 2)
+
+    if label_location == "train":
+        with pytest.raises(
+            ValueError,
+            match="y_train for anomaly_detection must be numeric",
+        ):
+            AutoML().fit(
+                X_train=X_train,
+                y_train=np.array(["normal"] * 40),
+                task="anomaly_detection",
+                estimator_list=["isolation_forest"],
+                max_iter=1,
+            )
+    else:
+        X_val = rng.randn(20, 2)
+        y_val = np.array(["normal"] * 10 + ["anomaly"] * 10)
+
+        with pytest.raises(
+            ValueError,
+            match="y_val for anomaly_detection must be numeric",
+        ):
+            AutoML().fit(
+                X_train=X_train,
+                X_val=X_val,
+                y_val=y_val,
+                task="anomaly_detection",
+                estimator_list=["isolation_forest"],
+                max_iter=2,
+            )
+
+
+def test_anomaly_detection_rejects_callable_metric():
+    X = np.random.RandomState(42).randn(50, 2)
+
+    def custom_metric(
+        X_val,
+        y_val,
+        estimator,
+        labels,
+        X_train,
+        y_train,
+        *args,
+        **kwargs,
+    ):
+        return 0.0, {}
+
+    with pytest.raises(
+        ValueError,
+        match="Callable metrics are not currently supported for anomaly_detection",
+    ):
+        AutoML().fit(
+            X_train=X,
+            task="anomaly_detection",
+            estimator_list=["isolation_forest"],
+            metric=custom_metric,
+            max_iter=1,
+        )