improve fallback reporting

Author: andygrove

common/src/main/scala/org/apache/comet/CometConf.scala

@@ -600,12 +600,6 @@ object CometConf extends ShimCometConf {
       .toSequence
       .createWithDefault(Seq("Range,InMemoryTableScan"))
 
-  val COMET_IGNORE_ANSI_MODE: ConfigEntry[Boolean] = conf("spark.comet.ansi.ignore")
-    .internal()
-    .doc("Internal config to avoid falling back to Spark when ANSI is enabled. Used for testing.")
-    .booleanConf
-    .createWithDefault(false)
-
   val COMET_CASE_CONVERSION_ENABLED: ConfigEntry[Boolean] =
     conf("spark.comet.caseConversion.enabled")
       .doc(
@@ -630,6 +624,14 @@ object CometConf extends ShimCometConf {
       .booleanConf
       .createWithDefault(false)
 
+  val COMET_ANSI_EXPR_ALLOW_INCOMPATIBLE: ConfigEntry[Boolean] =
+    conf("spark.comet.expression.allowIncompatibleAnsi")
+      .doc(
+        "Comet is not currently fully compatible with Spark for all expressions in ANSI mode. " +
+          s"Set this config to true to allow them anyway. $COMPAT_GUIDE.")
+      .booleanConf
+      .createWithDefault(false)
+
   val COMET_CAST_ALLOW_INCOMPATIBLE: ConfigEntry[Boolean] =
     conf("spark.comet.cast.allowIncompatible")
       .doc(

spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala

@@ -646,8 +646,12 @@ object QueryPlanSerde extends Logging with CometExprShim {
       handler match {
         case ansiExpr: IncompatAnsiExpr[T]
             if ansiExpr
-              .isAnsiMode(expr.asInstanceOf[T]) && !CometConf.COMET_IGNORE_ANSI_MODE.get() =>
-          withInfo(expr, "ANSI mode not supported")
+              .isAnsiMode(expr.asInstanceOf[T]) && !CometConf.COMET_ANSI_EXPR_ALLOW_INCOMPATIBLE
+              .get() =>
+          withInfo(
+            expr,
+            s"$expr is not fully compatible with Spark in ANSI mode. To enable it anyway, set " +
+              s"${CometConf.COMET_ANSI_EXPR_ALLOW_INCOMPATIBLE.key}=true. ${CometConf.COMPAT_GUIDE}.")
           None
 
         case _: IncompatExpr if !CometConf.COMET_EXPR_ALLOW_INCOMPATIBLE.get() =>

spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala

@@ -269,7 +269,7 @@ trait CometPlanStabilitySuite extends DisableAdaptiveExecutionSuite with TPCDSBa
       CometConf.COMET_ENABLED.key -> "true",
       CometConf.COMET_NATIVE_SCAN_ENABLED.key -> "true",
       CometConf.COMET_EXEC_ENABLED.key -> "true",
-      CometConf.COMET_IGNORE_ANSI_MODE.key -> "true",
+      CometConf.COMET_ANSI_EXPR_ALLOW_INCOMPATIBLE.key -> "true",
       CometConf.COMET_DPP_FALLBACK_ENABLED.key -> "false",
       SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> dppEnabled.toString,
       CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true",