diff --git a/gluten-ut/spark40/src/test/resources/hive-site.xml b/gluten-ut/spark40/src/test/resources/hive-site.xml new file mode 100644 index 00000000000..9a41dfea1be --- /dev/null +++ b/gluten-ut/spark40/src/test/resources/hive-site.xml @@ -0,0 +1,35 @@ + + + + + + + hive.in.test + true + Internal marker for test. + + + hadoop.tmp.dir + /tmp/hive_one + default is /tmp/hadoop-${user.name} and will be overridden + + + hadoop.this.is.a.test.key + 2018-11-17 13:33:33.333 + + \ No newline at end of file diff --git a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index e94e958a0f9..57a04f531db 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -652,7 +652,9 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenSQLJsonProtocolSuite] enableSuite[GlutenShufflePartitionsUtilSuite] enableSuite[GlutenSimpleSQLViewSuite] - // TODO: 4.x enableSuite[GlutenSparkPlanSuite] // 1 failure + enableSuite[GlutenSparkPlanSuite] + // Rewrite to check Gluten node + .exclude("SPARK-37779: ColumnarToRowExec should be canonicalizable after being (de)serialized") enableSuite[GlutenSparkPlannerSuite] enableSuite[GlutenSparkScriptTransformationSuite] enableSuite[GlutenSparkSqlParserSuite] @@ -769,7 +771,7 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenRuntimeConfigSuite] enableSuite[GlutenSSBQuerySuite] enableSuite[GlutenSessionStateSuite] - // TODO: 4.x enableSuite[GlutenSetCommandSuite] // 1 failure + enableSuite[GlutenSetCommandSuite] // TODO: 4.x enableSuite[GlutenSingleLevelAggregateHashMapSuite] // 1 failure enableSuite[GlutenSparkSessionBuilderSuite] // TODO: 4.x enableSuite[GlutenSparkSessionJobTaggingAndCancellationSuite] // 1 failure diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenSparkPlanSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenSparkPlanSuite.scala index a3f0a577d78..e97613d8a1f 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenSparkPlanSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenSparkPlanSuite.scala @@ -16,6 +16,32 @@ */ package org.apache.spark.sql.execution +import org.apache.gluten.execution.VeloxColumnarToRowExec + import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.internal.SQLConf + +class GlutenSparkPlanSuite extends SparkPlanSuite with GlutenSQLTestsTrait { -class GlutenSparkPlanSuite extends SparkPlanSuite with GlutenSQLTestsTrait {} + testGluten( + "SPARK-37779: ColumnarToRowExec should be canonicalizable after being (de)serialized") { + withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "parquet") { + withTempPath { + path => + spark.range(1).write.parquet(path.getAbsolutePath) + val df = spark.read.parquet(path.getAbsolutePath) + val columnarToRowExec = + df.queryExecution.executedPlan.collectFirst { case p: VeloxColumnarToRowExec => p }.get + try { + spark.range(1).queryExecution.executedPlan.foreach { + _ => + columnarToRowExec.canonicalized + () + } + } catch { + case e: Throwable => fail("VeloxColumnarToRowExec was not canonicalizable", e) + } + } + } + } +} diff --git a/gluten-ut/spark41/src/test/resources/hive-site.xml b/gluten-ut/spark41/src/test/resources/hive-site.xml new file mode 100644 index 00000000000..9a41dfea1be --- /dev/null +++ b/gluten-ut/spark41/src/test/resources/hive-site.xml @@ -0,0 +1,35 @@ + + + + + + + hive.in.test + true + Internal marker for test. + + + hadoop.tmp.dir + /tmp/hive_one + default is /tmp/hadoop-${user.name} and will be overridden + + + hadoop.this.is.a.test.key + 2018-11-17 13:33:33.333 + + \ No newline at end of file diff --git a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 0f2f10b24d4..54227fe7308 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -616,7 +616,9 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenSQLJsonProtocolSuite] enableSuite[GlutenShufflePartitionsUtilSuite] enableSuite[GlutenSimpleSQLViewSuite] - // TODO: 4.x enableSuite[GlutenSparkPlanSuite] // 1 failure + enableSuite[GlutenSparkPlanSuite] + // Rewrite to check Gluten node + .exclude("SPARK-37779: ColumnarToRowExec should be canonicalizable after being (de)serialized") enableSuite[GlutenSparkPlannerSuite] enableSuite[GlutenSparkScriptTransformationSuite] enableSuite[GlutenSparkSqlParserSuite] @@ -734,7 +736,7 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenRuntimeConfigSuite] enableSuite[GlutenSSBQuerySuite] enableSuite[GlutenSessionStateSuite] - // TODO: 4.x enableSuite[GlutenSetCommandSuite] // 1 failure + enableSuite[GlutenSetCommandSuite] // TODO: 4.x enableSuite[GlutenSingleLevelAggregateHashMapSuite] // 1 failure enableSuite[GlutenSparkSessionBuilderSuite] // TODO: 4.x enableSuite[GlutenSparkSessionJobTaggingAndCancellationSuite] // 1 failure diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenSparkPlanSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenSparkPlanSuite.scala index a3f0a577d78..e97613d8a1f 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenSparkPlanSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenSparkPlanSuite.scala @@ -16,6 +16,32 @@ */ package org.apache.spark.sql.execution +import org.apache.gluten.execution.VeloxColumnarToRowExec + import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.internal.SQLConf + +class GlutenSparkPlanSuite extends SparkPlanSuite with GlutenSQLTestsTrait { -class GlutenSparkPlanSuite extends SparkPlanSuite with GlutenSQLTestsTrait {} + testGluten( + "SPARK-37779: ColumnarToRowExec should be canonicalizable after being (de)serialized") { + withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "parquet") { + withTempPath { + path => + spark.range(1).write.parquet(path.getAbsolutePath) + val df = spark.read.parquet(path.getAbsolutePath) + val columnarToRowExec = + df.queryExecution.executedPlan.collectFirst { case p: VeloxColumnarToRowExec => p }.get + try { + spark.range(1).queryExecution.executedPlan.foreach { + _ => + columnarToRowExec.canonicalized + () + } + } catch { + case e: Throwable => fail("VeloxColumnarToRowExec was not canonicalizable", e) + } + } + } + } +}