From c96ef67ae09c076c889f010cbabd542fe4340899 Mon Sep 17 00:00:00 2001 From: Surbhi-Vijay Date: Fri, 6 Feb 2026 11:32:47 +0000 Subject: [PATCH 1/6] Fixed GlutenSetCommandSuite --- .../spark40/src/test/resources/hive-site.xml | 35 +++++++++++++++++++ .../utils/velox/VeloxTestSettings.scala | 2 +- .../utils/velox/VeloxTestSettings.scala | 2 +- 3 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 gluten-ut/spark40/src/test/resources/hive-site.xml diff --git a/gluten-ut/spark40/src/test/resources/hive-site.xml b/gluten-ut/spark40/src/test/resources/hive-site.xml new file mode 100644 index 00000000000..9a41dfea1be --- /dev/null +++ b/gluten-ut/spark40/src/test/resources/hive-site.xml @@ -0,0 +1,35 @@ + + + + + + + hive.in.test + true + Internal marker for test. + + + hadoop.tmp.dir + /tmp/hive_one + default is /tmp/hadoop-${user.name} and will be overridden + + + hadoop.this.is.a.test.key + 2018-11-17 13:33:33.333 + + \ No newline at end of file diff --git a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index e94e958a0f9..7cbd95e137a 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -769,7 +769,7 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenRuntimeConfigSuite] enableSuite[GlutenSSBQuerySuite] enableSuite[GlutenSessionStateSuite] - // TODO: 4.x enableSuite[GlutenSetCommandSuite] // 1 failure + enableSuite[GlutenSetCommandSuite] // TODO: 4.x enableSuite[GlutenSingleLevelAggregateHashMapSuite] // 1 failure enableSuite[GlutenSparkSessionBuilderSuite] // TODO: 4.x enableSuite[GlutenSparkSessionJobTaggingAndCancellationSuite] // 1 failure diff --git a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 0f2f10b24d4..b91ed3efe8d 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -734,7 +734,7 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenRuntimeConfigSuite] enableSuite[GlutenSSBQuerySuite] enableSuite[GlutenSessionStateSuite] - // TODO: 4.x enableSuite[GlutenSetCommandSuite] // 1 failure + enableSuite[GlutenSetCommandSuite] // TODO: 4.x enableSuite[GlutenSingleLevelAggregateHashMapSuite] // 1 failure enableSuite[GlutenSparkSessionBuilderSuite] // TODO: 4.x enableSuite[GlutenSparkSessionJobTaggingAndCancellationSuite] // 1 failure From d4712a515afdd35e7eb82f62f640c0b347916181 Mon Sep 17 00:00:00 2001 From: Surbhi-Vijay Date: Fri, 6 Feb 2026 12:10:40 +0000 Subject: [PATCH 2/6] Fixed GlutenSparkPlanSuite --- .../utils/velox/VeloxTestSettings.scala | 6 +++-- .../sql/execution/GlutenSparkPlanSuite.scala | 27 ++++++++++++++++++- .../utils/velox/VeloxTestSettings.scala | 6 +++-- .../sql/execution/GlutenSparkPlanSuite.scala | 27 ++++++++++++++++++- 4 files changed, 60 insertions(+), 6 deletions(-) diff --git a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 7cbd95e137a..4b6d108ed2e 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -248,7 +248,7 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenDataSourceV2MetricsSuite] enableSuite[GlutenDataSourceV2OptionSuite] enableSuite[GlutenDataSourceV2UtilsSuite] - // TODO: 4.x enableSuite[GlutenGroupBasedUpdateTableSuite] // 1 failure + enableSuite[GlutenGroupBasedUpdateTableSuite] // TODO: 4.x enableSuite[GlutenMergeIntoDataFrameSuite] // 1 failure enableSuite[GlutenProcedureSuite] enableSuite[GlutenPushablePredicateSuite] @@ -652,7 +652,9 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenSQLJsonProtocolSuite] enableSuite[GlutenShufflePartitionsUtilSuite] enableSuite[GlutenSimpleSQLViewSuite] - // TODO: 4.x enableSuite[GlutenSparkPlanSuite] // 1 failure + enableSuite[GlutenSparkPlanSuite] + // Rewrite to check Gluten node + .exclude("SPARK-37779: ColumnarToRowExec should be canonicalizable after being (de)serialized") enableSuite[GlutenSparkPlannerSuite] enableSuite[GlutenSparkScriptTransformationSuite] enableSuite[GlutenSparkSqlParserSuite] diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenSparkPlanSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenSparkPlanSuite.scala index a3f0a577d78..9a9d7ddbb18 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenSparkPlanSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenSparkPlanSuite.scala @@ -16,6 +16,31 @@ */ package org.apache.spark.sql.execution +import org.apache.gluten.execution.VeloxColumnarToRowExec import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.internal.SQLConf -class GlutenSparkPlanSuite extends SparkPlanSuite with GlutenSQLTestsTrait {} +class GlutenSparkPlanSuite extends SparkPlanSuite with GlutenSQLTestsTrait { + + testGluten( + "SPARK-37779: ColumnarToRowExec should be canonicalizable after being (de)serialized") { + withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "parquet") { + withTempPath { + path => + spark.range(1).write.parquet(path.getAbsolutePath) + val df = spark.read.parquet(path.getAbsolutePath) + val columnarToRowExec = + df.queryExecution.executedPlan.collectFirst { case p: VeloxColumnarToRowExec => p }.get + try { + spark.range(1).queryExecution.executedPlan.foreach { + _ => + columnarToRowExec.canonicalized + () + } + } catch { + case e: Throwable => fail("VeloxColumnarToRowExec was not canonicalizable", e) + } + } + } + } +} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index b91ed3efe8d..2cb69c4243e 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -259,7 +259,7 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenDataSourceV2MetricsSuite] enableSuite[GlutenDataSourceV2OptionSuite] enableSuite[GlutenDataSourceV2UtilsSuite] - // TODO: 4.x enableSuite[GlutenGroupBasedUpdateTableSuite] // 1 failure + enableSuite[GlutenGroupBasedUpdateTableSuite] // TODO: 4.x enableSuite[GlutenMergeIntoDataFrameSuite] // 1 failure enableSuite[GlutenProcedureSuite] enableSuite[GlutenPushablePredicateSuite] @@ -616,7 +616,9 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenSQLJsonProtocolSuite] enableSuite[GlutenShufflePartitionsUtilSuite] enableSuite[GlutenSimpleSQLViewSuite] - // TODO: 4.x enableSuite[GlutenSparkPlanSuite] // 1 failure + enableSuite[GlutenSparkPlanSuite] + // Rewrite to check Gluten node + .exclude("SPARK-37779: ColumnarToRowExec should be canonicalizable after being (de)serialized") enableSuite[GlutenSparkPlannerSuite] enableSuite[GlutenSparkScriptTransformationSuite] enableSuite[GlutenSparkSqlParserSuite] diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenSparkPlanSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenSparkPlanSuite.scala index a3f0a577d78..9a9d7ddbb18 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenSparkPlanSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenSparkPlanSuite.scala @@ -16,6 +16,31 @@ */ package org.apache.spark.sql.execution +import org.apache.gluten.execution.VeloxColumnarToRowExec import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.internal.SQLConf -class GlutenSparkPlanSuite extends SparkPlanSuite with GlutenSQLTestsTrait {} +class GlutenSparkPlanSuite extends SparkPlanSuite with GlutenSQLTestsTrait { + + testGluten( + "SPARK-37779: ColumnarToRowExec should be canonicalizable after being (de)serialized") { + withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "parquet") { + withTempPath { + path => + spark.range(1).write.parquet(path.getAbsolutePath) + val df = spark.read.parquet(path.getAbsolutePath) + val columnarToRowExec = + df.queryExecution.executedPlan.collectFirst { case p: VeloxColumnarToRowExec => p }.get + try { + spark.range(1).queryExecution.executedPlan.foreach { + _ => + columnarToRowExec.canonicalized + () + } + } catch { + case e: Throwable => fail("VeloxColumnarToRowExec was not canonicalizable", e) + } + } + } + } +} From bdbf9247a54e0c512500c4464e5f699f5e2360bb Mon Sep 17 00:00:00 2001 From: Surbhi-Vijay Date: Fri, 6 Feb 2026 12:25:15 +0000 Subject: [PATCH 3/6] Enable GlutenDisableUnnecessaryBucketedScanWithHiveSupportSuite & GlutenBucketedReadWithHiveSupportSuite --- .../org/apache/gluten/utils/velox/VeloxTestSettings.scala | 4 ++-- .../org/apache/gluten/utils/velox/VeloxTestSettings.scala | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 4b6d108ed2e..46b7bb1c548 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -741,11 +741,11 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenSaveLoadSuite] enableSuite[GlutenTableScanSuite] // Generated suites for org.apache.spark.sql.sources - // TODO: 4.x enableSuite[GlutenBucketedReadWithHiveSupportSuite] // 2 failures + enableSuite[GlutenBucketedReadWithHiveSupportSuite] // TODO: 4.x enableSuite[GlutenBucketedWriteWithHiveSupportSuite] // 1 failure // TODO: 4.x enableSuite[GlutenCommitFailureTestRelationSuite] // 2 failures enableSuite[GlutenDataSourceAnalysisSuite] - // TODO: 4.x enableSuite[GlutenDisableUnnecessaryBucketedScanWithHiveSupportSuite] // 2 failures + enableSuite[GlutenDisableUnnecessaryBucketedScanWithHiveSupportSuite] // TODO: 4.x enableSuite[GlutenJsonHadoopFsRelationSuite] // 2 failures // TODO: 4.x enableSuite[GlutenParquetHadoopFsRelationSuite] // 2 failures // TODO: 4.x enableSuite[GlutenSimpleTextHadoopFsRelationSuite] // 2 failures diff --git a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 2cb69c4243e..68647b767b4 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -707,11 +707,11 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenSaveLoadSuite] enableSuite[GlutenTableScanSuite] // Generated suites for org.apache.spark.sql.sources - // TODO: 4.x enableSuite[GlutenBucketedReadWithHiveSupportSuite] // 2 failures + enableSuite[GlutenBucketedReadWithHiveSupportSuite] // TODO: 4.x enableSuite[GlutenBucketedWriteWithHiveSupportSuite] // 2 failures // TODO: 4.x enableSuite[GlutenCommitFailureTestRelationSuite] // 2 failures enableSuite[GlutenDataSourceAnalysisSuite] - // TODO: 4.x enableSuite[GlutenDisableUnnecessaryBucketedScanWithHiveSupportSuite] // 2 failures + enableSuite[GlutenDisableUnnecessaryBucketedScanWithHiveSupportSuite] // TODO: 4.x enableSuite[GlutenJsonHadoopFsRelationSuite] // 2 failures // TODO: 4.x enableSuite[GlutenParquetHadoopFsRelationSuite] // 2 failures // TODO: 4.x enableSuite[GlutenSimpleTextHadoopFsRelationSuite] // 2 failures From 828f542e93e9ddafbd65738308d03e9191fa5e51 Mon Sep 17 00:00:00 2001 From: Surbhi-Vijay Date: Fri, 6 Feb 2026 12:36:29 +0000 Subject: [PATCH 4/6] Add hive-site.xml --- .../spark41/src/test/resources/hive-site.xml | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 gluten-ut/spark41/src/test/resources/hive-site.xml diff --git a/gluten-ut/spark41/src/test/resources/hive-site.xml b/gluten-ut/spark41/src/test/resources/hive-site.xml new file mode 100644 index 00000000000..9a41dfea1be --- /dev/null +++ b/gluten-ut/spark41/src/test/resources/hive-site.xml @@ -0,0 +1,35 @@ + + + + + + + hive.in.test + true + Internal marker for test. + + + hadoop.tmp.dir + /tmp/hive_one + default is /tmp/hadoop-${user.name} and will be overridden + + + hadoop.this.is.a.test.key + 2018-11-17 13:33:33.333 + + \ No newline at end of file From 655d390c78d74822944f1190324fff5588a07395 Mon Sep 17 00:00:00 2001 From: Surbhi-Vijay Date: Mon, 9 Feb 2026 07:52:51 +0000 Subject: [PATCH 5/6] Fix scalastyle --- .../org/apache/spark/sql/execution/GlutenSparkPlanSuite.scala | 1 + .../org/apache/spark/sql/execution/GlutenSparkPlanSuite.scala | 1 + 2 files changed, 2 insertions(+) diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenSparkPlanSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenSparkPlanSuite.scala index 9a9d7ddbb18..e97613d8a1f 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenSparkPlanSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenSparkPlanSuite.scala @@ -17,6 +17,7 @@ package org.apache.spark.sql.execution import org.apache.gluten.execution.VeloxColumnarToRowExec + import org.apache.spark.sql.GlutenSQLTestsTrait import org.apache.spark.sql.internal.SQLConf diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenSparkPlanSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenSparkPlanSuite.scala index 9a9d7ddbb18..e97613d8a1f 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenSparkPlanSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenSparkPlanSuite.scala @@ -17,6 +17,7 @@ package org.apache.spark.sql.execution import org.apache.gluten.execution.VeloxColumnarToRowExec + import org.apache.spark.sql.GlutenSQLTestsTrait import org.apache.spark.sql.internal.SQLConf From 2e9b11536073b3c32d16508387a4303569a6bb28 Mon Sep 17 00:00:00 2001 From: Surbhi-Vijay Date: Wed, 25 Feb 2026 11:23:29 +0000 Subject: [PATCH 6/6] Disable tests failing in pipeline --- .../org/apache/gluten/utils/velox/VeloxTestSettings.scala | 6 +++--- .../org/apache/gluten/utils/velox/VeloxTestSettings.scala | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 46b7bb1c548..57a04f531db 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -248,7 +248,7 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenDataSourceV2MetricsSuite] enableSuite[GlutenDataSourceV2OptionSuite] enableSuite[GlutenDataSourceV2UtilsSuite] - enableSuite[GlutenGroupBasedUpdateTableSuite] + // TODO: 4.x enableSuite[GlutenGroupBasedUpdateTableSuite] // 1 failure // TODO: 4.x enableSuite[GlutenMergeIntoDataFrameSuite] // 1 failure enableSuite[GlutenProcedureSuite] enableSuite[GlutenPushablePredicateSuite] @@ -741,11 +741,11 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenSaveLoadSuite] enableSuite[GlutenTableScanSuite] // Generated suites for org.apache.spark.sql.sources - enableSuite[GlutenBucketedReadWithHiveSupportSuite] + // TODO: 4.x enableSuite[GlutenBucketedReadWithHiveSupportSuite] // 2 failures // TODO: 4.x enableSuite[GlutenBucketedWriteWithHiveSupportSuite] // 1 failure // TODO: 4.x enableSuite[GlutenCommitFailureTestRelationSuite] // 2 failures enableSuite[GlutenDataSourceAnalysisSuite] - enableSuite[GlutenDisableUnnecessaryBucketedScanWithHiveSupportSuite] + // TODO: 4.x enableSuite[GlutenDisableUnnecessaryBucketedScanWithHiveSupportSuite] // 2 failures // TODO: 4.x enableSuite[GlutenJsonHadoopFsRelationSuite] // 2 failures // TODO: 4.x enableSuite[GlutenParquetHadoopFsRelationSuite] // 2 failures // TODO: 4.x enableSuite[GlutenSimpleTextHadoopFsRelationSuite] // 2 failures diff --git a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 68647b767b4..54227fe7308 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -259,7 +259,7 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenDataSourceV2MetricsSuite] enableSuite[GlutenDataSourceV2OptionSuite] enableSuite[GlutenDataSourceV2UtilsSuite] - enableSuite[GlutenGroupBasedUpdateTableSuite] + // TODO: 4.x enableSuite[GlutenGroupBasedUpdateTableSuite] // 1 failure // TODO: 4.x enableSuite[GlutenMergeIntoDataFrameSuite] // 1 failure enableSuite[GlutenProcedureSuite] enableSuite[GlutenPushablePredicateSuite] @@ -707,11 +707,11 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenSaveLoadSuite] enableSuite[GlutenTableScanSuite] // Generated suites for org.apache.spark.sql.sources - enableSuite[GlutenBucketedReadWithHiveSupportSuite] + // TODO: 4.x enableSuite[GlutenBucketedReadWithHiveSupportSuite] // 2 failures // TODO: 4.x enableSuite[GlutenBucketedWriteWithHiveSupportSuite] // 2 failures // TODO: 4.x enableSuite[GlutenCommitFailureTestRelationSuite] // 2 failures enableSuite[GlutenDataSourceAnalysisSuite] - enableSuite[GlutenDisableUnnecessaryBucketedScanWithHiveSupportSuite] + // TODO: 4.x enableSuite[GlutenDisableUnnecessaryBucketedScanWithHiveSupportSuite] // 2 failures // TODO: 4.x enableSuite[GlutenJsonHadoopFsRelationSuite] // 2 failures // TODO: 4.x enableSuite[GlutenParquetHadoopFsRelationSuite] // 2 failures // TODO: 4.x enableSuite[GlutenSimpleTextHadoopFsRelationSuite] // 2 failures