Skip to content

[VL] Fix GlutenSimpleSQLViewSuite in Spark 4.x #12502

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zml1206 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

[VL] Fix GlutenSimpleSQLViewSuite in Spark 4.x #12502

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -742,7 +742,9 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenSQLFunctionSuite]
enableSuite[GlutenSQLJsonProtocolSuite]
enableSuite[GlutenShufflePartitionsUtilSuite]
// TODO: 4.x enableSuite[GlutenSimpleSQLViewSuite] // 1 failure
enableSuite[GlutenSimpleSQLViewSuite]
// Velox returns a native FILE_NOT_FOUND error instead of Spark's structured error condition.
.exclude("alter temporary view should follow current storeAnalyzedPlanForView config")
enableSuite[GlutenSparkPlanSuite]
.exclude("SPARK-37779: ColumnarToRowExec should be canonicalizable after being (de)serialized")
enableSuite[GlutenSparkPlannerSuite]
Expand Down
43 changes: 41 additions & 2 deletions gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenSQLViewSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,45 @@
*/
package org.apache.spark.sql.execution

import org.apache.spark.sql.GlutenSQLTestsTrait
import org.apache.spark.SparkException
import org.apache.spark.sql.{GlutenSQLTestsTrait, Row}
import org.apache.spark.sql.internal.SQLConf.STORE_ANALYZED_PLAN_FOR_VIEW

class GlutenSimpleSQLViewSuite extends SimpleSQLViewSuite with GlutenSQLTestsTrait {}
class GlutenSimpleSQLViewSuite extends SimpleSQLViewSuite with GlutenSQLTestsTrait {
import testImplicits._

private def assertMissingFileError(f: => Unit): Unit = {
val exception = intercept[SparkException](f)
val messages = Iterator
.iterate[Throwable](exception)(_.getCause)
.takeWhile(_ != null)
.flatMap(e => Option(e.getMessage))

assert(messages.exists(_.contains("FILE_NOT_FOUND")))
}
Comment on lines +26 to +34

testGluten("alter temporary view should follow current storeAnalyzedPlanForView config") {
withTable("t") {
Seq(2, 3, 1).toDF("c1").write.format("parquet").saveAsTable("t")
withView("v1") {
withSQLConf(STORE_ANALYZED_PLAN_FOR_VIEW.key -> "true") {
sql("CREATE TEMPORARY VIEW v1 AS SELECT * FROM t")
Seq(4, 6, 5).toDF("c1").write.mode("overwrite").format("parquet").saveAsTable("t")
assertMissingFileError(sql("SELECT * FROM v1").collect())
}

withSQLConf(STORE_ANALYZED_PLAN_FOR_VIEW.key -> "false") {
sql("ALTER VIEW v1 AS SELECT * FROM t")
Seq(1, 3, 5).toDF("c1").write.mode("overwrite").format("parquet").saveAsTable("t")
checkAnswer(sql("SELECT * FROM v1"), Seq(Row(1), Row(3), Row(5)))
}

withSQLConf(STORE_ANALYZED_PLAN_FOR_VIEW.key -> "true") {
sql("ALTER VIEW v1 AS SELECT * FROM t")
Seq(2, 4, 6).toDF("c1").write.mode("overwrite").format("parquet").saveAsTable("t")
assertMissingFileError(sql("SELECT * FROM v1").collect())
}
}
}
}
}
4 changes: 3 additions & 1 deletion gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -721,7 +721,9 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenSQLFunctionSuite]
enableSuite[GlutenSQLJsonProtocolSuite]
enableSuite[GlutenShufflePartitionsUtilSuite]
// TODO: 4.x enableSuite[GlutenSimpleSQLViewSuite] // 1 failure remains after GLUTEN-11917 fix
enableSuite[GlutenSimpleSQLViewSuite]
// Velox returns a native FILE_NOT_FOUND error instead of Spark's structured error condition.
.exclude("alter temporary view should follow current storeAnalyzedPlanForView config")
enableSuite[GlutenSparkPlanSuite]
.exclude("SPARK-37779: ColumnarToRowExec should be canonicalizable after being (de)serialized")
enableSuite[GlutenSparkPlannerSuite]
Expand Down
43 changes: 41 additions & 2 deletions gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenSQLViewSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,45 @@
*/
package org.apache.spark.sql.execution

import org.apache.spark.sql.GlutenSQLTestsTrait
import org.apache.spark.SparkException
import org.apache.spark.sql.{GlutenSQLTestsTrait, Row}
import org.apache.spark.sql.internal.SQLConf.STORE_ANALYZED_PLAN_FOR_VIEW

class GlutenSimpleSQLViewSuite extends SimpleSQLViewSuite with GlutenSQLTestsTrait {}
class GlutenSimpleSQLViewSuite extends SimpleSQLViewSuite with GlutenSQLTestsTrait {
import testImplicits._

private def assertMissingFileError(f: => Unit): Unit = {
val exception = intercept[SparkException](f)
val messages = Iterator
.iterate[Throwable](exception)(_.getCause)
.takeWhile(_ != null)
.flatMap(e => Option(e.getMessage))

assert(messages.exists(_.contains("FILE_NOT_FOUND")))
}
Comment on lines +26 to +34

testGluten("alter temporary view should follow current storeAnalyzedPlanForView config") {
withTable("t") {
Seq(2, 3, 1).toDF("c1").write.format("parquet").saveAsTable("t")
withView("v1") {
withSQLConf(STORE_ANALYZED_PLAN_FOR_VIEW.key -> "true") {
sql("CREATE TEMPORARY VIEW v1 AS SELECT * FROM t")
Seq(4, 6, 5).toDF("c1").write.mode("overwrite").format("parquet").saveAsTable("t")
assertMissingFileError(sql("SELECT * FROM v1").collect())
}

withSQLConf(STORE_ANALYZED_PLAN_FOR_VIEW.key -> "false") {
sql("ALTER VIEW v1 AS SELECT * FROM t")
Seq(1, 3, 5).toDF("c1").write.mode("overwrite").format("parquet").saveAsTable("t")
checkAnswer(sql("SELECT * FROM v1"), Seq(Row(1), Row(3), Row(5)))
}

withSQLConf(STORE_ANALYZED_PLAN_FOR_VIEW.key -> "true") {
sql("ALTER VIEW v1 AS SELECT * FROM t")
Seq(2, 4, 6).toDF("c1").write.mode("overwrite").format("parquet").saveAsTable("t")
assertMissingFileError(sql("SELECT * FROM v1").collect())
}
}
}
}
}
Loading