Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
*/
package org.apache.gluten.execution

import org.apache.gluten.exception.GlutenException
import org.apache.gluten.extension.columnar.transition.{Convention, ConventionReq}

import org.apache.spark.sql.execution.SparkPlan
Expand All @@ -26,7 +25,7 @@ import org.apache.spark.sql.execution.SparkPlan
*
* The following Spark APIs are marked final so forbidden from overriding:
* - supportsColumnar
* - supportsRowBased (Spark version >= 3.3)
* - supportsRowBased
*
* Instead, subclasses are expected to implement the following APIs:
* - batchType
Expand All @@ -46,7 +45,6 @@ trait GlutenPlan
extends SparkPlan
with Convention.KnownBatchType
with Convention.KnownRowType
with GlutenPlan.SupportsRowBasedCompatible
with ConventionReq.KnownChildConvention {

final override val supportsColumnar: Boolean = {
Expand Down Expand Up @@ -77,12 +75,3 @@ trait GlutenPlan
})
}
}

object GlutenPlan {
// To be compatible with Spark (version < 3.3)
trait SupportsRowBasedCompatible {
def supportsRowBased(): Boolean = {
throw new GlutenException("Illegal state: The method is not expected to be called")
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.execution._
import org.apache.spark.sql.execution.adaptive.AQEShuffleReadExec
import org.apache.spark.sql.execution.debug.DebugExec
import org.apache.spark.util.SparkVersionUtil

package object transition {

Expand All @@ -35,8 +34,7 @@ package object transition {
// Extend this list in shim layer once Spark has more.
def canPropagateConvention(plan: SparkPlan): Boolean = plan match {
case p: DebugExec => true
case p: UnionExec if SparkVersionUtil.gteSpark33 =>
true
case p: UnionExec => true
case p: AQEShuffleReadExec => true
case p: InputAdapter => true
case p: WholeStageCodegenExec => true
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,7 @@ import org.apache.spark.sql.internal.SQLConf
object SparkPlanUtil {

def supportsRowBased(plan: SparkPlan): Boolean = {
val m = classOf[SparkPlan].getMethod("supportsRowBased")
m.invoke(plan).asInstanceOf[Boolean]
plan.supportsRowBased
}

def isPlannedV1Write(plan: DataWritingCommandExec): Boolean = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ object SparkVersionUtil {
private val comparedWithSpark33 = compareMajorMinorVersion((3, 3))
private val comparedWithSpark35 = compareMajorMinorVersion((3, 5))
val eqSpark33: Boolean = comparedWithSpark33 == 0
val gteSpark33: Boolean = comparedWithSpark33 >= 0
val gteSpark35: Boolean = comparedWithSpark35 >= 0
val gteSpark40: Boolean = compareMajorMinorVersion((4, 0)) >= 0
val gteSpark41: Boolean = compareMajorMinorVersion((4, 1)) >= 0
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,6 @@ case class ColumnarInputAdapter(child: SparkPlan)
extends InputAdapterGenerateTreeStringShim
with Convention.KnownBatchType
with Convention.KnownRowType
with GlutenPlan.SupportsRowBasedCompatible
with ConventionReq.KnownChildConvention {
override def output: Seq[Attribute] = child.output
final override val supportsColumnar: Boolean = true
Expand Down
Loading