Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
/*
* Copyright (c) 2026, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

/*** spark-rapids-shim-json-lines
{"spark": "330db"}
{"spark": "331"}
{"spark": "332"}
{"spark": "332db"}
{"spark": "333"}
{"spark": "334"}
{"spark": "340"}
{"spark": "341"}
{"spark": "341db"}
{"spark": "342"}
{"spark": "343"}
{"spark": "344"}
{"spark": "350"}
{"spark": "350db143"}
{"spark": "351"}
{"spark": "352"}
{"spark": "353"}
{"spark": "354"}
{"spark": "355"}
{"spark": "356"}
{"spark": "357"}
{"spark": "358"}
{"spark": "400"}
{"spark": "400db173"}
{"spark": "401"}
{"spark": "402"}
{"spark": "411"}
spark-rapids-shim-json-lines ***/
package com.nvidia.spark.rapids.shims

import com.nvidia.spark.rapids.{
ExprChecks,
ExprRule,
GpuCast,
GpuExpression,
GpuOverrides,
TypeSig,
UnaryExprMeta
}

import org.apache.spark.sql.catalyst.expressions.{CheckOverflowInTableInsert, Expression}
import org.apache.spark.sql.rapids.GpuCheckOverflowInTableInsert

object CheckOverflowInTableInsertShims {
val exprs: Map[Class[_ <: Expression], ExprRule[_ <: Expression]] = {
Seq(
// Add expression CheckOverflowInTableInsert starting Spark-3.3.1+.
// Accepts all types as input as the child Cast does the type checking and the calculations.
GpuOverrides.expr[CheckOverflowInTableInsert](
"Casting a numeric value as another numeric type in store assignment",
ExprChecks.unaryProjectInputMatchesOutput(
TypeSig.all,
TypeSig.all),
(t, conf, p, r) => new UnaryExprMeta[CheckOverflowInTableInsert](t, conf, p, r) {
override def convertToGpu(child: Expression): GpuExpression = {
child match {
case c: GpuCast => GpuCheckOverflowInTableInsert(c, t.columnName)
case _ =>
throw new IllegalStateException("Expression child is not of Type GpuCast")
}
}
})
).map(r => (r.getClassFor.asSubclass(classOf[Expression]), r)).toMap
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ trait Spark321PlusDBShims extends SparkShims
TypeSig.all,
Map("partitionSpec" ->
new InputCheck(TypeSig.commonCudfTypes + TypeSig.NULL + TypeSig.DECIMAL_128,
TypeSig.all, Nil))),
TypeSig.all, List.empty))),
(runningWindowFunctionExec, conf, p, r) =>
new GpuRunningWindowExecMeta(runningWindowFunctionExec, conf, p, r)
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,29 +30,16 @@ import org.apache.spark.sql.catalyst.plans.physical.SinglePartition
import org.apache.spark.sql.execution.{ColumnarToRowTransition, SparkPlan}
import org.apache.spark.sql.execution.adaptive.ShuffleQueryStageExec
import org.apache.spark.sql.execution.exchange.{EXECUTOR_BROADCAST, ShuffleExchangeExec, ShuffleExchangeLike}
import org.apache.spark.sql.rapids.{GpuCheckOverflowInTableInsert, GpuElementAtMeta}
import org.apache.spark.sql.rapids.GpuElementAtMeta
import org.apache.spark.sql.rapids.execution.{GpuBroadcastHashJoinExec, GpuBroadcastNestedLoopJoinExec}

trait Spark330PlusDBShims extends Spark321PlusDBShims {
override def getExprs: Map[Class[_ <: Expression], ExprRule[_ <: Expression]] = {
val shimExprs: Map[Class[_ <: Expression], ExprRule[_ <: Expression]] = Seq(
GpuOverrides.expr[CheckOverflowInTableInsert](
"Casting a numeric value as another numeric type in store assignment",
ExprChecks.unaryProjectInputMatchesOutput(
TypeSig.all,
TypeSig.all),
(t, conf, p, r) => new UnaryExprMeta[CheckOverflowInTableInsert](t, conf, p, r) {
override def convertToGpu(child: Expression): GpuExpression = {
child match {
case c: GpuCast => GpuCheckOverflowInTableInsert(c, t.columnName)
case _ =>
throw new IllegalStateException("Expression child is not of Type GpuCast")
}
}
}),
GpuElementAtMeta.elementAtRule(true)
).map(r => (r.getClassFor.asSubclass(classOf[Expression]), r)).toMap
super.getExprs ++ shimExprs ++ DayTimeIntervalShims.exprs ++ RoundingShims.exprs
super.getExprs ++ CheckOverflowInTableInsertShims.exprs ++ shimExprs ++
DayTimeIntervalShims.exprs ++ RoundingShims.exprs
}

override def getExecs: Map[Class[_ <: SparkPlan], ExecRule[_ <: SparkPlan]] =
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2022-2023, NVIDIA CORPORATION.
* Copyright (c) 2022-2026, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -22,17 +22,18 @@ package com.nvidia.spark.rapids.shims
import com.nvidia.spark.rapids._

import org.apache.spark.sql.catalyst.expressions.Expression
import org.apache.spark.sql.execution.command.{CreateDataSourceTableAsSelectCommand, DataWritingCommand, RunnableCommand}
import org.apache.spark.sql.execution.command.{DataWritingCommand, RunnableCommand}

object SparkShimImpl extends Spark330PlusDBShims {
// AnsiCast is removed from Spark3.4.0
override def ansiCastRule: ExprRule[_ <: Expression] = null

override def getDataWriteCmds: Map[Class[_ <: DataWritingCommand],
DataWritingCommandRule[_ <: DataWritingCommand]] = {
Seq(GpuOverrides.dataWriteCmd[CreateDataSourceTableAsSelectCommand](
"Create table with select command",
(a, conf, p, r) => new CreateDataSourceTableAsSelectCommandMeta(a, conf, p, r))
Seq(
GpuOverrides.dataWriteCmdFromShim(
CreateDataSourceTableAsSelectRules.dataWriteCmd,
(a, conf, p, r) => new CreateDataSourceTableAsSelectCommandMeta(a, conf, p, r))
).map(r => (r.getClassFor.asSubclass(classOf[DataWritingCommand]), r)).toMap
}

Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,6 @@ import com.nvidia.spark.rapids.Arm.{closeOnExcept, withResource}
import com.nvidia.spark.rapids.RapidsPluginImplicits._
import com.nvidia.spark.rapids.shims.NullIntolerantShim

import org.apache.spark.internal.Logging
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
import org.apache.spark.sql.catalyst.expressions.Expression
import org.apache.spark.sql.catalyst.trees.{CurrentOrigin, Origin}
Expand Down Expand Up @@ -279,7 +278,7 @@ case class GpuDecimalRemainder(
left: Expression,
right: Expression,
failOnError: Boolean = SQLConf.get.ansiEnabled)
extends GpuRemainderBase(left, right) with Logging {
extends GpuRemainderBase(left, right) {

// scalastyle:off
// The formula follows Hive which is based on the SQL standard and MS SQL:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2023-2025, NVIDIA CORPORATION.
* Copyright (c) 2023-2026, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -25,14 +25,14 @@ import org.apache.spark.sql.rapids.shims.ArrowUtilsShim
import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.vectorized.ColumnarBatch

case class GpuGroupedPythonRunnerFactory(
class GpuGroupedPythonRunnerFactory(
conf: org.apache.spark.sql.internal.SQLConf,
chainedFunc: Seq[(ChainedPythonFunctions, Long)],
argOffsets: Array[Array[Int]],
dedupAttrs: StructType,
pythonOutputSchema: StructType,
evalType: Int,
argNames: Option[Array[Array[Option[String]]]] = None) {
argNames: Option[Array[Array[Option[String]]]]) extends Serializable {
// Configs from DB runtime
val maxBytes = conf.pandasZeroConfConversionGroupbyApplyMaxBytesPerSlice
val zeroConfEnabled = conf.pandasZeroConfConversionGroupbyApplyEnabled
Expand Down

This file was deleted.

This file was deleted.

Loading
Loading