diff --git a/sql-plugin-shims/src/main/scala/org/apache/spark/sql/errors/ConvUtils.scala b/sql-plugin-shims/src/main/scala/org/apache/spark/sql/errors/ConvUtils.scala new file mode 100644 index 00000000000..abd23b7f158 --- /dev/null +++ b/sql-plugin-shims/src/main/scala/org/apache/spark/sql/errors/ConvUtils.scala @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2025-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.errors + +import java.lang.reflect.InvocationTargetException + +object ConvUtils { + private val queryExecutionErrorsCompanion = + "org.apache.spark.sql.errors.QueryExecutionErrors$" + + def overflowInConvError(): Unit = { + try { + val companion = Class.forName(queryExecutionErrorsCompanion).getField("MODULE$").get(null) + val method = companion.getClass.getMethods.find { method => + method.getName == "overflowInConvError" && method.getParameterCount == 1 + }.getOrElse { + throw new UnsupportedOperationException() + } + throw method.invoke(companion, null.asInstanceOf[AnyRef]).asInstanceOf[Throwable] + } catch { + case _: ClassNotFoundException | _: NoSuchFieldException => + throw new UnsupportedOperationException() + case e: InvocationTargetException => + throw e.getCause + } + } +} diff --git a/sql-plugin-shims/src/main/scala/org/apache/spark/sql/rapids/shims/SparkSessionUtils.scala b/sql-plugin-shims/src/main/scala/org/apache/spark/sql/rapids/shims/SparkSessionUtils.scala new file mode 100644 index 00000000000..a68f2d6bafd --- /dev/null +++ b/sql-plugin-shims/src/main/scala/org/apache/spark/sql/rapids/shims/SparkSessionUtils.scala @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2025-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.rapids.shims + +import java.lang.reflect.InvocationTargetException + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.execution.SparkPlan + +object SparkSessionUtils { + + def sessionFromPlan(plan: SparkPlan): SparkSession = { + invokeNoArg(plan, "session").asInstanceOf[SparkSession] + } + + def leafNodeDefaultParallelism(ss: SparkSession): Int = { + invokeNoArg(ss, "leafNodeDefaultParallelism").asInstanceOf[Int] + } + + private def invokeNoArg(target: AnyRef, methodName: String): AnyRef = { + try { + target.getClass.getMethod(methodName).invoke(target) + } catch { + case e: InvocationTargetException => + throw e.getCause + } + } +} diff --git a/sql-plugin-shims/src/main/spark330/scala/com/nvidia/spark/rapids/shims/CreateDataSourceTableAsSelectRules.scala b/sql-plugin-shims/src/main/spark330/scala/com/nvidia/spark/rapids/shims/CreateDataSourceTableAsSelectRules.scala new file mode 100644 index 00000000000..6e7f9d737ed --- /dev/null +++ b/sql-plugin-shims/src/main/spark330/scala/com/nvidia/spark/rapids/shims/CreateDataSourceTableAsSelectRules.scala @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "330"} +{"spark": "330db"} +{"spark": "331"} +{"spark": "332"} +{"spark": "333"} +{"spark": "334"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims + +import com.nvidia.spark.rapids.ShimDataWritingCommandRule + +import org.apache.spark.sql.execution.command.CreateDataSourceTableAsSelectCommand + +object CreateDataSourceTableAsSelectRules { + val dataWriteCmd: ShimDataWritingCommandRule[CreateDataSourceTableAsSelectCommand] = + ShimDataWritingCommandRule[CreateDataSourceTableAsSelectCommand]( + "Create table with select command") +} diff --git a/sql-plugin-shims/src/main/spark330/scala/com/nvidia/spark/rapids/shims/SequenceSizeTooLongErrorBuilder.scala b/sql-plugin-shims/src/main/spark330/scala/com/nvidia/spark/rapids/shims/SequenceSizeTooLongErrorBuilder.scala new file mode 100644 index 00000000000..065c6ceaee5 --- /dev/null +++ b/sql-plugin-shims/src/main/spark330/scala/com/nvidia/spark/rapids/shims/SequenceSizeTooLongErrorBuilder.scala @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2024-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "330"} +{"spark": "330db"} +{"spark": "331"} +{"spark": "332"} +{"spark": "332db"} +{"spark": "333"} +{"spark": "340"} +{"spark": "341"} +{"spark": "341db"} +{"spark": "350"} +{"spark": "350db143"} +spark-rapids-shim-json-lines ***/ +package org.apache.spark.sql.rapids.shims + +import org.apache.spark.unsafe.array.ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH + +trait SequenceSizeTooLongErrorBuilder { + + def getTooLongSequenceErrorString(sequenceSize: Int, functionName: String): String = { + // For these Spark versions, the sequence length and function name + // do not appear in the exception message. + s"Too long sequence found. Should be <= $MAX_ROUNDED_ARRAY_LENGTH" + } +} diff --git a/sql-plugin-shims/src/main/spark330/scala/com/nvidia/spark/rapids/shims/spark330/SparkShimServiceProvider.scala b/sql-plugin-shims/src/main/spark330/scala/com/nvidia/spark/rapids/shims/spark330/SparkShimServiceProvider.scala new file mode 100644 index 00000000000..4e209fe7450 --- /dev/null +++ b/sql-plugin-shims/src/main/spark330/scala/com/nvidia/spark/rapids/shims/spark330/SparkShimServiceProvider.scala @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2021-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "330"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims.spark330 + +import com.nvidia.spark.rapids.SparkShimVersion + +object SparkShimServiceProvider { + val VERSION = SparkShimVersion(3, 3, 0) + val VERSIONNAMES = Seq(s"$VERSION") +} + +class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { + + override def getShimVersion: SparkShimVersion = SparkShimServiceProvider.VERSION + + def matchesVersion(version: String): Boolean = { + SparkShimServiceProvider.VERSIONNAMES.contains(version) + } +} diff --git a/sql-plugin-shims/src/main/spark330/scala/org/apache/spark/sql/rapids/ShuffleManagerShims.scala b/sql-plugin-shims/src/main/spark330/scala/org/apache/spark/sql/rapids/ShuffleManagerShims.scala new file mode 100644 index 00000000000..2864730e543 --- /dev/null +++ b/sql-plugin-shims/src/main/spark330/scala/org/apache/spark/sql/rapids/ShuffleManagerShims.scala @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "330"} +{"spark": "330db"} +{"spark": "331"} +{"spark": "332"} +{"spark": "332db"} +{"spark": "333"} +{"spark": "334"} +{"spark": "340"} +{"spark": "341"} +{"spark": "341db"} +{"spark": "342"} +{"spark": "343"} +{"spark": "344"} +{"spark": "350"} +{"spark": "350db143"} +{"spark": "351"} +{"spark": "352"} +{"spark": "353"} +{"spark": "354"} +{"spark": "355"} +{"spark": "356"} +{"spark": "357"} +{"spark": "358"} +{"spark": "400"} +{"spark": "401"} +{"spark": "402"} +{"spark": "411"} +spark-rapids-shim-json-lines ***/ +package org.apache.spark.sql.rapids + +import org.apache.spark.TaskContext +import org.apache.spark.shuffle.{ShuffleHandle, ShuffleManager, ShuffleReader, ShuffleReadMetricsReporter} + +/** + * Shim object to handle version-specific differences in ShuffleManager APIs. + */ +object ShuffleManagerShims { + /** + * Call ShuffleManager.getReader with the appropriate signature for this Spark version. + * This method is overridden in version-specific shims. + */ + def getReader[K, C]( + manager: ShuffleManager, + handle: ShuffleHandle, + startMapIndex: Int, + endMapIndex: Int, + startPartition: Int, + endPartition: Int, + context: TaskContext, + metrics: ShuffleReadMetricsReporter): ShuffleReader[K, C] = { + manager.getReader(handle, startMapIndex, endMapIndex, startPartition, + endPartition, context, metrics) + } +} diff --git a/sql-plugin-shims/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/FileCommitProtocolShims.scala b/sql-plugin-shims/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/FileCommitProtocolShims.scala new file mode 100644 index 00000000000..7eaa28d0d8d --- /dev/null +++ b/sql-plugin-shims/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/FileCommitProtocolShims.scala @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "330"} +{"spark": "330db"} +{"spark": "331"} +{"spark": "332"} +{"spark": "332db"} +{"spark": "333"} +{"spark": "334"} +{"spark": "340"} +{"spark": "341"} +{"spark": "341db"} +{"spark": "342"} +{"spark": "343"} +{"spark": "344"} +{"spark": "350"} +{"spark": "350db143"} +{"spark": "351"} +{"spark": "352"} +{"spark": "353"} +{"spark": "354"} +{"spark": "355"} +{"spark": "356"} +{"spark": "357"} +{"spark": "358"} +{"spark": "400"} +{"spark": "400db173"} +{"spark": "401"} +{"spark": "402"} +spark-rapids-shim-json-lines ***/ +package org.apache.spark.sql.rapids.shims + +import org.apache.hadoop.mapreduce.TaskAttemptContext + +import org.apache.spark.internal.io.FileCommitProtocol + +/** + * Shim for FileCommitProtocol.newTaskTempFile API. + * In Spark <= 4.0.x, we use the deprecated (ext: String) signature. + * In Spark 4.1.0+, we use the new (spec: FileNameSpec) signature. + */ +object FileCommitProtocolShims { + @scala.annotation.nowarn( + "msg=method newTaskTempFile in class FileCommitProtocol is deprecated" + ) + def newTaskTempFile( + committer: FileCommitProtocol, + taskContext: TaskAttemptContext, + dir: Option[String], + ext: String): String = { + committer.newTaskTempFile(taskContext, dir, ext) + } + + @scala.annotation.nowarn( + "msg=method newTaskTempFileAbsPath in class FileCommitProtocol is deprecated" + ) + def newTaskTempFileAbsPath( + committer: FileCommitProtocol, + taskContext: TaskAttemptContext, + absoluteDir: String, + ext: String): String = { + committer.newTaskTempFileAbsPath(taskContext, absoluteDir, ext) + } +} diff --git a/sql-plugin-shims/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/OriginContextShim.scala b/sql-plugin-shims/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/OriginContextShim.scala new file mode 100644 index 00000000000..75f9f446197 --- /dev/null +++ b/sql-plugin-shims/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/OriginContextShim.scala @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "330"} +{"spark": "331"} +{"spark": "332"} +{"spark": "333"} +{"spark": "334"} +spark-rapids-shim-json-lines ***/ +package org.apache.spark.sql.rapids.shims + +import org.apache.spark.sql.catalyst.trees.Origin + +// Apache Spark 3.3.x carries SPARK-39175 with `Origin.context: String`. +object OriginContextShim { + def queryContext(origin: Origin): String = origin.context + def contextSummary(origin: Origin): String = origin.context +} diff --git a/sql-plugin-shims/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/SparkUpgradeExceptionShims.scala b/sql-plugin-shims/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/SparkUpgradeExceptionShims.scala new file mode 100644 index 00000000000..b0cab9169e4 --- /dev/null +++ b/sql-plugin-shims/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/SparkUpgradeExceptionShims.scala @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2022-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "330"} +{"spark": "331"} +{"spark": "332"} +{"spark": "333"} +{"spark": "334"} +spark-rapids-shim-json-lines ***/ +package org.apache.spark.sql.rapids.shims + +import org.apache.spark.SparkUpgradeException + +object SparkUpgradeExceptionShims { + + def newSparkUpgradeException( + version: String, + message: String, + cause: Throwable): SparkUpgradeException = { + new SparkUpgradeException( + "INCONSISTENT_BEHAVIOR_CROSS_VERSION", + Array(version, message), + cause) + } + + // Used in tests to compare the class seen in an exception to + // `SparkUpgradeException` which is private in Spark + def getSparkUpgradeExceptionClass: Class[_] = { + classOf[SparkUpgradeException] + } +} diff --git a/sql-plugin-shims/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/TrampolineConnectShims.scala b/sql-plugin-shims/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/TrampolineConnectShims.scala new file mode 100644 index 00000000000..e0dd693ac85 --- /dev/null +++ b/sql-plugin-shims/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/TrampolineConnectShims.scala @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2025-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +/*** spark-rapids-shim-json-lines +{"spark": "330"} +{"spark": "330db"} +{"spark": "331"} +{"spark": "332"} +{"spark": "332db"} +{"spark": "333"} +{"spark": "334"} +{"spark": "340"} +{"spark": "341"} +{"spark": "341db"} +{"spark": "342"} +{"spark": "343"} +{"spark": "344"} +{"spark": "350"} +{"spark": "350db143"} +{"spark": "351"} +{"spark": "352"} +{"spark": "353"} +{"spark": "354"} +{"spark": "355"} +{"spark": "356"} +{"spark": "357"} +{"spark": "358"} +spark-rapids-shim-json-lines ***/ +package org.apache.spark.sql.rapids.shims + +import org.apache.avro.Schema + +import org.apache.spark.sql.{Dataset, SparkSession} +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan + +object TrampolineConnectShims { + + type SparkSession = org.apache.spark.sql.SparkSession + type DataFrame = org.apache.spark.sql.DataFrame + type Dataset = org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] + + def cleanupAnyExistingSession(): Unit = SparkSession.cleanupAnyExistingSession() + + def getActiveSession: SparkSession = { + SparkSession.getActiveSession.getOrElse( + throw new IllegalStateException("No active SparkSession found") + ) + } + + def createSchemaParser(): Schema.Parser = { + new Schema.Parser().setValidateDefaults(false).setValidate(false) + } + + def createDataFrame(spark: SparkSession, plan: LogicalPlan): DataFrame = { + Dataset.ofRows(spark, plan) + } + + def getBuilder(): SparkSession.Builder = { + SparkSession.builder() + } + + def hasActiveSession: Boolean = { + SparkSession.getActiveSession.isDefined + } +} diff --git a/sql-plugin-shims/src/main/spark330/scala/org/apache/spark/storage/ShuffleClientShims.scala b/sql-plugin-shims/src/main/spark330/scala/org/apache/spark/storage/ShuffleClientShims.scala new file mode 100644 index 00000000000..79dab4e56e0 --- /dev/null +++ b/sql-plugin-shims/src/main/spark330/scala/org/apache/spark/storage/ShuffleClientShims.scala @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "330"} +{"spark": "330db"} +{"spark": "331"} +{"spark": "332"} +{"spark": "332db"} +{"spark": "333"} +{"spark": "334"} +{"spark": "340"} +{"spark": "341"} +{"spark": "341db"} +{"spark": "342"} +{"spark": "343"} +{"spark": "344"} +{"spark": "350"} +{"spark": "350db143"} +{"spark": "351"} +{"spark": "352"} +{"spark": "353"} +{"spark": "354"} +{"spark": "355"} +{"spark": "356"} +{"spark": "357"} +{"spark": "358"} +{"spark": "400"} +{"spark": "401"} +{"spark": "402"} +{"spark": "411"} +spark-rapids-shim-json-lines ***/ +package org.apache.spark.storage + +import org.apache.spark.network.shuffle.BlockStoreClient +import org.apache.spark.network.shuffle.checksum.Cause + +object ShuffleClientShims { + def diagnoseCorruption( + client: BlockStoreClient, + host: String, + port: Int, + execId: String, + blockId: BlockId, + checksum: Long, + algorithm: String): Cause = { + blockId match { + case shuffleBlock: ShuffleBlockId => + client.diagnoseCorruption(host, port, execId, + shuffleBlock.shuffleId, shuffleBlock.mapId, shuffleBlock.reduceId, + checksum, algorithm) + case _ => + throw new IllegalArgumentException(s"Unexpected block type: ${blockId.getClass}") + } + } +} diff --git a/sql-plugin-shims/src/main/spark330db/scala/com/nvidia/spark/rapids/DatabricksShimServiceProvider.scala b/sql-plugin-shims/src/main/spark330db/scala/com/nvidia/spark/rapids/DatabricksShimServiceProvider.scala new file mode 100644 index 00000000000..2aae0326162 --- /dev/null +++ b/sql-plugin-shims/src/main/spark330db/scala/com/nvidia/spark/rapids/DatabricksShimServiceProvider.scala @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2024-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "330db"} +{"spark": "332db"} +{"spark": "341db"} +{"spark": "350db143"} +{"spark": "400db173"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids + +object DatabricksShimServiceProvider { + val log = org.slf4j.LoggerFactory.getLogger(getClass().getName().stripSuffix("$")) + + def matchesVersion(dbrVersion: String, + shimMatchEnabled: Boolean = true, + disclaimer: String = "" + ): Boolean = { + var ignoreExceptions = true + try { + val sparkBuildInfo = org.apache.spark.BuildInfo + val databricksBuildInfo = com.databricks.BuildInfo + val matchRes = sparkBuildInfo.dbrVersion.startsWith(dbrVersion) + val matchStatus = if (matchRes) "SUCCESS" else "FAILURE" + val logMessage = + s"""Databricks Runtime Build Info match: $matchStatus + |\tDBR_VERSION: ${sparkBuildInfo.dbrVersion} + |\tspark.BuildInfo.gitHash: ${sparkBuildInfo.gitHash} + |\tdatabricks.BuildInfo.gitHash: ${databricksBuildInfo.gitHash} + |\tdatabricks.BuildInfo.gitTimestamp: ${databricksBuildInfo.gitTimestamp}""" + .stripMargin + if (matchRes) { + log.warn(logMessage) + if (shimMatchEnabled) { + if (disclaimer.nonEmpty) { + log.warn(disclaimer) + } + } else { + ignoreExceptions = false + sys.error(disclaimer) + } + } else { + log.debug(logMessage) + } + matchRes + } catch { + case x: Throwable if ignoreExceptions => + log.debug("Databricks detection failed: " + x, x) + false + } + } +} diff --git a/sql-plugin-shims/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/spark330db/SparkShimServiceProvider.scala b/sql-plugin-shims/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/spark330db/SparkShimServiceProvider.scala new file mode 100644 index 00000000000..149c45b7dab --- /dev/null +++ b/sql-plugin-shims/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/spark330db/SparkShimServiceProvider.scala @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2022-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "330db"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims.spark330db + +import com.nvidia.spark.rapids._ + +object SparkShimServiceProvider { + // DB version should conform to "major.minor" and has no patch version. + // Refer to VersionUtils.getVersionForJni + val VERSION = DatabricksShimVersion(3, 3, 0, "11.3") +} + +class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { + + override def getShimVersion: ShimVersion = SparkShimServiceProvider.VERSION + + def matchesVersion(version: String): Boolean = { + DatabricksShimServiceProvider.matchesVersion("11.3.x") + } +} diff --git a/sql-plugin-shims/src/main/spark330db/scala/org/apache/spark/sql/rapids/shims/OriginContextShim.scala b/sql-plugin-shims/src/main/spark330db/scala/org/apache/spark/sql/rapids/shims/OriginContextShim.scala new file mode 100644 index 00000000000..50d91c1abdb --- /dev/null +++ b/sql-plugin-shims/src/main/spark330db/scala/org/apache/spark/sql/rapids/shims/OriginContextShim.scala @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "330db"} +{"spark": "332db"} +spark-rapids-shim-json-lines ***/ +package org.apache.spark.sql.rapids.shims + +import org.apache.spark.sql.catalyst.trees.{Origin, SQLQueryContext} + +// Databricks 3.3.x back-ported SPARK-39175 and typed `Origin.context` as +// `SQLQueryContext` directly — same shape as Apache 3.4+. +object OriginContextShim { + def queryContext(origin: Origin): SQLQueryContext = origin.context + def contextSummary(origin: Origin): String = origin.context match { + case null => "" + case ctx => ctx.summary + } +} diff --git a/sql-plugin-shims/src/main/spark330db/scala/org/apache/spark/sql/rapids/shims/SparkDateTimeExceptionShims.scala b/sql-plugin-shims/src/main/spark330db/scala/org/apache/spark/sql/rapids/shims/SparkDateTimeExceptionShims.scala new file mode 100644 index 00000000000..a33ac76fdc5 --- /dev/null +++ b/sql-plugin-shims/src/main/spark330db/scala/org/apache/spark/sql/rapids/shims/SparkDateTimeExceptionShims.scala @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2022-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "330db"} +spark-rapids-shim-json-lines ***/ +package org.apache.spark.sql.rapids.shims + +import org.apache.spark.{QueryContext, SparkDateTimeException} + +object SparkDateTimeExceptionShims { + + def newSparkDateTimeException( + errorClass: String, + messageParameters: Map[String, String], + context: Array[QueryContext], + summary: String): SparkDateTimeException = { + new SparkDateTimeException( + errorClass, + None, + Array.empty, + context, + summary) + } +} diff --git a/sql-plugin-shims/src/main/spark330db/scala/org/apache/spark/sql/rapids/shims/SparkUpgradeExceptionShims.scala b/sql-plugin-shims/src/main/spark330db/scala/org/apache/spark/sql/rapids/shims/SparkUpgradeExceptionShims.scala new file mode 100644 index 00000000000..feedbd11cf9 --- /dev/null +++ b/sql-plugin-shims/src/main/spark330db/scala/org/apache/spark/sql/rapids/shims/SparkUpgradeExceptionShims.scala @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2022-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "330db"} +spark-rapids-shim-json-lines ***/ +package org.apache.spark.sql.rapids.shims + +import org.apache.spark.SparkUpgradeException + +object SparkUpgradeExceptionShims { + + def newSparkUpgradeException( + version: String, + message: String, + cause: Throwable): SparkUpgradeException = { + new SparkUpgradeException( + "INCONSISTENT_BEHAVIOR_CROSS_VERSION", + None, + Array(version, message), + cause) + } + + // Used in tests to compare the class seen in an exception to + // `SparkUpgradeException` which is private in Spark + def getSparkUpgradeExceptionClass: Class[_] = { + classOf[SparkUpgradeException] + } +}