diff --git a/sql-plugin-shims/src/main/spark350/scala/com/nvidia/spark/rapids/shims/spark350/SparkShimServiceProvider.scala b/sql-plugin-shims/src/main/spark350/scala/com/nvidia/spark/rapids/shims/spark350/SparkShimServiceProvider.scala new file mode 100644 index 00000000000..c70f14547d1 --- /dev/null +++ b/sql-plugin-shims/src/main/spark350/scala/com/nvidia/spark/rapids/shims/spark350/SparkShimServiceProvider.scala @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2023-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "350"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims.spark350 + +import com.nvidia.spark.rapids.SparkShimVersion + +object SparkShimServiceProvider { + val VERSION = SparkShimVersion(3, 5, 0) + val VERSIONNAMES = Seq(s"$VERSION") +} + +class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { + + override def getShimVersion: SparkShimVersion = SparkShimServiceProvider.VERSION + + override def matchesVersion(version: String): Boolean = { + SparkShimServiceProvider.VERSIONNAMES.contains(version) + } +} diff --git a/sql-plugin-shims/src/main/spark350db143/scala/com/nvidia/spark/rapids/shims/spark350db143/SparkShimServiceProvider.scala b/sql-plugin-shims/src/main/spark350db143/scala/com/nvidia/spark/rapids/shims/spark350db143/SparkShimServiceProvider.scala new file mode 100644 index 00000000000..55bcad35ea9 --- /dev/null +++ b/sql-plugin-shims/src/main/spark350db143/scala/com/nvidia/spark/rapids/shims/spark350db143/SparkShimServiceProvider.scala @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2023-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "350db143"} +{"spark": "400db173"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims.spark350db143 + +import com.nvidia.spark.rapids._ + +import org.apache.spark.SparkEnv + +object SparkShimServiceProvider { + // DB version should conform to "major.minor" and has no patch version. + // Refer to VersionUtils.getVersionForJni + val VERSION = DatabricksShimVersion(3, 5, 0, "14.3") +} + +class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { + + override def getShimVersion: ShimVersion = SparkShimServiceProvider.VERSION + + def matchesVersion(version: String): Boolean = { + val shimEnabledProp = "spark.rapids.shims.spark350db143" + ".enabled" + // disabled by default + val shimEnabled = Option(SparkEnv.get) + .flatMap(_.conf.getOption(shimEnabledProp).map(_.toBoolean)) + .getOrElse(true) + + DatabricksShimServiceProvider.matchesVersion( + dbrVersion = "14.3.x", + shimMatchEnabled = shimEnabled, + disclaimer = "Development of support for Databricks 14.3.x is still in progress: " + + "https://github.com/NVIDIA/spark-rapids/issues/10661" + ) + } +} diff --git a/sql-plugin-shims/src/main/spark350db143/scala/org/apache/spark/sql/rapids/shims/SequenceSizeExceededLimitErrorBuilder.scala b/sql-plugin-shims/src/main/spark350db143/scala/org/apache/spark/sql/rapids/shims/SequenceSizeExceededLimitErrorBuilder.scala new file mode 100644 index 00000000000..c42f21207a1 --- /dev/null +++ b/sql-plugin-shims/src/main/spark350db143/scala/org/apache/spark/sql/rapids/shims/SequenceSizeExceededLimitErrorBuilder.scala @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "350db143"} +{"spark": "400"} +{"spark": "400db173"} +{"spark": "401"} +{"spark": "402"} +{"spark": "411"} +spark-rapids-shim-json-lines ***/ +package org.apache.spark.sql.rapids.shims + +import org.apache.spark.sql.errors.QueryExecutionErrors + +trait SequenceSizeExceededLimitErrorBuilder { + def getTooLongSequenceErrorString(sequenceSize: Int, functionName: String): String = { + QueryExecutionErrors.createArrayWithElementsExceedLimitError(functionName, sequenceSize) + .getMessage + } +} diff --git a/sql-plugin-shims/src/main/spark351/scala/com/nvidia/spark/rapids/shims/spark351/SparkShimServiceProvider.scala b/sql-plugin-shims/src/main/spark351/scala/com/nvidia/spark/rapids/shims/spark351/SparkShimServiceProvider.scala new file mode 100644 index 00000000000..a1a1f5a51a9 --- /dev/null +++ b/sql-plugin-shims/src/main/spark351/scala/com/nvidia/spark/rapids/shims/spark351/SparkShimServiceProvider.scala @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2023-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "351"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims.spark351 + +import com.nvidia.spark.rapids.SparkShimVersion + +object SparkShimServiceProvider { + val VERSION = SparkShimVersion(3, 5, 1) + val VERSIONNAMES = Seq(s"$VERSION", s"$VERSION-SNAPSHOT") +} + +class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { + + override def getShimVersion: SparkShimVersion = SparkShimServiceProvider.VERSION + + override def matchesVersion(version: String): Boolean = { + SparkShimServiceProvider.VERSIONNAMES.contains(version) + } +} diff --git a/sql-plugin-shims/src/main/spark352/scala/com/nvidia/spark/rapids/shims/spark352/SparkShimServiceProvider.scala b/sql-plugin-shims/src/main/spark352/scala/com/nvidia/spark/rapids/shims/spark352/SparkShimServiceProvider.scala new file mode 100644 index 00000000000..a6cb8368ba2 --- /dev/null +++ b/sql-plugin-shims/src/main/spark352/scala/com/nvidia/spark/rapids/shims/spark352/SparkShimServiceProvider.scala @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2024-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "352"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims.spark352 + +import com.nvidia.spark.rapids.SparkShimVersion + +object SparkShimServiceProvider { + val VERSION = SparkShimVersion(3, 5, 2) + val VERSIONNAMES = Seq(s"$VERSION") +} + +class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { + + override def getShimVersion: SparkShimVersion = SparkShimServiceProvider.VERSION + + override def matchesVersion(version: String): Boolean = { + SparkShimServiceProvider.VERSIONNAMES.contains(version) + } +} diff --git a/sql-plugin-shims/src/main/spark353/scala/com/nvidia/spark/rapids/shims/spark353/SparkShimServiceProvider.scala b/sql-plugin-shims/src/main/spark353/scala/com/nvidia/spark/rapids/shims/spark353/SparkShimServiceProvider.scala new file mode 100644 index 00000000000..2945458b56f --- /dev/null +++ b/sql-plugin-shims/src/main/spark353/scala/com/nvidia/spark/rapids/shims/spark353/SparkShimServiceProvider.scala @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2024-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "353"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims.spark353 + +import com.nvidia.spark.rapids.SparkShimVersion + +object SparkShimServiceProvider { + val VERSION = SparkShimVersion(3, 5, 3) + val VERSIONNAMES = Seq(s"$VERSION") +} + +class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { + + override def getShimVersion: SparkShimVersion = SparkShimServiceProvider.VERSION + + override def matchesVersion(version: String): Boolean = { + SparkShimServiceProvider.VERSIONNAMES.contains(version) + } +} diff --git a/sql-plugin-shims/src/main/spark354/scala/com/nvidia/spark/rapids/shims/spark354/SparkShimServiceProvider.scala b/sql-plugin-shims/src/main/spark354/scala/com/nvidia/spark/rapids/shims/spark354/SparkShimServiceProvider.scala new file mode 100644 index 00000000000..7002ba891cd --- /dev/null +++ b/sql-plugin-shims/src/main/spark354/scala/com/nvidia/spark/rapids/shims/spark354/SparkShimServiceProvider.scala @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2025-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "354"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims.spark354 + +import com.nvidia.spark.rapids.SparkShimVersion + +object SparkShimServiceProvider { + val VERSION = SparkShimVersion(3, 5, 4) + val VERSIONNAMES = Seq(s"$VERSION") +} + +class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { + + override def getShimVersion: SparkShimVersion = SparkShimServiceProvider.VERSION + + override def matchesVersion(version: String): Boolean = { + SparkShimServiceProvider.VERSIONNAMES.contains(version) + } +} diff --git a/sql-plugin-shims/src/main/spark355/scala/com/nvidia/spark/rapids/shims/spark355/SparkShimServiceProvider.scala b/sql-plugin-shims/src/main/spark355/scala/com/nvidia/spark/rapids/shims/spark355/SparkShimServiceProvider.scala new file mode 100644 index 00000000000..776871d176c --- /dev/null +++ b/sql-plugin-shims/src/main/spark355/scala/com/nvidia/spark/rapids/shims/spark355/SparkShimServiceProvider.scala @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2025-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "355"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims.spark355 + +import com.nvidia.spark.rapids.SparkShimVersion + +object SparkShimServiceProvider { + val VERSION = SparkShimVersion(3, 5, 5) + val VERSIONNAMES = Seq(s"$VERSION") +} + +class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { + + override def getShimVersion: SparkShimVersion = SparkShimServiceProvider.VERSION + + override def matchesVersion(version: String): Boolean = { + SparkShimServiceProvider.VERSIONNAMES.contains(version) + } +} diff --git a/sql-plugin-shims/src/main/spark356/scala/com/nvidia/spark/rapids/shims/spark356/SparkShimServiceProvider.scala b/sql-plugin-shims/src/main/spark356/scala/com/nvidia/spark/rapids/shims/spark356/SparkShimServiceProvider.scala new file mode 100644 index 00000000000..f9ce0d6aeee --- /dev/null +++ b/sql-plugin-shims/src/main/spark356/scala/com/nvidia/spark/rapids/shims/spark356/SparkShimServiceProvider.scala @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2025-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "356"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims.spark356 + +import com.nvidia.spark.rapids.SparkShimVersion + +object SparkShimServiceProvider { + val VERSION = SparkShimVersion(3, 5, 6) + val VERSIONNAMES = Seq(s"$VERSION") +} + +class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { + + override def getShimVersion: SparkShimVersion = SparkShimServiceProvider.VERSION + + override def matchesVersion(version: String): Boolean = { + SparkShimServiceProvider.VERSIONNAMES.contains(version) + } +} diff --git a/sql-plugin-shims/src/main/spark357/scala/com/nvidia/spark/rapids/shims/spark357/SparkShimServiceProvider.scala b/sql-plugin-shims/src/main/spark357/scala/com/nvidia/spark/rapids/shims/spark357/SparkShimServiceProvider.scala new file mode 100644 index 00000000000..bd376eee633 --- /dev/null +++ b/sql-plugin-shims/src/main/spark357/scala/com/nvidia/spark/rapids/shims/spark357/SparkShimServiceProvider.scala @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2025-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "357"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims.spark357 + +import com.nvidia.spark.rapids.SparkShimVersion + +object SparkShimServiceProvider { + val VERSION = SparkShimVersion(3, 5, 7) + val VERSIONNAMES = Seq(s"$VERSION") +} + +class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { + + override def getShimVersion: SparkShimVersion = SparkShimServiceProvider.VERSION + + override def matchesVersion(version: String): Boolean = { + SparkShimServiceProvider.VERSIONNAMES.contains(version) + } +} diff --git a/sql-plugin-shims/src/main/spark358/scala/com/nvidia/spark/rapids/shims/spark358/SparkShimServiceProvider.scala b/sql-plugin-shims/src/main/spark358/scala/com/nvidia/spark/rapids/shims/spark358/SparkShimServiceProvider.scala new file mode 100644 index 00000000000..99585c30d57 --- /dev/null +++ b/sql-plugin-shims/src/main/spark358/scala/com/nvidia/spark/rapids/shims/spark358/SparkShimServiceProvider.scala @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "358"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims.spark358 + +import com.nvidia.spark.rapids.SparkShimVersion + +object SparkShimServiceProvider { + val VERSION = SparkShimVersion(3, 5, 8) + val VERSIONNAMES = Seq(s"$VERSION") +} + +class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { + + override def getShimVersion: SparkShimVersion = SparkShimServiceProvider.VERSION + + override def matchesVersion(version: String): Boolean = { + SparkShimServiceProvider.VERSIONNAMES.contains(version) + } +} diff --git a/sql-plugin-shims/src/main/spark400/scala/com/nvidia/spark/rapids/shims/spark400/SparkShimServiceProvider.scala b/sql-plugin-shims/src/main/spark400/scala/com/nvidia/spark/rapids/shims/spark400/SparkShimServiceProvider.scala new file mode 100644 index 00000000000..95525c673f7 --- /dev/null +++ b/sql-plugin-shims/src/main/spark400/scala/com/nvidia/spark/rapids/shims/spark400/SparkShimServiceProvider.scala @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2024-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "400"} +{"spark": "400db173"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims.spark400 + +import com.nvidia.spark.rapids.SparkShimVersion + +object SparkShimServiceProvider { + val VERSION = SparkShimVersion(4, 0, 0) + val VERSIONNAMES = Seq(s"$VERSION") +} + +class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { + + override def getShimVersion: SparkShimVersion = SparkShimServiceProvider.VERSION + + override def matchesVersion(version: String): Boolean = { + SparkShimServiceProvider.VERSIONNAMES.contains(version) + } +} diff --git a/sql-plugin-shims/src/main/spark400/scala/org/apache/spark/sql/rapids/shims/OriginContextShim.scala b/sql-plugin-shims/src/main/spark400/scala/org/apache/spark/sql/rapids/shims/OriginContextShim.scala new file mode 100644 index 00000000000..7f0892adf61 --- /dev/null +++ b/sql-plugin-shims/src/main/spark400/scala/org/apache/spark/sql/rapids/shims/OriginContextShim.scala @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "400"} +{"spark": "400db173"} +{"spark": "401"} +{"spark": "402"} +{"spark": "411"} +spark-rapids-shim-json-lines ***/ +package org.apache.spark.sql.rapids.shims + +import org.apache.spark.sql.catalyst.trees.{Origin, SQLQueryContext} + +// Spark 4.0 widened `Origin.context` to `org.apache.spark.QueryContext`, while +// `QueryExecutionErrors` still takes the `SQLQueryContext` subtype — narrow here. +object OriginContextShim { + def queryContext(origin: Origin): SQLQueryContext = origin.context match { + case ctx: SQLQueryContext => ctx + case _ => null + } + def contextSummary(origin: Origin): String = origin.context match { + case null => "" + case ctx => ctx.summary + } +} diff --git a/sql-plugin-shims/src/main/spark400/scala/org/apache/spark/sql/rapids/shims/TrampolineConnectShims.scala b/sql-plugin-shims/src/main/spark400/scala/org/apache/spark/sql/rapids/shims/TrampolineConnectShims.scala new file mode 100644 index 00000000000..9a753370800 --- /dev/null +++ b/sql-plugin-shims/src/main/spark400/scala/org/apache/spark/sql/rapids/shims/TrampolineConnectShims.scala @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2025-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +/*** spark-rapids-shim-json-lines +{"spark": "400"} +{"spark": "400db173"} +{"spark": "401"} +{"spark": "402"} +{"spark": "411"} +spark-rapids-shim-json-lines ***/ +package org.apache.spark.sql.rapids.shims + +import org.apache.avro.NameValidator +import org.apache.avro.Schema + +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +object TrampolineConnectShims { + + type SparkSession = org.apache.spark.sql.classic.SparkSession + type DataFrame = org.apache.spark.sql.classic.DataFrame + type Dataset = org.apache.spark.sql.classic.Dataset[org.apache.spark.sql.Row] + + def cleanupAnyExistingSession(): Unit = { + org.apache.spark.sql.classic.SparkSession.cleanupAnyExistingSession() + } + + def createDataFrame(spark: SparkSession, plan: LogicalPlan): DataFrame = { + org.apache.spark.sql.classic.Dataset.ofRows(spark, plan) + } + + def getBuilder(): org.apache.spark.sql.classic.SparkSession.Builder = { + org.apache.spark.sql.classic.SparkSession.builder() + } + + def hasActiveSession: Boolean = { + org.apache.spark.sql.classic.SparkSession.getActiveSession.isDefined + } + + def getActiveSession: SparkSession = { + org.apache.spark.sql.classic.SparkSession.getActiveSession.getOrElse( + throw new IllegalStateException("No active SparkSession found") + ) + } + + def createSchemaParser(): Schema.Parser = { + // Spark-4.0+ depends on Avro-1.12.0 where validate() is removed and we need to use + // NameValidator interface instead of validate() method. + new Schema.Parser(NameValidator.NO_VALIDATION).setValidateDefaults(false) + } +} diff --git a/sql-plugin-shims/src/main/spark400db173/scala/com/nvidia/spark/rapids/shims/spark400db173/SparkShimServiceProvider.scala b/sql-plugin-shims/src/main/spark400db173/scala/com/nvidia/spark/rapids/shims/spark400db173/SparkShimServiceProvider.scala new file mode 100644 index 00000000000..8f793d05165 --- /dev/null +++ b/sql-plugin-shims/src/main/spark400db173/scala/com/nvidia/spark/rapids/shims/spark400db173/SparkShimServiceProvider.scala @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "400db173"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims.spark400db173 + +import com.nvidia.spark.rapids._ + +import org.apache.spark.SparkEnv + +object SparkShimServiceProvider { + // DB version should conform to "major.minor" and has no patch version. + // Refer to VersionUtils.getVersionForJni + val VERSION = DatabricksShimVersion(4, 0, 0, "17.3") +} + +class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { + + override def getShimVersion: ShimVersion = SparkShimServiceProvider.VERSION + + def matchesVersion(version: String): Boolean = { + val shimEnabledProp = "spark.rapids.shims.spark400db173" + ".enabled" + val shimEnabled = Option(SparkEnv.get) + .flatMap(_.conf.getOption(shimEnabledProp).map(_.toBoolean)) + .getOrElse(true) + + DatabricksShimServiceProvider.matchesVersion( + dbrVersion = "17.3.x", + shimMatchEnabled = shimEnabled, + disclaimer = "Development of support for Databricks 17.3.x is still in progress: " + + "https://github.com/NVIDIA/spark-rapids/issues/14015" + ) + } +} diff --git a/sql-plugin-shims/src/main/spark400db173/scala/org/apache/spark/sql/rapids/ShuffleManagerShims.scala b/sql-plugin-shims/src/main/spark400db173/scala/org/apache/spark/sql/rapids/ShuffleManagerShims.scala new file mode 100644 index 00000000000..2f4f8b5feda --- /dev/null +++ b/sql-plugin-shims/src/main/spark400db173/scala/org/apache/spark/sql/rapids/ShuffleManagerShims.scala @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "400db173"} +spark-rapids-shim-json-lines ***/ +package org.apache.spark.sql.rapids + +import org.apache.spark.TaskContext +import org.apache.spark.shuffle.{ShuffleHandle, ShuffleManager, ShuffleReader, ShuffleReadMetricsReporter} + +object ShuffleManagerShims { + def getReader[K, C]( + manager: ShuffleManager, + handle: ShuffleHandle, + startMapIndex: Int, + endMapIndex: Int, + startPartition: Int, + endPartition: Int, + context: TaskContext, + metrics: ShuffleReadMetricsReporter): ShuffleReader[K, C] = { + manager.getReader(handle, startMapIndex, endMapIndex, startPartition, + endPartition, context, metrics, false) + } +} diff --git a/sql-plugin-shims/src/main/spark400db173/scala/org/apache/spark/storage/ShuffleClientShims.scala b/sql-plugin-shims/src/main/spark400db173/scala/org/apache/spark/storage/ShuffleClientShims.scala new file mode 100644 index 00000000000..12a6ac402df --- /dev/null +++ b/sql-plugin-shims/src/main/spark400db173/scala/org/apache/spark/storage/ShuffleClientShims.scala @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "400db173"} +spark-rapids-shim-json-lines ***/ +package org.apache.spark.storage + +import org.apache.spark.network.shuffle.BlockStoreClient +import org.apache.spark.network.shuffle.checksum.Cause + +object ShuffleClientShims { + def diagnoseCorruption( + client: BlockStoreClient, + host: String, + port: Int, + execId: String, + blockId: BlockId, + checksum: Long, + algorithm: String): Cause = { + client.diagnoseCorruption(host, port, execId, blockId.name, checksum, algorithm) + } +} diff --git a/sql-plugin-shims/src/main/spark401/scala/com/nvidia/spark/rapids/shims/spark401/SparkShimServiceProvider.scala b/sql-plugin-shims/src/main/spark401/scala/com/nvidia/spark/rapids/shims/spark401/SparkShimServiceProvider.scala new file mode 100644 index 00000000000..1779ea4f649 --- /dev/null +++ b/sql-plugin-shims/src/main/spark401/scala/com/nvidia/spark/rapids/shims/spark401/SparkShimServiceProvider.scala @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2025-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "401"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims.spark401 + +import com.nvidia.spark.rapids.SparkShimVersion + +object SparkShimServiceProvider { + val VERSION = SparkShimVersion(4, 0, 1) + val VERSIONNAMES = Seq(s"$VERSION") +} + +class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { + + override def getShimVersion: SparkShimVersion = SparkShimServiceProvider.VERSION + + override def matchesVersion(version: String): Boolean = { + SparkShimServiceProvider.VERSIONNAMES.contains(version) + } +} diff --git a/sql-plugin-shims/src/main/spark402/scala/com/nvidia/spark/rapids/shims/spark402/SparkShimServiceProvider.scala b/sql-plugin-shims/src/main/spark402/scala/com/nvidia/spark/rapids/shims/spark402/SparkShimServiceProvider.scala new file mode 100644 index 00000000000..11650cec175 --- /dev/null +++ b/sql-plugin-shims/src/main/spark402/scala/com/nvidia/spark/rapids/shims/spark402/SparkShimServiceProvider.scala @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "402"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims.spark402 + +import com.nvidia.spark.rapids.SparkShimVersion + +object SparkShimServiceProvider { + val VERSION = SparkShimVersion(4, 0, 2) + val VERSIONNAMES = Seq(s"$VERSION") +} + +class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { + + override def getShimVersion: SparkShimVersion = SparkShimServiceProvider.VERSION + + override def matchesVersion(version: String): Boolean = { + SparkShimServiceProvider.VERSIONNAMES.contains(version) + } +} diff --git a/sql-plugin-shims/src/main/spark411/scala/com/nvidia/spark/rapids/shims/spark411/SparkShimServiceProvider.scala b/sql-plugin-shims/src/main/spark411/scala/com/nvidia/spark/rapids/shims/spark411/SparkShimServiceProvider.scala new file mode 100644 index 00000000000..bd5a848206a --- /dev/null +++ b/sql-plugin-shims/src/main/spark411/scala/com/nvidia/spark/rapids/shims/spark411/SparkShimServiceProvider.scala @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2025-2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "411"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims.spark411 + +import com.nvidia.spark.rapids.SparkShimVersion + +object SparkShimServiceProvider { + val VERSION = SparkShimVersion(4, 1, 1) + val VERSIONNAMES = Seq(s"$VERSION") +} + +class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { + + override def getShimVersion: SparkShimVersion = SparkShimServiceProvider.VERSION + + override def matchesVersion(version: String): Boolean = { + SparkShimServiceProvider.VERSIONNAMES.contains(version) + } +} diff --git a/sql-plugin-shims/src/main/spark411/scala/org/apache/spark/sql/rapids/shims/FileCommitProtocolShims.scala b/sql-plugin-shims/src/main/spark411/scala/org/apache/spark/sql/rapids/shims/FileCommitProtocolShims.scala new file mode 100644 index 00000000000..9f664bdb705 --- /dev/null +++ b/sql-plugin-shims/src/main/spark411/scala/org/apache/spark/sql/rapids/shims/FileCommitProtocolShims.scala @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "411"} +spark-rapids-shim-json-lines ***/ +package org.apache.spark.sql.rapids.shims + +import org.apache.hadoop.mapreduce.TaskAttemptContext + +import org.apache.spark.internal.io.{FileCommitProtocol, FileNameSpec} + +/** + * Shim for FileCommitProtocol.newTaskTempFile API in Spark 4.1.0+. + * Uses the new (spec: FileNameSpec) signature instead of deprecated (ext: String). + */ +object FileCommitProtocolShims { + def newTaskTempFile( + committer: FileCommitProtocol, + taskContext: TaskAttemptContext, + dir: Option[String], + ext: String): String = { + // FileNameSpec(prefix, suffix) - we put ext as suffix with empty prefix + committer.newTaskTempFile(taskContext, dir, FileNameSpec("", ext)) + } + + def newTaskTempFileAbsPath( + committer: FileCommitProtocol, + taskContext: TaskAttemptContext, + absoluteDir: String, + ext: String): String = { + // FileNameSpec(prefix, suffix) - we put ext as suffix with empty prefix + committer.newTaskTempFileAbsPath(taskContext, absoluteDir, FileNameSpec("", ext)) + } +}