diff --git a/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/LegacyBehaviorPolicyShim.scala b/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/LegacyBehaviorPolicyShim.scala index a3220b961cf..f3e4b8f3666 100644 --- a/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/LegacyBehaviorPolicyShim.scala +++ b/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/LegacyBehaviorPolicyShim.scala @@ -33,13 +33,13 @@ spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims -import org.apache.spark.sql.internal.{LegacyBehaviorPolicy, SQLConf} +import org.apache.spark.sql.internal.SQLConf object LegacyBehaviorPolicyShim { - val CORRECTED_STR: String = LegacyBehaviorPolicy.CORRECTED.toString - val EXCEPTION_STR: String = LegacyBehaviorPolicy.EXCEPTION.toString + val CORRECTED_STR: String = "CORRECTED" + val EXCEPTION_STR: String = "EXCEPTION" def isLegacyTimeParserPolicy(): Boolean = { - SQLConf.get.legacyTimeParserPolicy == LegacyBehaviorPolicy.LEGACY + SQLConf.get.legacyTimeParserPolicy.toString == "LEGACY" } } diff --git a/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/NullOutputStreamShim.scala b/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/NullOutputStreamShim.scala index 319964ee6e9..943e2dd3705 100644 --- a/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/NullOutputStreamShim.scala +++ b/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/NullOutputStreamShim.scala @@ -33,8 +33,11 @@ spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims -import org.apache.commons.io.output.NullOutputStream +import java.io.OutputStream object NullOutputStreamShim { - def INSTANCE = NullOutputStream.INSTANCE + val INSTANCE: OutputStream = new OutputStream { + override def write(b: Int): Unit = {} + override def write(b: Array[Byte], off: Int, len: Int): Unit = {} + } } diff --git a/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/spark350/SparkShimServiceProvider.scala b/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/spark350/SparkShimServiceProvider.scala deleted file mode 100644 index 91fcf7cf40a..00000000000 --- a/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/spark350/SparkShimServiceProvider.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "350"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims.spark350 - -import com.nvidia.spark.rapids.SparkShimVersion - -object SparkShimServiceProvider { - val VERSION = SparkShimVersion(3, 5, 0) - val VERSIONNAMES = Seq(s"$VERSION") -} - -class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { - - override def getShimVersion: SparkShimVersion = SparkShimServiceProvider.VERSION - - override def matchesVersion(version: String): Boolean = { - SparkShimServiceProvider.VERSIONNAMES.contains(version) - } -} diff --git a/sql-plugin/src/main/spark350db143/scala/com/nvidia/spark/rapids/shims/spark350db143/SparkShimServiceProvider.scala b/sql-plugin/src/main/spark350db143/scala/com/nvidia/spark/rapids/shims/spark350db143/SparkShimServiceProvider.scala deleted file mode 100644 index 55bcad35ea9..00000000000 --- a/sql-plugin/src/main/spark350db143/scala/com/nvidia/spark/rapids/shims/spark350db143/SparkShimServiceProvider.scala +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2023-2026, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "350db143"} -{"spark": "400db173"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims.spark350db143 - -import com.nvidia.spark.rapids._ - -import org.apache.spark.SparkEnv - -object SparkShimServiceProvider { - // DB version should conform to "major.minor" and has no patch version. - // Refer to VersionUtils.getVersionForJni - val VERSION = DatabricksShimVersion(3, 5, 0, "14.3") -} - -class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { - - override def getShimVersion: ShimVersion = SparkShimServiceProvider.VERSION - - def matchesVersion(version: String): Boolean = { - val shimEnabledProp = "spark.rapids.shims.spark350db143" + ".enabled" - // disabled by default - val shimEnabled = Option(SparkEnv.get) - .flatMap(_.conf.getOption(shimEnabledProp).map(_.toBoolean)) - .getOrElse(true) - - DatabricksShimServiceProvider.matchesVersion( - dbrVersion = "14.3.x", - shimMatchEnabled = shimEnabled, - disclaimer = "Development of support for Databricks 14.3.x is still in progress: " + - "https://github.com/NVIDIA/spark-rapids/issues/10661" - ) - } -} diff --git a/sql-plugin/src/main/spark350db143/scala/org/apache/spark/sql/rapids/shims/SequenceSizeExceededLimitErrorBuilder.scala b/sql-plugin/src/main/spark350db143/scala/org/apache/spark/sql/rapids/shims/SequenceSizeExceededLimitErrorBuilder.scala deleted file mode 100644 index c42f21207a1..00000000000 --- a/sql-plugin/src/main/spark350db143/scala/org/apache/spark/sql/rapids/shims/SequenceSizeExceededLimitErrorBuilder.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2024-2026, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "350db143"} -{"spark": "400"} -{"spark": "400db173"} -{"spark": "401"} -{"spark": "402"} -{"spark": "411"} -spark-rapids-shim-json-lines ***/ -package org.apache.spark.sql.rapids.shims - -import org.apache.spark.sql.errors.QueryExecutionErrors - -trait SequenceSizeExceededLimitErrorBuilder { - def getTooLongSequenceErrorString(sequenceSize: Int, functionName: String): String = { - QueryExecutionErrors.createArrayWithElementsExceedLimitError(functionName, sequenceSize) - .getMessage - } -} diff --git a/sql-plugin/src/main/spark351/scala/com/nvidia/spark/rapids/shims/spark351/SparkShimServiceProvider.scala b/sql-plugin/src/main/spark351/scala/com/nvidia/spark/rapids/shims/spark351/SparkShimServiceProvider.scala deleted file mode 100644 index 8b1bace8aa1..00000000000 --- a/sql-plugin/src/main/spark351/scala/com/nvidia/spark/rapids/shims/spark351/SparkShimServiceProvider.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "351"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims.spark351 - -import com.nvidia.spark.rapids.SparkShimVersion - -object SparkShimServiceProvider { - val VERSION = SparkShimVersion(3, 5, 1) - val VERSIONNAMES = Seq(s"$VERSION", s"$VERSION-SNAPSHOT") -} - -class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { - - override def getShimVersion: SparkShimVersion = SparkShimServiceProvider.VERSION - - override def matchesVersion(version: String): Boolean = { - SparkShimServiceProvider.VERSIONNAMES.contains(version) - } -} diff --git a/sql-plugin/src/main/spark352/scala/com/nvidia/spark/rapids/shims/spark352/SparkShimServiceProvider.scala b/sql-plugin/src/main/spark352/scala/com/nvidia/spark/rapids/shims/spark352/SparkShimServiceProvider.scala deleted file mode 100644 index 5a6ba4d43c9..00000000000 --- a/sql-plugin/src/main/spark352/scala/com/nvidia/spark/rapids/shims/spark352/SparkShimServiceProvider.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "352"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims.spark352 - -import com.nvidia.spark.rapids.SparkShimVersion - -object SparkShimServiceProvider { - val VERSION = SparkShimVersion(3, 5, 2) - val VERSIONNAMES = Seq(s"$VERSION") -} - -class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { - - override def getShimVersion: SparkShimVersion = SparkShimServiceProvider.VERSION - - override def matchesVersion(version: String): Boolean = { - SparkShimServiceProvider.VERSIONNAMES.contains(version) - } -} diff --git a/sql-plugin/src/main/spark353/scala/com/nvidia/spark/rapids/shims/spark353/SparkShimServiceProvider.scala b/sql-plugin/src/main/spark353/scala/com/nvidia/spark/rapids/shims/spark353/SparkShimServiceProvider.scala deleted file mode 100644 index 8eaf51f7177..00000000000 --- a/sql-plugin/src/main/spark353/scala/com/nvidia/spark/rapids/shims/spark353/SparkShimServiceProvider.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "353"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims.spark353 - -import com.nvidia.spark.rapids.SparkShimVersion - -object SparkShimServiceProvider { - val VERSION = SparkShimVersion(3, 5, 3) - val VERSIONNAMES = Seq(s"$VERSION") -} - -class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { - - override def getShimVersion: SparkShimVersion = SparkShimServiceProvider.VERSION - - override def matchesVersion(version: String): Boolean = { - SparkShimServiceProvider.VERSIONNAMES.contains(version) - } -} diff --git a/sql-plugin/src/main/spark354/scala/com/nvidia/spark/rapids/shims/spark354/SparkShimServiceProvider.scala b/sql-plugin/src/main/spark354/scala/com/nvidia/spark/rapids/shims/spark354/SparkShimServiceProvider.scala deleted file mode 100644 index 46d9887b0c8..00000000000 --- a/sql-plugin/src/main/spark354/scala/com/nvidia/spark/rapids/shims/spark354/SparkShimServiceProvider.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "354"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims.spark354 - -import com.nvidia.spark.rapids.SparkShimVersion - -object SparkShimServiceProvider { - val VERSION = SparkShimVersion(3, 5, 4) - val VERSIONNAMES = Seq(s"$VERSION") -} - -class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { - - override def getShimVersion: SparkShimVersion = SparkShimServiceProvider.VERSION - - override def matchesVersion(version: String): Boolean = { - SparkShimServiceProvider.VERSIONNAMES.contains(version) - } -} diff --git a/sql-plugin/src/main/spark355/scala/com/nvidia/spark/rapids/shims/spark355/SparkShimServiceProvider.scala b/sql-plugin/src/main/spark355/scala/com/nvidia/spark/rapids/shims/spark355/SparkShimServiceProvider.scala deleted file mode 100644 index 7175d0eb948..00000000000 --- a/sql-plugin/src/main/spark355/scala/com/nvidia/spark/rapids/shims/spark355/SparkShimServiceProvider.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "355"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims.spark355 - -import com.nvidia.spark.rapids.SparkShimVersion - -object SparkShimServiceProvider { - val VERSION = SparkShimVersion(3, 5, 5) - val VERSIONNAMES = Seq(s"$VERSION") -} - -class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { - - override def getShimVersion: SparkShimVersion = SparkShimServiceProvider.VERSION - - override def matchesVersion(version: String): Boolean = { - SparkShimServiceProvider.VERSIONNAMES.contains(version) - } -} diff --git a/sql-plugin/src/main/spark356/scala/com/nvidia/spark/rapids/shims/spark356/SparkShimServiceProvider.scala b/sql-plugin/src/main/spark356/scala/com/nvidia/spark/rapids/shims/spark356/SparkShimServiceProvider.scala deleted file mode 100644 index 9f55ea238cd..00000000000 --- a/sql-plugin/src/main/spark356/scala/com/nvidia/spark/rapids/shims/spark356/SparkShimServiceProvider.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "356"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims.spark356 - -import com.nvidia.spark.rapids.SparkShimVersion - -object SparkShimServiceProvider { - val VERSION = SparkShimVersion(3, 5, 6) - val VERSIONNAMES = Seq(s"$VERSION") -} - -class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { - - override def getShimVersion: SparkShimVersion = SparkShimServiceProvider.VERSION - - override def matchesVersion(version: String): Boolean = { - SparkShimServiceProvider.VERSIONNAMES.contains(version) - } -} diff --git a/sql-plugin/src/main/spark357/scala/com/nvidia/spark/rapids/shims/spark357/SparkShimServiceProvider.scala b/sql-plugin/src/main/spark357/scala/com/nvidia/spark/rapids/shims/spark357/SparkShimServiceProvider.scala deleted file mode 100644 index 971678a51bf..00000000000 --- a/sql-plugin/src/main/spark357/scala/com/nvidia/spark/rapids/shims/spark357/SparkShimServiceProvider.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "357"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims.spark357 - -import com.nvidia.spark.rapids.SparkShimVersion - -object SparkShimServiceProvider { - val VERSION = SparkShimVersion(3, 5, 7) - val VERSIONNAMES = Seq(s"$VERSION") -} - -class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { - - override def getShimVersion: SparkShimVersion = SparkShimServiceProvider.VERSION - - override def matchesVersion(version: String): Boolean = { - SparkShimServiceProvider.VERSIONNAMES.contains(version) - } -} diff --git a/sql-plugin/src/main/spark358/scala/com/nvidia/spark/rapids/shims/spark358/SparkShimServiceProvider.scala b/sql-plugin/src/main/spark358/scala/com/nvidia/spark/rapids/shims/spark358/SparkShimServiceProvider.scala deleted file mode 100644 index 99585c30d57..00000000000 --- a/sql-plugin/src/main/spark358/scala/com/nvidia/spark/rapids/shims/spark358/SparkShimServiceProvider.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2026, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "358"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims.spark358 - -import com.nvidia.spark.rapids.SparkShimVersion - -object SparkShimServiceProvider { - val VERSION = SparkShimVersion(3, 5, 8) - val VERSIONNAMES = Seq(s"$VERSION") -} - -class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { - - override def getShimVersion: SparkShimVersion = SparkShimServiceProvider.VERSION - - override def matchesVersion(version: String): Boolean = { - SparkShimServiceProvider.VERSIONNAMES.contains(version) - } -}