diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 3ba4981ff..1dafec6ec 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -48,7 +48,8 @@ jobs: - name: Set Python 3 as default for Spark 3 builds if: ${{ contains(matrix.maven_profile, 'spark3') }} - run: pyenv global 3 + # This can be removed once support for Python 2 and Spark 2 is removed and the default python executable is python3 + run: pyenv global 3 && echo "PYSPARK_PYTHON=$(which python3)" >> "$GITHUB_ENV" - name: Build with Maven run: mvn -Pthriftserver ${{ matrix.maven_profile }} -DskipTests -Dmaven.javadoc.skip=true -B -V -e verify diff --git a/integration-test/pom.xml b/integration-test/pom.xml index 209f161dd..97413e0bc 100644 --- a/integration-test/pom.xml +++ b/integration-test/pom.xml @@ -111,6 +111,11 @@ + + org.apache.hadoop + hadoop-hdfs + + org.apache.hadoop hadoop-hdfs diff --git a/pom.xml b/pom.xml index 8c5e10503..a88e449fa 100644 --- a/pom.xml +++ b/pom.xml @@ -81,19 +81,21 @@ 2.10.1 compile 1.7.36 - 1.2.25 + 1.2.26 2.4.5 2.4.5 ${spark.scala-2.11.version} 5.6.0 3.0.0 - 1.9 - 4.5.13 - 4.4.4 + 1.15 + 3.17.0 + 4.5.14 + 4.4.16 2.12.7 2.12.7.1 + 0.8.13 3.1.0 - 9.4.50.v20221201 + 9.4.56.v20240826 4.13.1 0.9.3 4.0.2 @@ -130,6 +132,8 @@ false + + false false @@ -307,6 +311,12 @@ ${commons-codec.version} + + org.apache.commons + commons-lang3 + ${commons-lang3.version} + + io.dropwizard.metrics metrics-core @@ -412,6 +422,18 @@ + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} + + + log4j + log4j + + + + org.apache.hadoop hadoop-hdfs @@ -789,7 +811,7 @@ org.apache.maven.plugins maven-shade-plugin - 3.2.1 + 3.5.0 @@ -810,6 +832,7 @@ false ${project.version} ${skipRTests} + ${skipPySpark2Tests} ${skipPySpark3Tests} ${test.redirectToFile} @@ -839,6 +862,7 @@ false ${project.version} ${skipRTests} + ${skipPySpark2Tests} ${skipPySpark3Tests} D @@ -1081,7 +1105,7 @@ org.jacoco jacoco-maven-plugin - 0.8.2 + ${jacoco.version} @@ -1163,7 +1187,7 @@ scala-2.12 2.12 - 2.12.15 + 2.12.18 @@ -1192,15 +1216,20 @@ spark3 - 3.2.3 + 3.5.6 + 3 + 3.3.4 1.8 0.10.9.7 3.7.0-M11 - 4.1.92.Final + 4.1.96.Final + 2.15.2 + 2.15.2 spark-${spark.version}-bin-hadoop${hadoop.major-minor.version} https://archive.apache.org/dist/spark/spark-${spark.version}/${spark.bin.name}.tgz + true diff --git a/python-api/setup.py b/python-api/setup.py index f7602e9f0..f284edb7d 100644 --- a/python-api/setup.py +++ b/python-api/setup.py @@ -31,7 +31,6 @@ 'cloudpickle>=0.2.1', 'configparser>=3.5.0', 'future>=0.15.2', - 'futures>=3.0.5', 'mock~=3.0.5', 'requests>=2.10.0', 'responses>=0.5.1', @@ -54,6 +53,9 @@ keywords='livy pyspark development', classifiers=CLASSIFIERS, install_requires=requirements, + extras_require={ + ':python_version == "2.7"': ['futures'] + }, setup_requires=['pytest-runner', 'flake8'], tests_require=['pytest'] ) diff --git a/repl/src/test/scala/org/apache/livy/repl/PythonInterpreterSpec.scala b/repl/src/test/scala/org/apache/livy/repl/PythonInterpreterSpec.scala index e2d63e8c4..bce8fd460 100644 --- a/repl/src/test/scala/org/apache/livy/repl/PythonInterpreterSpec.scala +++ b/repl/src/test/scala/org/apache/livy/repl/PythonInterpreterSpec.scala @@ -282,6 +282,11 @@ class Python2InterpreterSpec extends PythonBaseInterpreterSpec { implicit val formats = DefaultFormats + override protected def withFixture(test: NoArgTest): Outcome = { + assume(!sys.props.getOrElse("skipPySpark2Tests", "false").toBoolean, "Skipping PySpark2 tests.") + test() + } + override def createInterpreter(): Interpreter = { val sparkConf = new SparkConf() PythonInterpreter(sparkConf, new SparkEntries(sparkConf)) diff --git a/repl/src/test/scala/org/apache/livy/repl/PythonSessionSpec.scala b/repl/src/test/scala/org/apache/livy/repl/PythonSessionSpec.scala index b54be11a6..486dffef1 100644 --- a/repl/src/test/scala/org/apache/livy/repl/PythonSessionSpec.scala +++ b/repl/src/test/scala/org/apache/livy/repl/PythonSessionSpec.scala @@ -170,7 +170,13 @@ abstract class PythonSessionSpec extends BaseSessionSpec(PySpark) { } } -class Python2SessionSpec extends PythonSessionSpec +class Python2SessionSpec extends PythonSessionSpec { + + override protected def withFixture(test: NoArgTest): Outcome = { + assume(!sys.props.getOrElse("skipPySpark2Tests", "false").toBoolean, "Skipping PySpark2 tests.") + test() + } +} class Python3SessionSpec extends PythonSessionSpec with BeforeAndAfterAll { diff --git a/repl/src/test/scala/org/apache/livy/repl/SQLInterpreterSpec.scala b/repl/src/test/scala/org/apache/livy/repl/SQLInterpreterSpec.scala index 3d9d4aca3..5e839d4c8 100644 --- a/repl/src/test/scala/org/apache/livy/repl/SQLInterpreterSpec.scala +++ b/repl/src/test/scala/org/apache/livy/repl/SQLInterpreterSpec.scala @@ -193,7 +193,7 @@ class SQLInterpreterSpec extends BaseInterpreterSpec { assert(resp.isInstanceOf[Interpreter.ExecuteError]) val error = resp.asInstanceOf[Interpreter.ExecuteError] error.ename should be ("Error") - assert(error.evalue.contains("not found")) + assert(error.evalue.contains("not found") || error.evalue.contains("cannot be found")) } it should "fail if submitting multiple queries" in withInterpreter { interpreter => diff --git a/server/pom.xml b/server/pom.xml index f9c296e51..7e5bc0dc5 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -94,6 +94,11 @@ javax.servlet-api + + org.apache.commons + commons-lang3 + + org.apache.hadoop hadoop-auth diff --git a/server/src/main/scala/org/apache/livy/utils/LivySparkUtils.scala b/server/src/main/scala/org/apache/livy/utils/LivySparkUtils.scala index c94b199b3..c1b4320e9 100644 --- a/server/src/main/scala/org/apache/livy/utils/LivySparkUtils.scala +++ b/server/src/main/scala/org/apache/livy/utils/LivySparkUtils.scala @@ -30,6 +30,16 @@ object LivySparkUtils extends Logging { // For each Spark version we supported, we need to add this mapping relation in case Scala // version cannot be detected from "spark-submit --version". private val _defaultSparkScalaVersion = SortedMap( + // Spark 3.5 + Scala 2.12 + (3, 5) -> "2.12", + // Spark 3.4 + Scala 2.12 + (3, 4) -> "2.12", + // Spark 3.3 + Scala 2.12 + (3, 3) -> "2.12", + // Spark 3.2 + Scala 2.12 + (3, 2) -> "2.12", + // Spark 3.1 + Scala 2.12 + (3, 1) -> "2.12", // Spark 3.0 + Scala 2.12 (3, 0) -> "2.12", // Spark 2.4 + Scala 2.11 @@ -42,7 +52,7 @@ object LivySparkUtils extends Logging { // Supported Spark version private val MIN_VERSION = (2, 2) - private val MAX_VERSION = (3, 1) + private val MAX_VERSION = (3, 6) private val sparkVersionRegex = """version (.*)""".r.unanchored private val scalaVersionRegex = """Scala version (.*), Java""".r.unanchored diff --git a/server/src/main/scala/org/apache/livy/utils/SparkKubernetesApp.scala b/server/src/main/scala/org/apache/livy/utils/SparkKubernetesApp.scala index c4574deef..0f466095a 100644 --- a/server/src/main/scala/org/apache/livy/utils/SparkKubernetesApp.scala +++ b/server/src/main/scala/org/apache/livy/utils/SparkKubernetesApp.scala @@ -31,7 +31,7 @@ import scala.util.control.NonFatal import io.fabric8.kubernetes.api.model._ import io.fabric8.kubernetes.api.model.networking.v1.{Ingress, IngressBuilder} import io.fabric8.kubernetes.client.{Config, ConfigBuilder, _} -import org.apache.commons.lang.StringUtils +import org.apache.commons.lang3.StringUtils import org.apache.livy.{LivyConf, Logging} diff --git a/thriftserver/server/src/test/scala/org/apache/livy/thriftserver/ThriftServerSuites.scala b/thriftserver/server/src/test/scala/org/apache/livy/thriftserver/ThriftServerSuites.scala index c9f91e1d3..c0f4a36d3 100644 --- a/thriftserver/server/src/test/scala/org/apache/livy/thriftserver/ThriftServerSuites.scala +++ b/thriftserver/server/src/test/scala/org/apache/livy/thriftserver/ThriftServerSuites.scala @@ -516,7 +516,9 @@ class BinaryThriftServerSuite extends ThriftServerBaseTest with CommonThriftTest statement.close() } } - assert(caught.getMessage.contains("Database 'invalid_database' not found")) + val message = caught.getMessage + assert(message.contains("Database 'invalid_database' not found") || + message.contains("The schema `invalid_database` cannot be found")) } } @@ -530,8 +532,9 @@ class BinaryThriftServerSuite extends ThriftServerBaseTest with CommonThriftTest statement.close() } } - assert(caught.getMessage.replaceAll("`", "") - .contains("Table or view not found: global_temp.invalid_table")) + val message = caught.getMessage.replaceAll("`", "") + assert(message .contains("Table or view not found: global_temp.invalid_table") || + message.contains("The table or view global_temp.invalid_table cannot be found")) } } diff --git a/thriftserver/session/pom.xml b/thriftserver/session/pom.xml index bd66ebf28..d8d606a75 100644 --- a/thriftserver/session/pom.xml +++ b/thriftserver/session/pom.xml @@ -55,12 +55,6 @@ org.apache.spark spark-hive_${scala.binary.version} provided - - - * - * - - ${project.groupId}