diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml
index 1dafec6ec..754f26f58 100644
--- a/.github/workflows/integration-tests.yaml
+++ b/.github/workflows/integration-tests.yaml
@@ -31,6 +31,11 @@ jobs:
maven_profile:
- "-Pscala-2.11 -Pspark2 -DskipRTests"
- "-Pscala-2.12 -Pspark3"
+ jdk_path:
+ - "/usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java"
+ include:
+ - maven_profile: "-Pscala-2.12 -Pspark3"
+ jdk_path: "/usr/lib/jvm/java-17-openjdk-amd64/bin/java"
steps:
-
name: Checkout
@@ -50,6 +55,9 @@ jobs:
if: ${{ contains(matrix.maven_profile, 'spark3') }}
# This can be removed once support for Python 2 and Spark 2 is removed and the default python executable is python3
run: pyenv global 3 && echo "PYSPARK_PYTHON=$(which python3)" >> "$GITHUB_ENV"
+ -
+ name: Set JDK version
+ run: update-alternatives --set java ${{ matrix.jdk_path }}
-
name: Build with Maven
run: mvn -Pthriftserver ${{ matrix.maven_profile }} -DskipTests -Dmaven.javadoc.skip=true -B -V -e verify
diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml
index afddbf5d8..f946fd97c 100644
--- a/.github/workflows/unit-tests.yaml
+++ b/.github/workflows/unit-tests.yaml
@@ -30,6 +30,11 @@ jobs:
- "-Pscala-2.11 -Pspark2"
- "-Pscala-2.12 -Pspark2"
- "-Pscala-2.12 -Pspark3"
+ jdk_path:
+ - "/usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java"
+ include:
+ - maven_profile: "-Pscala-2.12 -Pspark3"
+ jdk_path: "/usr/lib/jvm/java-17-openjdk-amd64/bin/java"
steps:
-
name: Checkout
@@ -44,7 +49,10 @@ jobs:
key: ${{ runner.os }}-maven-${{ hashFiles('pom.xml', '*/pom.xml', 'thriftserver/*/pom.xml', 'core/*/pom.xml', 'repl/*/pom.xml', 'scala-api/*/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- -
+ -
+ name: Set JDK version
+ run: update-alternatives --set java ${{ matrix.jdk_path }}
+ -
name: Build with Maven
run: mvn -Pthriftserver ${{ matrix.maven_profile }} -DskipITs -Dmaven.javadoc.skip=true -B -V -e verify
-
diff --git a/bin/livy-server b/bin/livy-server
index a0e2fb7e2..b463423dc 100755
--- a/bin/livy-server
+++ b/bin/livy-server
@@ -102,6 +102,21 @@ start_livy_server() {
LIVY_CLASSPATH="$LIVY_CLASSPATH:$YARN_CONF_DIR"
fi
+ # Required for Java 17 support
+ LIVY_SERVER_JAVA_OPTS+=" -XX:+IgnoreUnrecognizedVMOptions"
+ LIVY_SERVER_JAVA_OPTS+=" --add-opens=java.base/java.lang.invoke=ALL-UNNAMED"
+ LIVY_SERVER_JAVA_OPTS+=" --add-opens=java.base/java.lang=ALL-UNNAMED"
+ LIVY_SERVER_JAVA_OPTS+=" --add-opens=java.base/java.nio=ALL-UNNAMED"
+ LIVY_SERVER_JAVA_OPTS+=" --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED"
+ LIVY_SERVER_JAVA_OPTS+=" --add-opens=java.base/sun.nio.ch=ALL-UNNAMED"
+ LIVY_SERVER_JAVA_OPTS+=" --add-opens=java.base/java.net=ALL-UNNAMED"
+ LIVY_SERVER_JAVA_OPTS+=" --add-opens=java.base/java.io=ALL-UNNAMED"
+ LIVY_SERVER_JAVA_OPTS+=" --add-opens=java.base/sun.util.calendar=ALL-UNNAMED"
+ LIVY_SERVER_JAVA_OPTS+=" --add-exports=java.base/sun.net.dns=ALL-UNNAMED"
+ LIVY_SERVER_JAVA_OPTS+=" --add-exports=java.base/sun.net.util=ALL-UNNAMED"
+ LIVY_SERVER_JAVA_OPTS+=" --add-exports=java.base/sun.security.x509=ALL_UNNAMED"
+ LIVY_SERVER_JAVA_OPTS+=" --add-exports=java.base/sun.security.util=ALL-UNNAMED
+
command="$RUNNER $LIVY_SERVER_JAVA_OPTS -cp $LIVY_CLASSPATH:$CLASSPATH org.apache.livy.server.LivyServer"
if [ $1 = "old" ]; then
diff --git a/dev/docker/livy-dev-base/Dockerfile b/dev/docker/livy-dev-base/Dockerfile
index 25da08725..07711a5f5 100644
--- a/dev/docker/livy-dev-base/Dockerfile
+++ b/dev/docker/livy-dev-base/Dockerfile
@@ -35,6 +35,7 @@ RUN apt-get install -qq \
git \
libkrb5-dev \
openjdk-8-jdk-headless \
+ openjdk-17-jdk-headless \
r-base \
maven \
software-properties-common \
diff --git a/dev/spark/bin/spark-submit b/dev/spark/bin/spark-submit
index 7d78f032b..f3883cc37 100755
--- a/dev/spark/bin/spark-submit
+++ b/dev/spark/bin/spark-submit
@@ -82,5 +82,26 @@ if [ -n "$HADOOP_CONF_DIR" ]; then
DRIVER_CP="$HADOOP_CONF_DIR:$DRIVER_CP"
fi
-echo "Running Spark: " $JAVA_HOME/bin/java $DRIVER_OPTS org.apache.spark.deploy.SparkSubmit "$@" >&2
+# Taken from Spark's extraJavaTestArgs / JavaModuleOptions
+# Since we are skipping SparkContext.supplementJavaModuleOptions(), we need to add these
+DRIVER_OPTS+=" -XX:+IgnoreUnrecognizedVMOptions"
+DRIVER_OPTS+=" --add-opens=java.base/java.lang=ALL-UNNAMED"
+DRIVER_OPTS+=" --add-opens=java.base/java.lang.invoke=ALL-UNNAMED"
+DRIVER_OPTS+=" --add-opens=java.base/java.lang.reflect=ALL-UNNAMED"
+DRIVER_OPTS+=" --add-opens=java.base/java.io=ALL-UNNAMED"
+DRIVER_OPTS+=" --add-opens=java.base/java.net=ALL-UNNAMED"
+DRIVER_OPTS+=" --add-opens=java.base/java.nio=ALL-UNNAMED"
+DRIVER_OPTS+=" --add-opens=java.base/java.util=ALL-UNNAMED"
+DRIVER_OPTS+=" --add-opens=java.base/java.util.concurrent=ALL-UNNAMED"
+DRIVER_OPTS+=" --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED"
+DRIVER_OPTS+=" --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED"
+DRIVER_OPTS+=" --add-opens=java.base/sun.nio.ch=ALL-UNNAMED"
+DRIVER_OPTS+=" --add-opens=java.base/sun.nio.cs=ALL-UNNAMED"
+DRIVER_OPTS+=" --add-opens=java.base/sun.security.action=ALL-UNNAMED"
+DRIVER_OPTS+=" --add-opens=java.base/sun.util.calendar=ALL-UNNAMED"
+DRIVER_OPTS+=" -Djdk.reflect.useDirectMethodHandle=false"
+DRIVER_OPTS+=" --add-exports=java.base/sun.net.dns=ALL-UNNAMED"
+DRIVER_OPTS+=" --add-exports=java.base/sun.net.util=ALL-UNNAMED"
+
+echo "Running Spark: " $JAVA_HOME/bin/java $DRIVER_OPTS -cp "$DRIVER_CP" org.apache.spark.deploy.SparkSubmit "$@" >&2
exec $JAVA_HOME/bin/java $DRIVER_OPTS -cp "$DRIVER_CP" org.apache.spark.deploy.SparkSubmit "$@"
diff --git a/integration-test/src/main/scala/org/apache/livy/test/framework/MiniCluster.scala b/integration-test/src/main/scala/org/apache/livy/test/framework/MiniCluster.scala
index e5b40871e..f1d51671a 100644
--- a/integration-test/src/main/scala/org/apache/livy/test/framework/MiniCluster.scala
+++ b/integration-test/src/main/scala/org/apache/livy/test/framework/MiniCluster.scala
@@ -220,6 +220,11 @@ class MiniCluster(config: Map[String, String]) extends Cluster with MiniClusterU
filtered.mkString(File.pathSeparator)
}
+ private def extraJavaTestArgs: Seq[String] = {
+ Option(System.getProperty("extraJavaTestArgs"))
+ .map(_.split("\\s+").toSeq).getOrElse(Nil)
+ }
+
override def deploy(): Unit = {
if (_tempDir.exists()) {
FileUtils.deleteQuietly(_tempDir)
@@ -240,8 +245,10 @@ class MiniCluster(config: Map[String, String]) extends Cluster with MiniClusterU
_configDir = mkdir("hadoop-conf")
saveProperties(config, new File(configDir, "cluster.conf"))
- hdfs = Some(start(MiniHdfsMain.getClass, new File(configDir, "core-site.xml")))
- yarn = Some(start(MiniYarnMain.getClass, new File(configDir, "yarn-site.xml")))
+ hdfs = Some(start(MiniHdfsMain.getClass, new File(configDir, "core-site.xml"),
+ extraJavaTestArgs))
+ yarn = Some(start(MiniYarnMain.getClass, new File(configDir, "yarn-site.xml"),
+ extraJavaTestArgs))
runLivy()
_hdfsScrathDir = fs.makeQualified(new Path("/"))
@@ -261,7 +268,7 @@ class MiniCluster(config: Map[String, String]) extends Cluster with MiniClusterU
.map { args =>
Seq(args, s"-Djacoco.args=$args")
}.getOrElse(Nil)
- val localLivy = start(MiniLivyMain.getClass, confFile, extraJavaArgs = jacocoArgs)
+ val localLivy = start(MiniLivyMain.getClass, confFile, jacocoArgs ++ extraJavaTestArgs)
val props = loadProperties(confFile)
_livyEndpoint = config.getOrElse("livyEndpoint", props("livy.server.server-url"))
diff --git a/pom.xml b/pom.xml
index a88e449fa..e8588f9a6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -106,6 +106,21 @@
3.0.8
2.6.5
1.8
+
+ -XX:+IgnoreUnrecognizedVMOptions
+ --add-opens=java.base/java.lang.invoke=ALL-UNNAMED
+ --add-opens=java.base/java.lang=ALL-UNNAMED
+ --add-opens=java.base/java.nio=ALL-UNNAMED
+ --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
+ --add-opens=java.base/sun.nio.ch=ALL-UNNAMED
+ --add-opens=java.base/java.net=ALL-UNNAMED
+ --add-opens=java.base/java.io=ALL-UNNAMED
+ --add-opens=java.base/sun.util.calendar=ALL-UNNAMED
+ --add-exports=java.base/sun.net.dns=ALL-UNNAMED
+ --add-exports=java.base/sun.net.util=ALL-UNNAMED
+ --add-exports=java.base/sun.security.x509=ALL-UNNAMED
+ --add-exports=java.base/sun.security.util=ALL-UNNAMED
+
true
${user.dir}
${execution.root}/dev/spark
@@ -837,7 +852,7 @@
${test.redirectToFile}
${test.redirectToFile}
- ${argLine} -Xmx2g
+ ${argLine} -Xmx2g ${extraJavaTestArgs}
false
@@ -845,7 +860,7 @@
org.scalatest
scalatest-maven-plugin
- 2.0.0
+ 2.2.0
true
@@ -864,12 +879,13 @@
${skipRTests}
${skipPySpark2Tests}
${skipPySpark3Tests}
+ ${extraJavaTestArgs}
D
${project.build.directory}/surefire-reports
.
WDF TestSuite.txt
- ${argLine} -Xmx2g
+ ${argLine} -Xmx2g ${extraJavaTestArgs}