[LIVY-1017] Support Java 17, set up JDK 17 based tests

.github/workflows/integration-tests.yaml

@@ -31,6 +31,11 @@ jobs:
         maven_profile:
         - "-Pscala-2.11 -Pspark2 -DskipRTests"
         - "-Pscala-2.12 -Pspark3"
+        jdk_path:
+        - "/usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java"
+        include:
+        - maven_profile: "-Pscala-2.12 -Pspark3"
+          jdk_path: "/usr/lib/jvm/java-17-openjdk-amd64/bin/java"
     steps:
     - 
       name: Checkout
@@ -50,6 +55,9 @@ jobs:
       if: ${{ contains(matrix.maven_profile, 'spark3') }}
       # This can be removed once support for Python 2 and Spark 2 is removed and the default python executable is python3
       run: pyenv global 3 && echo "PYSPARK_PYTHON=$(which python3)" >> "$GITHUB_ENV"
+    -
+      name: Set JDK version
+      run: update-alternatives --set java ${{ matrix.jdk_path }}
     - 
       name: Build with Maven
       run: mvn -Pthriftserver ${{ matrix.maven_profile }} -DskipTests -Dmaven.javadoc.skip=true -B -V -e verify

.github/workflows/unit-tests.yaml

@@ -30,6 +30,11 @@ jobs:
         - "-Pscala-2.11 -Pspark2"
         - "-Pscala-2.12 -Pspark2"
         - "-Pscala-2.12 -Pspark3"
+        jdk_path:
+        - "/usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java"
+        include:
+        - maven_profile: "-Pscala-2.12 -Pspark3"
+          jdk_path: "/usr/lib/jvm/java-17-openjdk-amd64/bin/java"
     steps:
     - 
       name: Checkout
@@ -44,7 +49,10 @@ jobs:
         key: ${{ runner.os }}-maven-${{ hashFiles('pom.xml', '*/pom.xml', 'thriftserver/*/pom.xml', 'core/*/pom.xml', 'repl/*/pom.xml', 'scala-api/*/pom.xml') }}
         restore-keys: |
           ${{ runner.os }}-maven-
-    - 
+    -
+      name: Set JDK version
+      run: update-alternatives --set java ${{ matrix.jdk_path }}
+    -
       name: Build with Maven
       run: mvn -Pthriftserver ${{ matrix.maven_profile }} -DskipITs -Dmaven.javadoc.skip=true -B -V -e verify
     -

bin/livy-server

@@ -102,6 +102,21 @@ start_livy_server() {
     LIVY_CLASSPATH="$LIVY_CLASSPATH:$YARN_CONF_DIR"
   fi
 
+  # Required for Java 17 support
+  LIVY_SERVER_JAVA_OPTS+=" -XX:+IgnoreUnrecognizedVMOptions"
+  LIVY_SERVER_JAVA_OPTS+=" --add-opens=java.base/java.lang.invoke=ALL-UNNAMED"
+  LIVY_SERVER_JAVA_OPTS+=" --add-opens=java.base/java.lang=ALL-UNNAMED"
+  LIVY_SERVER_JAVA_OPTS+=" --add-opens=java.base/java.nio=ALL-UNNAMED"
+  LIVY_SERVER_JAVA_OPTS+=" --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED"
+  LIVY_SERVER_JAVA_OPTS+=" --add-opens=java.base/sun.nio.ch=ALL-UNNAMED"
+  LIVY_SERVER_JAVA_OPTS+=" --add-opens=java.base/java.net=ALL-UNNAMED"
+  LIVY_SERVER_JAVA_OPTS+=" --add-opens=java.base/java.io=ALL-UNNAMED"
+  LIVY_SERVER_JAVA_OPTS+=" --add-opens=java.base/sun.util.calendar=ALL-UNNAMED"
+  LIVY_SERVER_JAVA_OPTS+=" --add-exports=java.base/sun.net.dns=ALL-UNNAMED"
+  LIVY_SERVER_JAVA_OPTS+=" --add-exports=java.base/sun.net.util=ALL-UNNAMED"
+  LIVY_SERVER_JAVA_OPTS+=" --add-exports=java.base/sun.security.x509=ALL_UNNAMED"
+  LIVY_SERVER_JAVA_OPTS+=" --add-exports=java.base/sun.security.util=ALL-UNNAMED
+
   command="$RUNNER $LIVY_SERVER_JAVA_OPTS -cp $LIVY_CLASSPATH:$CLASSPATH org.apache.livy.server.LivyServer"
 
   if [ $1 = "old" ]; then

dev/docker/livy-dev-base/Dockerfile

@@ -35,6 +35,7 @@ RUN apt-get install -qq \
     git \
     libkrb5-dev \
     openjdk-8-jdk-headless \
+    openjdk-17-jdk-headless \
     r-base \
     maven \
     software-properties-common \

dev/spark/bin/spark-submit

@@ -82,5 +82,26 @@ if [ -n "$HADOOP_CONF_DIR" ]; then
   DRIVER_CP="$HADOOP_CONF_DIR:$DRIVER_CP"
 fi
 
-echo "Running Spark: " $JAVA_HOME/bin/java $DRIVER_OPTS org.apache.spark.deploy.SparkSubmit "$@" >&2
+# Taken from Spark's extraJavaTestArgs / JavaModuleOptions
+# Since we are skipping SparkContext.supplementJavaModuleOptions(), we need to add these
+DRIVER_OPTS+=" -XX:+IgnoreUnrecognizedVMOptions"
+DRIVER_OPTS+=" --add-opens=java.base/java.lang=ALL-UNNAMED"
+DRIVER_OPTS+=" --add-opens=java.base/java.lang.invoke=ALL-UNNAMED"
+DRIVER_OPTS+=" --add-opens=java.base/java.lang.reflect=ALL-UNNAMED"
+DRIVER_OPTS+=" --add-opens=java.base/java.io=ALL-UNNAMED"
+DRIVER_OPTS+=" --add-opens=java.base/java.net=ALL-UNNAMED"
+DRIVER_OPTS+=" --add-opens=java.base/java.nio=ALL-UNNAMED"
+DRIVER_OPTS+=" --add-opens=java.base/java.util=ALL-UNNAMED"
+DRIVER_OPTS+=" --add-opens=java.base/java.util.concurrent=ALL-UNNAMED"
+DRIVER_OPTS+=" --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED"
+DRIVER_OPTS+=" --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED"
+DRIVER_OPTS+=" --add-opens=java.base/sun.nio.ch=ALL-UNNAMED"
+DRIVER_OPTS+=" --add-opens=java.base/sun.nio.cs=ALL-UNNAMED"
+DRIVER_OPTS+=" --add-opens=java.base/sun.security.action=ALL-UNNAMED"
+DRIVER_OPTS+=" --add-opens=java.base/sun.util.calendar=ALL-UNNAMED"
+DRIVER_OPTS+=" -Djdk.reflect.useDirectMethodHandle=false"
+DRIVER_OPTS+=" --add-exports=java.base/sun.net.dns=ALL-UNNAMED"
+DRIVER_OPTS+=" --add-exports=java.base/sun.net.util=ALL-UNNAMED"
+
+echo "Running Spark: " $JAVA_HOME/bin/java $DRIVER_OPTS -cp "$DRIVER_CP" org.apache.spark.deploy.SparkSubmit "$@" >&2
 exec $JAVA_HOME/bin/java $DRIVER_OPTS -cp "$DRIVER_CP" org.apache.spark.deploy.SparkSubmit "$@"

integration-test/src/main/scala/org/apache/livy/test/framework/MiniCluster.scala

@@ -220,6 +220,11 @@ class MiniCluster(config: Map[String, String]) extends Cluster with MiniClusterU
     filtered.mkString(File.pathSeparator)
   }
 
+  private def extraJavaTestArgs: Seq[String] = {
+    Option(System.getProperty("extraJavaTestArgs"))
+      .map(_.split("\\s+").toSeq).getOrElse(Nil)
+  }
+
   override def deploy(): Unit = {
     if (_tempDir.exists()) {
       FileUtils.deleteQuietly(_tempDir)
@@ -240,8 +245,10 @@ class MiniCluster(config: Map[String, String]) extends Cluster with MiniClusterU
 
     _configDir = mkdir("hadoop-conf")
     saveProperties(config, new File(configDir, "cluster.conf"))
-    hdfs = Some(start(MiniHdfsMain.getClass, new File(configDir, "core-site.xml")))
-    yarn = Some(start(MiniYarnMain.getClass, new File(configDir, "yarn-site.xml")))
+    hdfs = Some(start(MiniHdfsMain.getClass, new File(configDir, "core-site.xml"),
+      extraJavaTestArgs))
+    yarn = Some(start(MiniYarnMain.getClass, new File(configDir, "yarn-site.xml"),
+      extraJavaTestArgs))
     runLivy()
 
     _hdfsScrathDir = fs.makeQualified(new Path("/"))
@@ -261,7 +268,7 @@ class MiniCluster(config: Map[String, String]) extends Cluster with MiniClusterU
       .map { args =>
         Seq(args, s"-Djacoco.args=$args")
       }.getOrElse(Nil)
-    val localLivy = start(MiniLivyMain.getClass, confFile, extraJavaArgs = jacocoArgs)
+    val localLivy = start(MiniLivyMain.getClass, confFile, jacocoArgs ++ extraJavaTestArgs)
 
     val props = loadProperties(confFile)
     _livyEndpoint = config.getOrElse("livyEndpoint", props("livy.server.server-url"))

pom.xml

@@ -106,6 +106,21 @@
     <scalatest.version>3.0.8</scalatest.version>
     <scalatra.version>2.6.5</scalatra.version>
     <java.version>1.8</java.version>
+    <extraJavaTestArgs>
+      -XX:+IgnoreUnrecognizedVMOptions
+      --add-opens=java.base/java.lang.invoke=ALL-UNNAMED
+      --add-opens=java.base/java.lang=ALL-UNNAMED
+      --add-opens=java.base/java.nio=ALL-UNNAMED
+      --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
+      --add-opens=java.base/sun.nio.ch=ALL-UNNAMED
+      --add-opens=java.base/java.net=ALL-UNNAMED
+      --add-opens=java.base/java.io=ALL-UNNAMED
+      --add-opens=java.base/sun.util.calendar=ALL-UNNAMED
+      --add-exports=java.base/sun.net.dns=ALL-UNNAMED
+      --add-exports=java.base/sun.net.util=ALL-UNNAMED
+      --add-exports=java.base/sun.security.x509=ALL-UNNAMED
+      --add-exports=java.base/sun.security.util=ALL-UNNAMED
+    </extraJavaTestArgs>
     <test.redirectToFile>true</test.redirectToFile>
     <execution.root>${user.dir}</execution.root>
     <spark.home>${execution.root}/dev/spark</spark.home>
@@ -837,15 +852,15 @@
             </systemProperties>
             <redirectTestOutputToFile>${test.redirectToFile}</redirectTestOutputToFile>
             <useFile>${test.redirectToFile}</useFile>
-            <argLine>${argLine} -Xmx2g</argLine>
+            <argLine>${argLine} -Xmx2g ${extraJavaTestArgs}</argLine>
             <failIfNoTests>false</failIfNoTests>
           </configuration>
         </plugin>
 
         <plugin>
           <groupId>org.scalatest</groupId>
           <artifactId>scalatest-maven-plugin</artifactId>
-          <version>2.0.0</version>
+          <version>2.2.0</version>
           <configuration>
             <environmentVariables>
               <LIVY_TEST>true</LIVY_TEST>
@@ -864,12 +879,13 @@
               <skipRTests>${skipRTests}</skipRTests>
               <skipPySpark2Tests>${skipPySpark2Tests}</skipPySpark2Tests>
               <skipPySpark3Tests>${skipPySpark3Tests}</skipPySpark3Tests>
+              <extraJavaTestArgs>${extraJavaTestArgs}</extraJavaTestArgs>
             </systemProperties>
             <stdout>D</stdout>
             <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
             <junitxml>.</junitxml>
             <filereports>WDF TestSuite.txt</filereports>
-            <argLine>${argLine} -Xmx2g</argLine>
+            <argLine>${argLine} -Xmx2g ${extraJavaTestArgs}</argLine>
           </configuration>
           <executions>
             <execution>