Build with JDK 21 (#1556)

* Build with JDK 21

Add -Djava.security.manager=allow jvm flag to:
- tools executed via ctx.actions.run if host jdk is >= 17
- generated test executables if target jdk is >= 17

* allow security manager for coverage worker

* Add ci build with jdk21

* lint

* test all with jdk 21

* Add libxml

* Uncomment JunitRuntimePlatform test

Make sure compilation toolchain matches runtime target version

This test was failing with --java_language_version=21

Because it got compiled with newer jdk than was run.

Author: simuons

.bazelci/presubmit.yml

@@ -90,3 +90,15 @@ tasks:
     platform: ubuntu2004
     run_targets:
     -  "//tools:lint_check"
+  test_rules_scala_jdk21:
+    name: "./test_rules_scala with jdk21"
+    platform: ubuntu2004
+    shell_commands:
+      - sudo apt update && sudo apt install -y libxml2-utils
+      - mv tools/bazel.rc.buildkite tools/bazel.rc
+      - echo "import %workspace%/tools/bazel.rc" > .bazelrc
+      - echo "build --java_language_version=21" >> .bazelrc
+      - echo "build --java_runtime_version=21" >> .bazelrc
+      - echo "build --tool_java_language_version=21" >> .bazelrc
+      - echo "build --tool_java_runtime_version=21" >> .bazelrc
+      - "./test_rules_scala.sh"

WORKSPACE

@@ -203,3 +203,9 @@ repositories(
     ],
     maven_servers = MAVEN_SERVER_URLS,
 )
+
+load("//test/toolchains:jdk.bzl", "remote_jdk21_repositories", "remote_jdk21_toolchains")
+
+remote_jdk21_repositories()
+
+remote_jdk21_toolchains()

scala/private/common_attributes.bzl

@@ -84,6 +84,9 @@ implicit_deps = {
     "_java_runtime": attr.label(
         default = Label("@bazel_tools//tools/jdk:current_java_runtime"),
     ),
+    "_java_host_runtime": attr.label(
+        default = Label("@bazel_tools//tools/jdk:current_host_java_runtime"),
+    ),
     "_scalac": attr.label(
         executable = True,
         cfg = "exec",

scala/private/phases/phase_coverage.bzl

@@ -8,6 +8,10 @@ load(
     "@io_bazel_rules_scala//scala/private:coverage_replacements_provider.bzl",
     _coverage_replacements_provider = "coverage_replacements_provider",
 )
+load(
+    "@io_bazel_rules_scala//scala/private:rule_impls.bzl",
+    _allow_security_manager = "allow_security_manager",
+)
 
 def phase_coverage_library(ctx, p):
     args = struct(
@@ -60,7 +64,7 @@ def _phase_coverage(ctx, p, srcjars):
             outputs = [output_jar],
             executable = ctx.attr._code_coverage_instrumentation_worker.files_to_run,
             execution_requirements = {"supports-workers": "1"},
-            arguments = [args],
+            arguments = ["--jvm_flag=%s" % f for f in _allow_security_manager(ctx)] + [args],
         )
 
         replacements = {input_jar: output_jar}

scala/private/phases/phase_scalafmt.bzl

@@ -7,6 +7,10 @@ load(
     "@io_bazel_rules_scala//scala/private:paths.bzl",
     _scala_extension = "scala_extension",
 )
+load(
+    "//scala/private:rule_impls.bzl",
+    _allow_security_manager = "allow_security_manager",
+)
 
 def phase_scalafmt(ctx, p):
     if ctx.attr.format:
@@ -25,14 +29,15 @@ def _build_format(ctx):
     files = []
     srcs = []
     manifest_content = []
+    jvm_flags = ["-Dfile.encoding=UTF-8"] + _allow_security_manager(ctx)
     for src in ctx.files.srcs:
         # only format scala source files, not generated files
         if src.path.endswith(_scala_extension) and src.is_source:
             srcs.append(src)
             file = ctx.actions.declare_file("{}.fmt.output".format(src.short_path))
             files.append(file)
             ctx.actions.run(
-                arguments = ["--jvm_flag=-Dfile.encoding=UTF-8", _format_args(ctx, src, file)],
+                arguments = ["--jvm_flag=%s" % f for f in jvm_flags] + [_format_args(ctx, src, file)],
                 executable = ctx.executable._fmt,
                 outputs = [file],
                 inputs = [ctx.file.config, src],

scala/private/phases/phase_write_executable.bzl

@@ -5,6 +5,7 @@
 #
 load(
     "@io_bazel_rules_scala//scala/private:rule_impls.bzl",
+    "allow_security_manager",
     "expand_location",
     "first_non_empty",
     "is_windows",
@@ -22,14 +23,12 @@ def phase_write_executable_scalatest(ctx, p):
         ctx.toolchains["@io_bazel_rules_scala//scala:toolchain_type"].scala_test_jvm_flags,
     )
 
-    expanded_jvm_flags = [
-        "-DRULES_SCALA_MAIN_WS_NAME=%s" % ctx.workspace_name,
-        "-DRULES_SCALA_ARGS_FILE=%s" % p.runfiles.args_file.short_path.replace("../", "external/"),
-    ] + expand_location(ctx, final_jvm_flags)
-
     args = struct(
         rjars = p.coverage_runfiles.rjars,
-        jvm_flags = _allow_security_manager(ctx, expanded_jvm_flags),
+        jvm_flags = [
+            "-DRULES_SCALA_MAIN_WS_NAME=%s" % ctx.workspace_name,
+            "-DRULES_SCALA_ARGS_FILE=%s" % p.runfiles.args_file.short_path.replace("../", "external/"),
+        ] + expand_location(ctx, final_jvm_flags) + _allow_security_manager_for_specified_java_runtime(ctx),
         use_jacoco = ctx.configuration.coverage_enabled,
     )
     return _phase_write_executable_default(ctx, p, args)
@@ -44,7 +43,7 @@ def phase_write_executable_repl(ctx, p):
 def phase_write_executable_junit_test(ctx, p):
     args = struct(
         rjars = p.coverage_runfiles.rjars,
-        jvm_flags = _allow_security_manager(ctx, p.jvm_flags + ctx.attr.jvm_flags),
+        jvm_flags = p.jvm_flags + ctx.attr.jvm_flags + _allow_security_manager_for_specified_java_runtime(ctx),
         main_class = "com.google.testing.junit.runner.BazelTestRunner",
         use_jacoco = ctx.configuration.coverage_enabled,
     )
@@ -187,7 +186,12 @@ def _jar_path_based_on_java_bin(ctx):
     jar_path = java_bin_var.rpartition("/")[0] + "/jar"
     return jar_path
 
-def _allow_security_manager(ctx, jvm_flags):
-    java_toolchain = ctx.toolchains["@bazel_tools//tools/jdk:toolchain_type"]
-    java_runtime = specified_java_runtime(ctx, java_toolchain.java.java_runtime if java_toolchain else None)
-    return jvm_flags + (["-Djava.security.manager=allow"] if java_runtime and java_runtime.version >= 17 else [])
+# Allow security manager for generated test executables if they will be run with jdk >= 17
+def _allow_security_manager_for_specified_java_runtime(ctx):
+    return allow_security_manager(
+        ctx,
+        specified_java_runtime(
+            ctx,
+            default_runtime = ctx.attr._java_runtime[java_common.JavaRuntimeInfo],
+        ),
+    )

scala/private/rule_impls.bzl

@@ -121,7 +121,10 @@ def compile_scala(
     )
 
     # scalac_jvm_flags passed in on the target override scalac_jvm_flags passed in on the toolchain
-    final_scalac_jvm_flags = first_non_empty(scalac_jvm_flags, toolchain.scalac_jvm_flags)
+    # TODO: scalac worker is run with @bazel_tools//tools/jdk:runtime_toolchain_type
+    # which is different from rules_java where compilation runtime is used from
+    # @bazel_tools//tools/jdk:toolchain_type
+    final_scalac_jvm_flags = first_non_empty(scalac_jvm_flags, toolchain.scalac_jvm_flags) + allow_security_manager(ctx)
 
     ctx.actions.run(
         inputs = ins,
@@ -219,3 +222,10 @@ def java_bin_windows(ctx):
 
 def is_windows(ctx):
     return ctx.configuration.host_path_separator == ";"
+
+# Return a jvm flag allowing security manager for jdk runtime >= 17
+# If no runtime is supplied then runtime is taken from ctx.attr._java_host_runtime
+# This must be a runtime used in generated java_binary script (usually workers using SecurityManager)
+def allow_security_manager(ctx, runtime = None):
+    java_runtime = runtime if runtime else ctx.attr._java_host_runtime[java_common.JavaRuntimeInfo]
+    return ["-Djava.security.manager=allow"] if java_runtime.version >= 17 else []

scala/scalafmt/phase_scalafmt_ext.bzl

@@ -23,6 +23,9 @@ ext_scalafmt = {
             default = "//scala/scalafmt",
             executable = True,
         ),
+        "_java_host_runtime": attr.label(
+            default = Label("@bazel_tools//tools/jdk:current_host_java_runtime"),
+        ),
         "_runner": attr.label(
             allow_single_file = True,
             default = "//scala/scalafmt:runner",

scala_proto/private/scala_proto_aspect.bzl

@@ -5,6 +5,7 @@ load(
     "//scala/private:rule_impls.bzl",
     "compile_scala",
     "specified_java_compile_toolchain",
+    _allow_security_manager = "allow_security_manager",
 )
 load("//scala/private/toolchain_deps:toolchain_deps.bzl", "find_deps_info_on")
 load(
@@ -73,7 +74,7 @@ def _generate_sources(ctx, toolchain, proto):
 
     ctx.actions.run(
         executable = toolchain.worker,
-        arguments = [toolchain.worker_flags, args],
+        arguments = ["--jvm_flag=%s" % f for f in _allow_security_manager(ctx)] + [toolchain.worker_flags, args],
         inputs = depset(transitive = [descriptors, toolchain.generators_jars]),
         outputs = outputs.values(),
         tools = [toolchain.protoc],
@@ -203,6 +204,9 @@ def make_scala_proto_aspect(*extras):
         "_java_toolchain": attr.label(
             default = Label("@bazel_tools//tools/jdk:current_java_toolchain"),
         ),
+        "_java_host_runtime": attr.label(
+            default = Label("@bazel_tools//tools/jdk:current_host_java_runtime"),
+        ),
     }
     return aspect(
         implementation = _scala_proto_aspect_impl,

test/BUILD

@@ -781,6 +781,8 @@ scala_library(
     srcs = [
         "src/main/scala/scalarules/test/junit/runtime_platform/JunitRuntimePlatformTest.java",
     ],
+    # make sure java compilation toolchain matches runtime toolchain ie --target
+    java_compile_toolchain = "@bazel_tools//tools/jdk:toolchain_java11",
     deps = ["@io_bazel_rules_scala_junit_junit"],
 )
 

test/toolchains/BUILD.bazel

@@ -1,4 +1,26 @@
 load("//scala:scala_toolchain.bzl", "scala_toolchain")
+load(
+    "@bazel_tools//tools/jdk:default_java_toolchain.bzl",
+    "BASE_JDK9_JVM_OPTS",
+    "DEFAULT_JAVACOPTS",
+    "DEFAULT_TOOLCHAIN_CONFIGURATION",
+    "default_java_toolchain",
+)
+
+default_java_toolchain(
+    name = "java21_toolchain",
+    configuration = DEFAULT_TOOLCHAIN_CONFIGURATION,
+    java_runtime = select({
+        "@platforms//os:linux": "@remotejdk21_linux//:jdk",
+        "@platforms//os:macos": "@remotejdk21_macos//:jdk",
+        "@platforms//os:windows": "@remotejdk21_win//:jdk",
+    }),
+    javacopts = DEFAULT_JAVACOPTS,
+    jvm_opts = BASE_JDK9_JVM_OPTS,
+    source_version = "21",
+    target_version = "21",
+    visibility = ["//visibility:public"],
+)
 
 scala_toolchain(
     name = "ast_plus_one_deps_unused_deps_warn_impl",

test/toolchains/jdk.bzl

@@ -0,0 +1,54 @@
+load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
+load("@bazel_tools//tools/jdk:remote_java_repository.bzl", "remote_java_repository")
+
+def remote_jdk21_repositories():
+    maybe(
+        remote_java_repository,
+        name = "remotejdk21_linux",
+        target_compatible_with = [
+            "@platforms//os:linux",
+            "@platforms//cpu:x86_64",
+        ],
+        sha256 = "5ad730fbee6bb49bfff10bf39e84392e728d89103d3474a7e5def0fd134b300a",
+        strip_prefix = "zulu21.32.17-ca-jdk21.0.2-linux_x64",
+        urls = [
+            "https://mirror.bazel.build/cdn.azul.com/zulu/bin/zulu21.32.17-ca-jdk21.0.2-linux_x64.tar.gz",
+            "https://cdn.azul.com/zulu/bin/zulu21.32.17-ca-jdk21.0.2-linux_x64.tar.gz",
+        ],
+        version = "21",
+    )
+
+    maybe(
+        remote_java_repository,
+        name = "remotejdk21_macos",
+        target_compatible_with = [
+            "@platforms//os:macos",
+            "@platforms//cpu:x86_64",
+        ],
+        sha256 = "3ad8fe288eb57d975c2786ae453a036aa46e47ab2ac3d81538ebae2a54d3c025",
+        strip_prefix = "zulu21.32.17-ca-jdk21.0.2-macosx_x64",
+        urls = [
+            "https://mirror.bazel.build/cdn.azul.com/zulu/bin/zulu21.32.17-ca-jdk21.0.2-macosx_x64.tar.gz",
+            "https://cdn.azul.com/zulu/bin/zulu21.32.17-ca-jdk21.0.2-macosx_x64.tar.gz",
+        ],
+        version = "21",
+    )
+
+    maybe(
+        remote_java_repository,
+        name = "remotejdk21_win",
+        target_compatible_with = [
+            "@platforms//os:windows",
+            "@platforms//cpu:x86_64",
+        ],
+        sha256 = "f7cc15ca17295e69c907402dfe8db240db446e75d3b150da7bf67243cded93de",
+        strip_prefix = "zulu21.32.17-ca-jdk21.0.2-win_x64",
+        urls = [
+            "https://mirror.bazel.build/cdn.azul.com/zulu/bin/zulu21.32.17-ca-jdk21.0.2-win_x64.zip",
+            "https://cdn.azul.com/zulu/bin/zulu21.32.17-ca-jdk21.0.2-win_x64.zip",
+        ],
+        version = "21",
+    )
+
+def remote_jdk21_toolchains():
+    native.register_toolchains("//test/toolchains:java21_toolchain_definition")

twitter_scrooge/twitter_scrooge.bzl

@@ -13,6 +13,7 @@ load(
 )
 load(
     "//scala/private:rule_impls.bzl",
+    "allow_security_manager",
     "compile_java",
     "compile_scala",
 )
@@ -205,7 +206,7 @@ def _generate_jvm_code(ctx, label, compile_thrifts, include_thrifts, jar_output,
         # be correctly handled since the executable is a jvm app that will
         # consume the flags on startup.
         #arguments = ["--jvm_flag=%s" % flag for flag in ctx.attr.jvm_flags] +
-        arguments = ["@" + argfile.path],
+        arguments = ["--jvm_flag=%s" % f for f in allow_security_manager(ctx)] + ["@" + argfile.path],
     )
 
 def _compiled_jar_file(actions, scrooge_jar):
@@ -434,6 +435,7 @@ common_attrs = {
             ),
         ],
     ),
+    "_java_host_runtime": attr.label(default = Label("@bazel_tools//tools/jdk:current_host_java_runtime")),
 }
 
 common_aspect_providers = [