Merge ac999a4 into 5961cb4

Author: genie-openj9

closed/openjdk-tag.gmk

@@ -1 +1 @@
-OPENJDK_TAG := jdk-11.0.5+10
+OPENJDK_TAG := jdk-11.0.6+2

make/Help.gmk

@@ -119,7 +119,7 @@ print-configurations:
 run-test-prebuilt:
 	@( cd $(topdir) && \
 	    $(MAKE) --no-print-directory -r -R -I make/common/ -f make/RunTestsPrebuilt.gmk \
-	    run-test-prebuilt TEST="$(TEST)" )
+	    run-test-prebuilt CUSTOM_MAKE_DIR=$(CUSTOM_MAKE_DIR) TEST="$(TEST)" )
 
 ALL_GLOBAL_TARGETS := help print-configurations run-test-prebuilt
 

make/Main.gmk

@@ -637,6 +637,7 @@ else
   # Declare dependencies between hotspot-<variant>* targets
   $(foreach v, $(JVM_VARIANTS), \
       $(eval hotspot-$v: hotspot-$v-gensrc hotspot-$v-libs) \
+      $(eval hotspot-$v-gensrc: java.base-copy) \
       $(eval hotspot-$v-libs: hotspot-$v-gensrc java.base-copy) \
   )
 

make/RunTests.gmk

@@ -82,8 +82,8 @@ ifneq ($(TEST_VM_OPTS), )
 endif
 
 $(eval $(call ParseKeywordVariable, TEST_OPTS, \
-    KEYWORDS := JOBS TIMEOUT, \
-    STRING_KEYWORDS := VM_OPTIONS, \
+    SINGLE_KEYWORDS := JOBS TIMEOUT_FACTOR, \
+    STRING_KEYWORDS := VM_OPTIONS JAVA_OPTIONS, \
 ))
 
 # Helper function to propagate TEST_OPTS values.
@@ -102,10 +102,14 @@ ifeq ($(OPENJDK_TARGET_OS), windows)
   ifndef _NT_SYMBOL_PATH
     # Can't use PathList here as it adds quotes around the value.
     _NT_SYMBOL_PATH := \
-        $(subst $(SPACE),;, $(foreach p, $(sort $(dir $(wildcard \
-        $(addprefix $(SYMBOLS_IMAGE_DIR)/bin/, *.pdb */*.pdb)))), $(call FixPath, $p)))
+        $(subst $(SPACE),;,$(strip \
+            $(foreach p, $(sort $(dir $(wildcard \
+                $(addprefix $(SYMBOLS_IMAGE_DIR)/bin/, *.pdb */*.pdb)))), \
+              $(call FixPath, $p) \
+            ) \
+        ))
     export _NT_SYMBOL_PATH
-    $(info _NT_SYMBOL_PATH $(_NT_SYMBOL_PATH))
+    $(info _NT_SYMBOL_PATH=$(_NT_SYMBOL_PATH))
   endif
 endif
 
@@ -140,23 +144,51 @@ endif
 GTEST_LAUNCHER_DIRS := $(patsubst %/gtestLauncher, %, $(wildcard $(TEST_IMAGE_DIR)/hotspot/gtest/*/gtestLauncher))
 GTEST_VARIANTS := $(strip $(patsubst $(TEST_IMAGE_DIR)/hotspot/gtest/%, %, $(GTEST_LAUNCHER_DIRS)))
 
+################################################################################
+# Setup global test running parameters
+################################################################################
+
+# Each factor variable comes in 3 variants. The first one is reserved for users
+# to use on command line. The other two are for predifined configurations in JDL
+# and for machine specific configurations respectively.
+TEST_JOBS_FACTOR ?= 1
+TEST_JOBS_FACTOR_JDL ?= 1
+TEST_JOBS_FACTOR_MACHINE ?= 1
+
+ifeq ($(TEST_JOBS), 0)
+  # Concurrency based on min(cores / 2, 12) * TEST_JOBS_FACTOR
+  TEST_JOBS := $(shell $(AWK) \
+    'BEGIN { \
+      c = $(NUM_CORES) / 2; \
+      if (c > 12) c = 12; \
+      c = c * $(TEST_JOBS_FACTOR); \
+      c = c * $(TEST_JOBS_FACTOR_JDL); \
+      c = c * $(TEST_JOBS_FACTOR_MACHINE); \
+      if (c < 1) c = 1; \
+      printf "%.0f", c; \
+    }')
+endif
+
 ################################################################################
 # Parse control variables
 ################################################################################
 
 ifneq ($(TEST_OPTS), )
   # Inform the user
   $(info Running tests using TEST_OPTS control variable '$(TEST_OPTS)')
+endif
 
-  $(eval $(call SetTestOpt,VM_OPTIONS,JTREG))
-  $(eval $(call SetTestOpt,VM_OPTIONS,GTEST))
+$(eval $(call SetTestOpt,VM_OPTIONS,JTREG))
+$(eval $(call SetTestOpt,JAVA_OPTIONS,JTREG))
+$(eval $(call SetTestOpt,VM_OPTIONS,GTEST))
+$(eval $(call SetTestOpt,JAVA_OPTIONS,GTEST))
 
-  $(eval $(call SetTestOpt,JOBS,JTREG))
-  $(eval $(call SetTestOpt,TIMEOUT,JTREG))
-endif
+$(eval $(call SetTestOpt,JOBS,JTREG))
+$(eval $(call SetTestOpt,TIMEOUT_FACTOR,JTREG))
 
 $(eval $(call ParseKeywordVariable, JTREG, \
-    KEYWORDS := JOBS TIMEOUT TEST_MODE ASSERT VERBOSE RETAIN MAX_MEM, \
+    SINGLE_KEYWORDS := JOBS TIMEOUT_FACTOR TEST_MODE ASSERT VERBOSE RETAIN MAX_MEM \
+        EXTRA_PROBLEM_LISTS KEYWORDS, \
     STRING_KEYWORDS := OPTIONS JAVA_OPTIONS VM_OPTIONS, \
 ))
 
@@ -166,8 +198,8 @@ ifneq ($(JTREG), )
 endif
 
 $(eval $(call ParseKeywordVariable, GTEST, \
-    KEYWORDS := REPEAT, \
-    STRING_KEYWORDS := OPTIONS VM_OPTIONS, \
+    SINGLE_KEYWORDS := REPEAT, \
+    STRING_KEYWORDS := OPTIONS VM_OPTIONS JAVA_OPTIONS, \
 ))
 
 ifneq ($(GTEST), )
@@ -396,15 +428,16 @@ define SetupRunGtestTestBody
     $1_GTEST_REPEAT :=--gtest_repeat=$$(GTEST_REPEAT)
   endif
 
-  run-test-$1:
+  run-test-$1: $(TEST_PREREQS)
 	$$(call LogWarn)
 	$$(call LogWarn, Running test '$$($1_TEST)')
 	$$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR))
 	$$(call ExecuteWithLog, $$($1_TEST_SUPPORT_DIR)/gtest, \
 	    $$(FIXPATH) $$(TEST_IMAGE_DIR)/hotspot/gtest/$$($1_VARIANT)/gtestLauncher \
-	         -jdk $(JDK_IMAGE_DIR) $$($1_GTEST_FILTER) \
-	         --gtest_output=xml:$$($1_TEST_RESULTS_DIR)/gtest.xml \
-	         $$($1_GTEST_REPEAT) $$(GTEST_OPTIONS) $$(GTEST_VM_OPTIONS) \
+	        -jdk $(JDK_IMAGE_DIR) $$($1_GTEST_FILTER) \
+	        --gtest_output=xml:$$($1_TEST_RESULTS_DIR)/gtest.xml \
+	        $$($1_GTEST_REPEAT) $$(GTEST_OPTIONS) $$(GTEST_VM_OPTIONS) \
+	        $$($1_GTEST_JAVA_OPTIONS) \
 	        > >($(TEE) $$($1_TEST_RESULTS_DIR)/gtest.txt) \
 	    && $$(ECHO) $$$$? > $$($1_EXITCODE) \
 	    || $$(ECHO) $$$$? > $$($1_EXITCODE) \
@@ -475,12 +508,11 @@ define SetupRunJtregTestBody
 
   $1_TEST_NAME := $$(strip $$(patsubst jtreg:%, %, $$($1_TEST)))
 
-  $1_COMPONENT := \
+  $1_TEST_ROOT := \
       $$(strip $$(foreach root, $$(JTREG_TESTROOTS), \
-        $$(if $$(filter $$(root)%, $$(JTREG_TOPDIR)/$$($1_TEST_NAME)), \
-          $$(lastword $$(subst /, $$(SPACE), $$(root))) \
-        ) \
+        $$(if $$(filter $$(root)%, $$(JTREG_TOPDIR)/$$($1_TEST_NAME)), $$(root)) \
       ))
+  $1_COMPONENT := $$(lastword $$(subst /, $$(SPACE), $$($1_TEST_ROOT)))
   # This will work only as long as just hotspot has the additional "jtreg" directory
   ifeq ($$($1_COMPONENT), jtreg)
     $1_COMPONENT := hotspot
@@ -503,6 +535,9 @@ define SetupRunJtregTestBody
   $$(eval $$(call SetJtregValue,$1,JTREG_BASIC_OPTIONS))
   $$(eval $$(call SetJtregValue,$1,JTREG_PROBLEM_LIST))
 
+  # Only the problem list for the current test root should be used.
+  $1_JTREG_PROBLEM_LIST := $$(filter $$($1_TEST_ROOT)%, $$($1_JTREG_PROBLEM_LIST))
+
   ifneq ($(TEST_JOBS), 0)
     $$(eval $$(call SetJtregValue,$1,JTREG_JOBS,$$(TEST_JOBS)))
   else
@@ -515,9 +550,9 @@ define SetupRunJtregTestBody
 
   # SPARC is in general slower per core so need to scale up timeouts a bit.
   ifeq ($(OPENJDK_TARGET_CPU_ARCH), sparc)
-    JTREG_TIMEOUT ?= 8
+    JTREG_TIMEOUT_FACTOR ?= 8
   else
-    JTREG_TIMEOUT ?= 4
+    JTREG_TIMEOUT_FACTOR ?= 4
   endif
   JTREG_VERBOSE ?= fail,error,summary
   JTREG_RETAIN ?= fail,error
@@ -529,10 +564,10 @@ define SetupRunJtregTestBody
 
   $1_JTREG_BASIC_OPTIONS += -$$($1_JTREG_TEST_MODE) \
       -verbose:$$(JTREG_VERBOSE) -retain:$$(JTREG_RETAIN) \
-      -concurrency:$$($1_JTREG_JOBS) -timeoutFactor:$$(JTREG_TIMEOUT) \
+      -concurrency:$$($1_JTREG_JOBS) -timeoutFactor:$$(JTREG_TIMEOUT_FACTOR) \
       -vmoption:-XX:MaxRAMPercentage=$$($1_JTREG_MAX_RAM_PERCENTAGE)
 
-  $1_JTREG_BASIC_OPTIONS += -automatic -keywords:\!ignore -ignore:quiet
+  $1_JTREG_BASIC_OPTIONS += -automatic -ignore:quiet
 
   # Make it possible to specify the JIB_DATA_DIR for tests using the
   # JIB Artifact resolver
@@ -562,8 +597,16 @@ define SetupRunJtregTestBody
     $1_JTREG_BASIC_OPTIONS += $$(addprefix -exclude:, $$($1_JTREG_PROBLEM_LIST))
   endif
 
-  ifneq ($$(JIB_JAR), )
-    $1_JTREG_BASIC_OPTIONS += -cpa:$$(JIB_JAR)
+  ifneq ($$(JTREG_EXTRA_PROBLEM_LISTS), )
+    # Accept both absolute paths as well as relative to the current test root.
+    $1_JTREG_BASIC_OPTIONS += $$(addprefix -exclude:, $$(wildcard \
+        $$(JTREG_EXTRA_PROBLEM_LISTS) \
+        $$(addprefix $$($1_TEST_ROOT)/, $$(JTREG_EXTRA_PROBLEM_LISTS)) \
+    ))
+  endif
+
+  ifneq ($$(JIB_HOME), )
+    $1_JTREG_BASIC_OPTIONS += -e:JIB_HOME=$$(JIB_HOME)
   endif
 
   $1_JTREG_BASIC_OPTIONS += -e:TEST_IMAGE_GRAAL_DIR=${TEST_IMAGE_DIR}/hotspot/jtreg/graal
@@ -572,10 +615,21 @@ define SetupRunJtregTestBody
     $1_JTREG_LAUNCHER_OPTIONS += -Djava.library.path="$(JTREG_FAILURE_HANDLER_DIR)"
   endif
 
+  ifneq ($$(JTREG_KEYWORDS), )
+    # The keywords string may contain problematic characters and may be quoted
+    # already when it arrives here. Remove any existing quotes and replace them
+    # with one set of single quotes.
+    $1_JTREG_KEYWORDS := \
+        $$(strip $$(subst $$(SQUOTE),,$$(subst $$(DQUOTE),,$$(JTREG_KEYWORDS))))
+    ifneq ($$($1_JTREG_KEYWORDS), )
+      $1_JTREG_BASIC_OPTIONS += -k:'$$($1_JTREG_KEYWORDS)'
+    endif
+  endif
+
   clean-workdir-$1:
 	$$(RM) -r $$($1_TEST_SUPPORT_DIR)
 
-  run-test-$1: clean-workdir-$1
+  run-test-$1: clean-workdir-$1 $(TEST_PREREQS)
 	$$(call LogWarn)
 	$$(call LogWarn, Running test '$$($1_TEST)')
 	$$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR))
@@ -662,7 +716,7 @@ define SetupRunSpecialTestBody
     $$(error Invalid special test specification: $$($1_TEST_NAME))
   endif
 
-  run-test-$1:
+  run-test-$1: $(TEST_PREREQS)
 	$$(call LogWarn)
 	$$(call LogWarn, Running test '$$($1_TEST)')
 	$$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR))

make/RunTestsPrebuilt.gmk

@@ -49,10 +49,11 @@ TOPDIR := $(strip $(patsubst %/make/, %, $(dir $(makefile_path))))
 # given.
 # Note: No spaces are allowed around the arguments.
 #
-# $1: The name of the argument
+# $1: The name of the variable
 # $2: The default value, if any, or OPTIONAL (do not provide a default but
 #     do not exit if it is missing)
 # $3: If NO_CHECK, disable checking for target file/directory existence
+#     If MKDIR, create the default directory
 define SetupVariable
   ifeq ($$($1), )
     ifeq ($2, )
@@ -75,10 +76,17 @@ define SetupVariable
   endif
   # If $1 has a value (is not optional), and $3 is not set (to NO_CHECK),
   # and if wildcard is empty, then complain that the file is missing.
-  ifeq ($$(strip $$(if $$($1), , OPTIONAL) $$(wildcard $$($1)) $3), )
-    $$(info Error: Prebuilt variable $1 points to missing file/directory:)
-    $$(info '$$($1)')
-    $$(error Cannot continue.)
+  ifeq ($3, MKDIR)
+    ifneq ($$(findstring $$(LOG), info debug trace), )
+      $$(info Creating directory for $1)
+    endif
+    $$(shell mkdir -p $$($1))
+  else ifneq ($3, NO_CHECK)
+    ifeq ($$(strip $$(if $$($1), , OPTIONAL) $$(wildcard $$($1))), )
+      $$(info Error: Prebuilt variable $1 points to missing file/directory:)
+      $$(info '$$($1)')
+      $$(error Cannot continue.)
+    endif
   endif
 endef
 
@@ -106,14 +114,14 @@ endef
 # Verify that user has given correct additional input.
 
 # These variables are absolutely necessary
-$(eval $(call SetupVariable,OUTPUTDIR))
+$(eval $(call SetupVariable,OUTPUTDIR,$(TOPDIR)/build/run-test-prebuilt,MKDIR))
 $(eval $(call SetupVariable,BOOT_JDK))
 $(eval $(call SetupVariable,JT_HOME))
 
 # These can have default values based on the ones above
 $(eval $(call SetupVariable,JDK_IMAGE_DIR,$(OUTPUTDIR)/images/jdk))
 $(eval $(call SetupVariable,TEST_IMAGE_DIR,$(OUTPUTDIR)/images/test))
-$(eval $(call SetupVariable,SYMBOLS_IMAGE_DIR,$(OUTPUTDIR)/images/symbols))
+$(eval $(call SetupVariable,SYMBOLS_IMAGE_DIR,$(OUTPUTDIR)/images/symbols,NO_CHECK))
 
 # Provide default values for tools that we need
 $(eval $(call SetupVariable,MAKE,make,NO_CHECK))
@@ -202,8 +210,8 @@ endif
 
 ifeq ($(OPENJDK_TARGET_OS), windows)
   ifeq ($(wildcard $(TEST_IMAGE_DIR)/bin/fixpath.exe), )
-    $$(info Error: fixpath is missing from test image '$(TEST_IMAGE_DIR)')
-    $$(error Cannot continue.)
+    $(info Error: fixpath is missing from test image '$(TEST_IMAGE_DIR)')
+    $(error Cannot continue.)
   endif
   FIXPATH := $(TEST_IMAGE_DIR)/bin/fixpath.exe -c
   PATH_SEP:=;
@@ -214,27 +222,32 @@ endif
 
 # Check number of cores and memory in MB
 ifeq ($(OPENJDK_TARGET_OS), linux)
-    NUM_CORES := $(shell $(CAT) /proc/cpuinfo  | $(GREP) -c processor)
-    MEMORY_SIZE := $(shell \
+  NUM_CORES := $(shell $(CAT) /proc/cpuinfo  | $(GREP) -c processor)
+  MEMORY_SIZE := $(shell \
       $(EXPR) `$(CAT) /proc/meminfo | $(GREP) MemTotal | $(AWK) '{print $$2}'` / 1024 \
-    )
+  )
 else ifeq ($(OPENJDK_TARGET_OS), macosx)
-    NUM_CORES := $(shell /usr/sbin/sysctl -n hw.ncpu)
-    MEMORY_SIZE := $(shell $(EXPR) `/usr/sbin/sysctl -n hw.memsize` / 1024 / 1024)
+  NUM_CORES := $(shell /usr/sbin/sysctl -n hw.ncpu)
+  MEMORY_SIZE := $(shell $(EXPR) `/usr/sbin/sysctl -n hw.memsize` / 1024 / 1024)
 else ifeq ($(OPENJDK_TARGET_OS), solaris)
-    NUM_CORES := $(shell LC_MESSAGES=C /usr/sbin/psrinfo -v | $(GREP) -c on-line)
-    MEMORY_SIZE := $(shell \
+  NUM_CORES := $(shell LC_MESSAGES=C /usr/sbin/psrinfo -v | $(GREP) -c on-line)
+  MEMORY_SIZE := $(shell \
       /usr/sbin/prtconf 2> /dev/null | $(GREP) "^Memory [Ss]ize" | $(AWK) '{print $$3}' \
-    )
+  )
 else ifeq ($(OPENJDK_TARGET_OS), windows)
-    NUM_CORES := $(NUMBER_OF_PROCESSORS)
-    MEMORY_SIZE := $(shell \
+  NUM_CORES := $(NUMBER_OF_PROCESSORS)
+  MEMORY_SIZE := $(shell \
       $(EXPR) `wmic computersystem get totalphysicalmemory -value | $(GREP) = \
           | $(CUT) -d "=" -f 2-` / 1024 / 1024 \
-    )
-else
-    NUM_CORES := 1
-    MEMORY_SIZE := 1024
+  )
+endif
+ifeq ($(NUM_CORES), )
+  $(warn Could not find number of CPUs, assuming 1)
+  NUM_CORES := 1
+endif
+ifeq ($(MEMORY_SIZE), )
+  $(warn Could not find memory size, assuming 1024 MB)
+  MEMORY_SIZE := 1024
 endif
 
 ################################################################################
@@ -289,9 +302,6 @@ run-test-prebuilt:
 	@$(RM) -f $(MAKESUPPORT_OUTPUTDIR)/exit-with-error
 	@cd $(TOPDIR) && $(MAKE) $(MAKE_ARGS) -f make/RunTests.gmk run-test \
 	    TEST="$(TEST)"
-	@if test -f $(MAKESUPPORT_OUTPUTDIR)/exit-with-error ; then \
-	  exit 1 ; \
-	fi
 
 all: run-test-prebuilt
 

make/RunTestsPrebuiltSpec.gmk

@@ -124,7 +124,7 @@ JLINK := $(FIXPATH) $(JLINK_CMD)
 JMOD := $(FIXPATH) $(JMOD_CMD)
 JARSIGNER := $(FIXPATH) $(JARSIGNER_CMD)
 
-BUILD_JAVA := $(JAVA)
+BUILD_JAVA := $(JDK_IMAGE_DIR)/bin/JAVA
 ################################################################################
 # Some common tools. Assume most common name and no path.
 AWK := awk
@@ -172,3 +172,17 @@ UNZIP := unzip
 EXPR := expr
 FILE := file
 HG := hg
+
+# On Solaris gnu versions of some tools are required.
+ifeq ($(OPENJDK_BUILD_OS), solaris)
+  AWK := gawk
+  GREP := ggrep
+  EGREP := ggrep -E
+  FGREP := grep -F
+  SED := gsed
+  TAR := gtar
+endif
+
+ifeq ($(OPENJDK_BUILD_OS), windows)
+  CYGPATH := cygpath
+endif