8374169: too many unsafe intrinsics (simplify unsafe intrinsic processing)

src/hotspot/share/c1/c1_Compiler.cpp

@@ -109,21 +109,25 @@ bool Compiler::is_intrinsic_supported(const methodHandle& method) {
 
 bool Compiler::is_intrinsic_supported(vmIntrinsics::ID id) {
   switch (id) {
-  case vmIntrinsics::_compareAndSetLong:
+  case vmIntrinsics::_compareAndSetInt: //FIXME
+  case vmIntrinsics::_compareAndSetLong: //FIXME
+  case vmIntrinsics::_compareAndSetReferenceMO:
+    // all platforms must support at least these three
     break;
-  case vmIntrinsics::_getAndAddInt:
+  //case vmIntrinsics::_getAndOperatePrimitiveBitsMO: //FIXME
+  case vmIntrinsics::_getAndAddInt:    //FIXME
     if (!VM_Version::supports_atomic_getadd4()) return false;
     break;
-  case vmIntrinsics::_getAndAddLong:
+  case vmIntrinsics::_getAndAddLong:    //FIXME
     if (!VM_Version::supports_atomic_getadd8()) return false;
     break;
-  case vmIntrinsics::_getAndSetInt:
+  case vmIntrinsics::_getAndSetInt:    //FIXME
     if (!VM_Version::supports_atomic_getset4()) return false;
     break;
-  case vmIntrinsics::_getAndSetLong:
+  case vmIntrinsics::_getAndSetLong:    //FIXME
     if (!VM_Version::supports_atomic_getset8()) return false;
     break;
-  case vmIntrinsics::_getAndSetReference:
+  case vmIntrinsics::_getAndSetReferenceMO:
 #ifdef _LP64
     if (!UseCompressedOops && !VM_Version::supports_atomic_getset8()) return false;
     if (UseCompressedOops && !VM_Version::supports_atomic_getset4()) return false;
@@ -176,50 +180,10 @@ bool Compiler::is_intrinsic_supported(vmIntrinsics::ID id) {
   case vmIntrinsics::_dpow:
   case vmIntrinsics::_fmaD:
   case vmIntrinsics::_fmaF:
-  case vmIntrinsics::_getReference:
-  case vmIntrinsics::_getBoolean:
-  case vmIntrinsics::_getByte:
-  case vmIntrinsics::_getShort:
-  case vmIntrinsics::_getChar:
-  case vmIntrinsics::_getInt:
-  case vmIntrinsics::_getLong:
-  case vmIntrinsics::_getFloat:
-  case vmIntrinsics::_getDouble:
-  case vmIntrinsics::_putReference:
-  case vmIntrinsics::_putBoolean:
-  case vmIntrinsics::_putByte:
-  case vmIntrinsics::_putShort:
-  case vmIntrinsics::_putChar:
-  case vmIntrinsics::_putInt:
-  case vmIntrinsics::_putLong:
-  case vmIntrinsics::_putFloat:
-  case vmIntrinsics::_putDouble:
-  case vmIntrinsics::_getReferenceVolatile:
-  case vmIntrinsics::_getBooleanVolatile:
-  case vmIntrinsics::_getByteVolatile:
-  case vmIntrinsics::_getShortVolatile:
-  case vmIntrinsics::_getCharVolatile:
-  case vmIntrinsics::_getIntVolatile:
-  case vmIntrinsics::_getLongVolatile:
-  case vmIntrinsics::_getFloatVolatile:
-  case vmIntrinsics::_getDoubleVolatile:
-  case vmIntrinsics::_putReferenceVolatile:
-  case vmIntrinsics::_putBooleanVolatile:
-  case vmIntrinsics::_putByteVolatile:
-  case vmIntrinsics::_putShortVolatile:
-  case vmIntrinsics::_putCharVolatile:
-  case vmIntrinsics::_putIntVolatile:
-  case vmIntrinsics::_putLongVolatile:
-  case vmIntrinsics::_putFloatVolatile:
-  case vmIntrinsics::_putDoubleVolatile:
-  case vmIntrinsics::_getShortUnaligned:
-  case vmIntrinsics::_getCharUnaligned:
-  case vmIntrinsics::_getIntUnaligned:
-  case vmIntrinsics::_getLongUnaligned:
-  case vmIntrinsics::_putShortUnaligned:
-  case vmIntrinsics::_putCharUnaligned:
-  case vmIntrinsics::_putIntUnaligned:
-  case vmIntrinsics::_putLongUnaligned:
+  case vmIntrinsics::_getPrimitiveBitsMO:
+  case vmIntrinsics::_putPrimitiveBitsMO:
+  case vmIntrinsics::_getReferenceMO:
+  case vmIntrinsics::_putReferenceMO:
   case vmIntrinsics::_Preconditions_checkIndex:
   case vmIntrinsics::_Preconditions_checkLongIndex:
   case vmIntrinsics::_updateCRC32:
@@ -230,8 +194,6 @@ bool Compiler::is_intrinsic_supported(vmIntrinsics::ID id) {
   case vmIntrinsics::_updateDirectByteBufferCRC32C:
 #endif
   case vmIntrinsics::_vectorizedMismatch:
-  case vmIntrinsics::_compareAndSetInt:
-  case vmIntrinsics::_compareAndSetReference:
   case vmIntrinsics::_getCharStringU:
   case vmIntrinsics::_putCharStringU:
 #ifdef JFR_HAVE_INTRINSICS

src/hotspot/share/c1/c1_GraphBuilder.hpp

@@ -351,6 +351,7 @@ class GraphBuilder {
   // inliners
   bool try_inline(           ciMethod* callee, bool holder_known, bool ignore_return, Bytecodes::Code bc = Bytecodes::_illegal, Value receiver = nullptr);
   bool try_inline_intrinsics(ciMethod* callee, bool ignore_return = false);
+  bool try_inline_polymorphic_intrinsic(ciMethod* callee, bool ignore_return = false);
   bool try_inline_full(      ciMethod* callee, bool holder_known, bool ignore_return, Bytecodes::Code bc = Bytecodes::_illegal, Value receiver = nullptr);
   bool try_inline_jsr(int jsr_dest_bci);
 
@@ -373,10 +374,13 @@ class GraphBuilder {
   void pop_scope();
   void pop_scope_for_jsr();
 
-  void append_unsafe_get(ciMethod* callee, BasicType t, bool is_volatile);
-  void append_unsafe_put(ciMethod* callee, BasicType t, bool is_volatile);
-  void append_unsafe_CAS(ciMethod* callee);
-  void append_unsafe_get_and_set(ciMethod* callee, bool is_add);
+  void append_unsafe_get(ciMethod* callee, vmIntrinsics::MemoryOrder mo, BasicType t, int prefix_size);
+  void append_unsafe_put(ciMethod* callee, vmIntrinsics::MemoryOrder mo, BasicType t, int prefix_size);
+  void append_unsafe_CAS(ciMethod* callee, vmIntrinsics::MemoryOrder mo, BasicType t, int prefix_size);
+  void append_unsafe_get_and_set(ciMethod* callee, vmIntrinsics::MemoryOrder mo, BasicType t, vmIntrinsics::BitsOperation op, int prefix_size);
+  Value adjust_unsafe_container(Value x, BasicType from, BasicType to);
+  Value adjust_unsafe_offset(Value offset);
+
   void append_char_access(ciMethod* callee, bool is_store);
   void append_alloc_array_copy(ciMethod* callee);
 

src/hotspot/share/c1/c1_Instruction.cpp

@@ -543,6 +543,42 @@ void BlockBegin::set_end(BlockEnd* new_end) { // Assumes that no predecessor of
 }
 
 
+Intrinsic*
+Intrinsic::with_polymorphic_prefix(vmIntrinsics::MemoryOrder memory_order,
+                                   BasicType basic_type,
+                                   vmIntrinsics::BitsOperation bits_op) {
+  vmIntrinsics::PolymorphicPrefix pfx = vmIntrinsics::polymorphic_prefix(id());
+  switch (pfx) {
+  case vmIntrinsics::PP_MO:
+    assert(memory_order != vmIntrinsics::MO_NONE, "");
+    assert(basic_type == T_OBJECT || basic_type == T_ILLEGAL, "");
+    assert(bits_op == vmIntrinsics::OP_NONE, "");
+    break;
+  case vmIntrinsics::PP_MO_BT:
+    assert(memory_order != vmIntrinsics::MO_NONE, "");
+    assert(is_java_primitive(basic_type), "");
+    assert(bits_op == vmIntrinsics::OP_NONE, "");
+    break;
+  case vmIntrinsics::PP_MO_BT_OP:
+    assert(memory_order != vmIntrinsics::MO_NONE, "");
+    assert(is_java_primitive(basic_type), "");
+    assert(bits_op != vmIntrinsics::OP_NONE, "");
+    break;
+  default:
+    ShouldNotReachHere();
+  }
+  // make sure previous settings are empty
+  assert(_basic_type == T_ILLEGAL, "");
+  assert(_memory_order == vmIntrinsics::MO_NONE, "");
+  assert(_bits_op == vmIntrinsics::OP_NONE, "");
+  // install the settings; some might still be empty
+  _memory_order = memory_order;
+  _basic_type = basic_type;
+  _bits_op = bits_op;
+  return this;
+}
+
+
 void BlockBegin::disconnect_edge(BlockBegin* from, BlockBegin* to) {
   // disconnect any edges between from and to
 #ifndef PRODUCT

src/hotspot/share/c1/c1_Instruction.hpp

@@ -1501,14 +1501,18 @@ LEAF(MonitorExit, AccessMonitor)
   }
 };
 
-
 LEAF(Intrinsic, StateSplit)
  private:
   vmIntrinsics::ID _id;
   ArgsNonNullState _nonnull_state;
   Values*          _args;
   Value            _recv;
 
+  // polymorphic prefix arguments, if applicable, set neutrally if not:
+  vmIntrinsics::MemoryOrder   _memory_order;
+  BasicType                   _basic_type;
+  vmIntrinsics::BitsOperation _bits_op;
+
  public:
   // preserves_state can be set to true for Intrinsics
   // which are guaranteed to preserve register state across any slow
@@ -1528,6 +1532,9 @@ LEAF(Intrinsic, StateSplit)
   , _id(id)
   , _args(args)
   , _recv(nullptr)
+  , _memory_order(vmIntrinsics::MO_NONE)
+  , _basic_type(T_ILLEGAL)
+  , _bits_op(vmIntrinsics::OP_NONE)
   {
     assert(args != nullptr, "args must exist");
     ASSERT_VALUES
@@ -1567,6 +1574,25 @@ LEAF(Intrinsic, StateSplit)
     StateSplit::input_values_do(f);
     for (int i = 0; i < _args->length(); i++) f->visit(_args->adr_at(i));
   }
+
+  // prefix arguments
+  vmIntrinsics::MemoryOrder memory_order() const {
+    assert(_memory_order != vmIntrinsics::MO_NONE, "must be present");
+    return _memory_order;
+  }
+  BasicType basic_type() const {
+    assert(_basic_type != T_ILLEGAL, "must be present");
+    return _basic_type;
+  }
+  vmIntrinsics::BitsOperation bits_op() const {
+    assert(_bits_op != vmIntrinsics::OP_NONE, "must be present");
+    return _bits_op;
+  }
+  // one-time setup after creation, required for polymorphic intrinsics
+  Intrinsic* with_polymorphic_prefix(
+      vmIntrinsics::MemoryOrder memory_order,
+      BasicType basic_type = T_OBJECT,
+      vmIntrinsics::BitsOperation bits_op = vmIntrinsics::OP_NONE);
 };
 
 

src/hotspot/share/c1/c1_LIRGenerator.cpp

@@ -2869,14 +2869,21 @@ void LIRGenerator::do_Intrinsic(Intrinsic* x) {
     do_PreconditionsCheckIndex(x, T_LONG);
     break;
 
-  case vmIntrinsics::_compareAndSetReference:
-    do_CompareAndSwap(x, objectType);
-    break;
-  case vmIntrinsics::_compareAndSetInt:
-    do_CompareAndSwap(x, intType);
-    break;
-  case vmIntrinsics::_compareAndSetLong:
-    do_CompareAndSwap(x, longType);
+  case vmIntrinsics::_compareAndSetReferenceMO:
+  case vmIntrinsics::_compareAndSetPrimitiveBitsMO:
+    switch (x->basic_type()) {
+    case T_OBJECT:
+      do_CompareAndSwap(x, objectType);
+      break;
+    case T_INT:
+      do_CompareAndSwap(x, intType);
+      break;
+    case T_LONG:
+      do_CompareAndSwap(x, longType);
+      break;
+    default:
+      ShouldNotReachHere();
+    }
     break;
 
   case vmIntrinsics::_loadFence :
@@ -2917,7 +2924,10 @@ void LIRGenerator::do_Intrinsic(Intrinsic* x) {
     do_blackhole(x);
     break;
 
-  default: ShouldNotReachHere(); break;
+  default:
+    guarantee(false, "unhandled do_Intrinsic %d: %s",
+              (int)x->id(), vmIntrinsics::name_at(x->id()));
+    break;
   }
 }
 

src/hotspot/share/classfile/classFileParser.cpp

@@ -4947,11 +4947,16 @@ static void check_methods_for_intrinsics(const InstanceKlass* ik,
       if (CheckIntrinsics) {
         // Check if an intrinsic is defined for method 'method',
         // but the method is not annotated with @IntrinsicCandidate.
-        if (method->intrinsic_id() != vmIntrinsics::_none &&
-            !method->intrinsic_candidate()) {
-              tty->print("Compiler intrinsic is defined for method [%s], "
-              "but the method is not annotated with @IntrinsicCandidate.%s",
+        vmIntrinsicID iid = method->intrinsic_id();
+        bool is_wrapper = (iid != vmIntrinsics::_none &&
+                           vmIntrinsics::flags_for(iid) == vmIntrinsics::F_PW);
+        bool expect_annotation = (iid != vmIntrinsics::_none && !is_wrapper);
+        if (method->intrinsic_candidate() != expect_annotation) {
+              tty->print("Compiler intrinsic%s is defined for method [%s], "
+              "but the method is %s annotated with @IntrinsicCandidate.%s",
+              (is_wrapper ? " wrapper" : ""),
               method->name_and_sig_as_C_string(),
+              (expect_annotation ? "not" : "incorrectly"),
               NOT_DEBUG(" Method will not be inlined.") DEBUG_ONLY(" Exiting.")
             );
           tty->cr();
@@ -4980,6 +4985,7 @@ static void check_methods_for_intrinsics(const InstanceKlass* ik,
       // The check is potentially expensive, therefore it is available
       // only in debug builds.
 
+      int fails = 0;  // collect all the failing methods at once
       for (auto id : EnumRange<vmIntrinsicID>{}) {
         if (vmIntrinsics::_compiledLambdaForm == id) {
           // The _compiledLamdbdaForm intrinsic is a special marker for bytecode
@@ -5008,16 +5014,20 @@ static void check_methods_for_intrinsics(const InstanceKlass* ik,
           if (!match) {
             char buf[1000];
             tty->print("Compiler intrinsic is defined for method [%s], "
-                       "but the method is not available in class [%s].%s",
+                       "but the method is not available in class [%s].",
                         vmIntrinsics::short_name_as_C_string(id, buf, sizeof(buf)),
-                        ik->name()->as_C_string(),
-                        NOT_DEBUG("") DEBUG_ONLY(" Exiting.")
+                        ik->name()->as_C_string()
             );
             tty->cr();
-            DEBUG_ONLY(vm_exit(1));
+            ++fails;
           }
         }
       } // end for
+      if (fails > 0) {
+        tty->print_cr("Compiler intrinsic not available for %d methods.%s",
+                      fails, NOT_DEBUG("") DEBUG_ONLY(" Exiting."));
+        DEBUG_ONLY(vm_exit(1));
+      }
     } // CheckIntrinsics
 #endif // ASSERT
   }