diff --git a/src/hotspot/share/c1/c1_Canonicalizer.cpp b/src/hotspot/share/c1/c1_Canonicalizer.cpp index 573e1ac24d73c..5000adb544092 100644 --- a/src/hotspot/share/c1/c1_Canonicalizer.cpp +++ b/src/hotspot/share/c1/c1_Canonicalizer.cpp @@ -842,7 +842,6 @@ void Canonicalizer::do_OsrEntry (OsrEntry* x) {} void Canonicalizer::do_ExceptionObject(ExceptionObject* x) {} void Canonicalizer::do_UnsafeGet (UnsafeGet* x) {} void Canonicalizer::do_UnsafePut (UnsafePut* x) {} -void Canonicalizer::do_UnsafeGetAndSet(UnsafeGetAndSet* x) {} void Canonicalizer::do_ProfileCall (ProfileCall* x) {} void Canonicalizer::do_ProfileReturnType(ProfileReturnType* x) {} void Canonicalizer::do_ProfileInvoke (ProfileInvoke* x) {} diff --git a/src/hotspot/share/c1/c1_Canonicalizer.hpp b/src/hotspot/share/c1/c1_Canonicalizer.hpp index f1c99d4996c0e..3d099a05180a0 100644 --- a/src/hotspot/share/c1/c1_Canonicalizer.hpp +++ b/src/hotspot/share/c1/c1_Canonicalizer.hpp @@ -90,7 +90,6 @@ class Canonicalizer: InstructionVisitor { virtual void do_ExceptionObject(ExceptionObject* x); virtual void do_UnsafeGet (UnsafeGet* x); virtual void do_UnsafePut (UnsafePut* x); - virtual void do_UnsafeGetAndSet(UnsafeGetAndSet* x); virtual void do_ProfileCall (ProfileCall* x); virtual void do_ProfileReturnType (ProfileReturnType* x); virtual void do_ProfileInvoke (ProfileInvoke* x); diff --git a/src/hotspot/share/c1/c1_Compiler.cpp b/src/hotspot/share/c1/c1_Compiler.cpp index d6cabf13d0c95..e0f19281c813e 100644 --- a/src/hotspot/share/c1/c1_Compiler.cpp +++ b/src/hotspot/share/c1/c1_Compiler.cpp @@ -109,21 +109,20 @@ bool Compiler::is_intrinsic_supported(const methodHandle& method) { bool Compiler::is_intrinsic_supported(vmIntrinsics::ID id) { switch (id) { - case vmIntrinsics::_compareAndSetLong: + case vmIntrinsics::_compareAndSetPrimitiveBitsMO: + case vmIntrinsics::_compareAndSetReferenceMO: + // all platforms must support at least T_OBJECT, T_INT, T_LONG break; - case vmIntrinsics::_getAndAddInt: - if (!VM_Version::supports_atomic_getadd4()) return false; - break; - case vmIntrinsics::_getAndAddLong: - if (!VM_Version::supports_atomic_getadd8()) return false; - break; - case vmIntrinsics::_getAndSetInt: - if (!VM_Version::supports_atomic_getset4()) return false; - break; - case vmIntrinsics::_getAndSetLong: - if (!VM_Version::supports_atomic_getset8()) return false; + case vmIntrinsics::_getAndOperatePrimitiveBitsMO: + if (!(VM_Version::supports_atomic_getadd4() || + VM_Version::supports_atomic_getadd8() || + VM_Version::supports_atomic_getset4() || + VM_Version::supports_atomic_getset8())) { + // if any of the hardware ops are present, try the expansion + return false; + } break; - case vmIntrinsics::_getAndSetReference: + case vmIntrinsics::_getAndSetReferenceMO: #ifdef _LP64 if (!UseCompressedOops && !VM_Version::supports_atomic_getset8()) return false; if (UseCompressedOops && !VM_Version::supports_atomic_getset4()) return false; @@ -176,50 +175,10 @@ bool Compiler::is_intrinsic_supported(vmIntrinsics::ID id) { case vmIntrinsics::_dpow: case vmIntrinsics::_fmaD: case vmIntrinsics::_fmaF: - case vmIntrinsics::_getReference: - case vmIntrinsics::_getBoolean: - case vmIntrinsics::_getByte: - case vmIntrinsics::_getShort: - case vmIntrinsics::_getChar: - case vmIntrinsics::_getInt: - case vmIntrinsics::_getLong: - case vmIntrinsics::_getFloat: - case vmIntrinsics::_getDouble: - case vmIntrinsics::_putReference: - case vmIntrinsics::_putBoolean: - case vmIntrinsics::_putByte: - case vmIntrinsics::_putShort: - case vmIntrinsics::_putChar: - case vmIntrinsics::_putInt: - case vmIntrinsics::_putLong: - case vmIntrinsics::_putFloat: - case vmIntrinsics::_putDouble: - case vmIntrinsics::_getReferenceVolatile: - case vmIntrinsics::_getBooleanVolatile: - case vmIntrinsics::_getByteVolatile: - case vmIntrinsics::_getShortVolatile: - case vmIntrinsics::_getCharVolatile: - case vmIntrinsics::_getIntVolatile: - case vmIntrinsics::_getLongVolatile: - case vmIntrinsics::_getFloatVolatile: - case vmIntrinsics::_getDoubleVolatile: - case vmIntrinsics::_putReferenceVolatile: - case vmIntrinsics::_putBooleanVolatile: - case vmIntrinsics::_putByteVolatile: - case vmIntrinsics::_putShortVolatile: - case vmIntrinsics::_putCharVolatile: - case vmIntrinsics::_putIntVolatile: - case vmIntrinsics::_putLongVolatile: - case vmIntrinsics::_putFloatVolatile: - case vmIntrinsics::_putDoubleVolatile: - case vmIntrinsics::_getShortUnaligned: - case vmIntrinsics::_getCharUnaligned: - case vmIntrinsics::_getIntUnaligned: - case vmIntrinsics::_getLongUnaligned: - case vmIntrinsics::_putShortUnaligned: - case vmIntrinsics::_putCharUnaligned: - case vmIntrinsics::_putIntUnaligned: - case vmIntrinsics::_putLongUnaligned: + case vmIntrinsics::_getPrimitiveBitsMO: + case vmIntrinsics::_putPrimitiveBitsMO: + case vmIntrinsics::_getReferenceMO: + case vmIntrinsics::_putReferenceMO: case vmIntrinsics::_Preconditions_checkIndex: case vmIntrinsics::_Preconditions_checkLongIndex: case vmIntrinsics::_updateCRC32: @@ -230,8 +189,6 @@ bool Compiler::is_intrinsic_supported(vmIntrinsics::ID id) { case vmIntrinsics::_updateDirectByteBufferCRC32C: #endif case vmIntrinsics::_vectorizedMismatch: - case vmIntrinsics::_compareAndSetInt: - case vmIntrinsics::_compareAndSetReference: case vmIntrinsics::_getCharStringU: case vmIntrinsics::_putCharStringU: #ifdef JFR_HAVE_INTRINSICS diff --git a/src/hotspot/share/c1/c1_GraphBuilder.cpp b/src/hotspot/share/c1/c1_GraphBuilder.cpp index f910ecadc164a..6203de519bab4 100644 --- a/src/hotspot/share/c1/c1_GraphBuilder.cpp +++ b/src/hotspot/share/c1/c1_GraphBuilder.cpp @@ -1258,8 +1258,34 @@ void GraphBuilder::compare_op(ValueType* type, Bytecodes::Code code) { } +#define CanonicalizeConversions (!UseNewCode3) // New code marker; FIXME. void GraphBuilder::convert(Bytecodes::Code op, BasicType from, BasicType to) { - push(as_ValueType(to), append(new Convert(op, pop(as_ValueType(from)), as_ValueType(to)))); + ValueType* xt = as_ValueType(from); + ValueType* rt = as_ValueType(to); + Value x = pop(xt); + if (CanonicalizeConversions && + x->as_Convert() != nullptr && rt->is_int()) { + Value v = nullptr; + Bytecodes::Code op2 = match_conversion(x, &v); + if (op2 == Bytecodes::_i2l || op2 == Bytecodes::_i2d) { + // Canonicalize l2i(x=i2l(v)) => v and same for d2i(x=i2d(v)) + // Back-to-back l2i/i2l pair show up frequently in unsafe wrappers. + ipush(v); return; + } + if (op2 == op || (op == Bytecodes::_i2s && op2 == Bytecodes::_i2b)) { + // Canonicalize i2b(x=i2b(v)) => x=i2b(v) and same for i2c and i2s + // Also, having got this far, do i2s(x=i2b(v)) => x=i2b(v). + ipush(x); return; + } + if ((op == Bytecodes::_i2s || op == Bytecodes::_i2c || op == Bytecodes::_i2b) + && + (op2 == Bytecodes::_i2s || op2 == Bytecodes::_i2c)) { + // Do i2b(x=i2[sc](v)) => i2b(v), i2[cs](x=i2[sc](v)) => i2[cs](v) + // This shows up naturally when a cast follows a field load. + x = v; // and fall through + } + } + push(as_ValueType(to), append(new Convert(op, x, rt))); } @@ -3211,7 +3237,7 @@ void GraphBuilder::setup_osr_entry_block() { Value off_val = append(new Constant(new IntConstant(offset))); get = append(new UnsafeGet(as_BasicType(local->type()), e, off_val, - false/*is_volatile*/, + vmIntrinsics::MO_PLAIN/*!is_volatile*/, true/*is_raw*/)); } _state->store_local(index, get); @@ -3568,59 +3594,42 @@ void GraphBuilder::build_graph_for_intrinsic(ciMethod* callee, bool ignore_retur assert(id != vmIntrinsics::_none, "must be a VM intrinsic"); // Some intrinsics need special IR nodes. + Value arg, subst; switch(id) { - case vmIntrinsics::_getReference : append_unsafe_get(callee, T_OBJECT, false); return; - case vmIntrinsics::_getBoolean : append_unsafe_get(callee, T_BOOLEAN, false); return; - case vmIntrinsics::_getByte : append_unsafe_get(callee, T_BYTE, false); return; - case vmIntrinsics::_getShort : append_unsafe_get(callee, T_SHORT, false); return; - case vmIntrinsics::_getChar : append_unsafe_get(callee, T_CHAR, false); return; - case vmIntrinsics::_getInt : append_unsafe_get(callee, T_INT, false); return; - case vmIntrinsics::_getLong : append_unsafe_get(callee, T_LONG, false); return; - case vmIntrinsics::_getFloat : append_unsafe_get(callee, T_FLOAT, false); return; - case vmIntrinsics::_getDouble : append_unsafe_get(callee, T_DOUBLE, false); return; - case vmIntrinsics::_putReference : append_unsafe_put(callee, T_OBJECT, false); return; - case vmIntrinsics::_putBoolean : append_unsafe_put(callee, T_BOOLEAN, false); return; - case vmIntrinsics::_putByte : append_unsafe_put(callee, T_BYTE, false); return; - case vmIntrinsics::_putShort : append_unsafe_put(callee, T_SHORT, false); return; - case vmIntrinsics::_putChar : append_unsafe_put(callee, T_CHAR, false); return; - case vmIntrinsics::_putInt : append_unsafe_put(callee, T_INT, false); return; - case vmIntrinsics::_putLong : append_unsafe_put(callee, T_LONG, false); return; - case vmIntrinsics::_putFloat : append_unsafe_put(callee, T_FLOAT, false); return; - case vmIntrinsics::_putDouble : append_unsafe_put(callee, T_DOUBLE, false); return; - case vmIntrinsics::_getShortUnaligned : append_unsafe_get(callee, T_SHORT, false); return; - case vmIntrinsics::_getCharUnaligned : append_unsafe_get(callee, T_CHAR, false); return; - case vmIntrinsics::_getIntUnaligned : append_unsafe_get(callee, T_INT, false); return; - case vmIntrinsics::_getLongUnaligned : append_unsafe_get(callee, T_LONG, false); return; - case vmIntrinsics::_putShortUnaligned : append_unsafe_put(callee, T_SHORT, false); return; - case vmIntrinsics::_putCharUnaligned : append_unsafe_put(callee, T_CHAR, false); return; - case vmIntrinsics::_putIntUnaligned : append_unsafe_put(callee, T_INT, false); return; - case vmIntrinsics::_putLongUnaligned : append_unsafe_put(callee, T_LONG, false); return; - case vmIntrinsics::_getReferenceVolatile : append_unsafe_get(callee, T_OBJECT, true); return; - case vmIntrinsics::_getBooleanVolatile : append_unsafe_get(callee, T_BOOLEAN, true); return; - case vmIntrinsics::_getByteVolatile : append_unsafe_get(callee, T_BYTE, true); return; - case vmIntrinsics::_getShortVolatile : append_unsafe_get(callee, T_SHORT, true); return; - case vmIntrinsics::_getCharVolatile : append_unsafe_get(callee, T_CHAR, true); return; - case vmIntrinsics::_getIntVolatile : append_unsafe_get(callee, T_INT, true); return; - case vmIntrinsics::_getLongVolatile : append_unsafe_get(callee, T_LONG, true); return; - case vmIntrinsics::_getFloatVolatile : append_unsafe_get(callee, T_FLOAT, true); return; - case vmIntrinsics::_getDoubleVolatile : append_unsafe_get(callee, T_DOUBLE, true); return; - case vmIntrinsics::_putReferenceVolatile : append_unsafe_put(callee, T_OBJECT, true); return; - case vmIntrinsics::_putBooleanVolatile : append_unsafe_put(callee, T_BOOLEAN, true); return; - case vmIntrinsics::_putByteVolatile : append_unsafe_put(callee, T_BYTE, true); return; - case vmIntrinsics::_putShortVolatile : append_unsafe_put(callee, T_SHORT, true); return; - case vmIntrinsics::_putCharVolatile : append_unsafe_put(callee, T_CHAR, true); return; - case vmIntrinsics::_putIntVolatile : append_unsafe_put(callee, T_INT, true); return; - case vmIntrinsics::_putLongVolatile : append_unsafe_put(callee, T_LONG, true); return; - case vmIntrinsics::_putFloatVolatile : append_unsafe_put(callee, T_FLOAT, true); return; - case vmIntrinsics::_putDoubleVolatile : append_unsafe_put(callee, T_DOUBLE, true); return; - case vmIntrinsics::_compareAndSetLong: - case vmIntrinsics::_compareAndSetInt: - case vmIntrinsics::_compareAndSetReference : append_unsafe_CAS(callee); return; - case vmIntrinsics::_getAndAddInt: - case vmIntrinsics::_getAndAddLong : append_unsafe_get_and_set(callee, true); return; - case vmIntrinsics::_getAndSetInt : - case vmIntrinsics::_getAndSetLong : - case vmIntrinsics::_getAndSetReference : append_unsafe_get_and_set(callee, false); return; + // canonicalize back-to-back float-vs-int, long-vs-double conversions + case vmIntrinsics::_floatToRawIntBits: + arg = fpop(); + if (CanonicalizeConversions && + vmIntrinsics::_intBitsToFloat == match_unary_intrinsic(arg, &subst)) { + ipush(subst); return; // eliminated a back-to-back pair + } else { + fpush(arg); break; // no such luck; generate the intrinsic + } + case vmIntrinsics::_intBitsToFloat: + arg = ipop(); + if (CanonicalizeConversions && + vmIntrinsics::_floatToRawIntBits == match_unary_intrinsic(arg, &subst)) { + fpush(subst); return; // eliminated a back-to-back pair + } else { + ipush(arg); break; // no such luck; generate the intrinsic + } + case vmIntrinsics::_doubleToRawLongBits: + arg = dpop(); + if (CanonicalizeConversions && + vmIntrinsics::_longBitsToDouble == match_unary_intrinsic(arg, &subst)) { + lpush(subst); return; // eliminated a back-to-back pair + } else { + dpush(arg); break; // no such luck; generate the intrinsic + } + case vmIntrinsics::_longBitsToDouble: + arg = lpop(); + if (CanonicalizeConversions && + vmIntrinsics::_doubleToRawLongBits == match_unary_intrinsic(arg, &subst)) { + dpush(subst); return; // eliminated a back-to-back pair + } else { + lpush(arg); break; // no such luck; generate the intrinsic + } + case vmIntrinsics::_getCharStringU : append_char_access(callee, false); return; case vmIntrinsics::_putCharStringU : append_char_access(callee, true); return; case vmIntrinsics::_clone : append_alloc_array_copy(callee); return; @@ -3655,7 +3664,7 @@ void GraphBuilder::build_graph_for_intrinsic(ciMethod* callee, bool ignore_retur } } - Intrinsic* result = new Intrinsic(result_type, callee->intrinsic_id(), + Intrinsic* result = new Intrinsic(result_type, id, args, has_receiver, state_before, vmIntrinsics::preserves_state(id), vmIntrinsics::can_trap(id)); @@ -3670,6 +3679,115 @@ void GraphBuilder::build_graph_for_intrinsic(ciMethod* callee, bool ignore_retur } } + +bool GraphBuilder::try_inline_polymorphic_intrinsic(ciMethod* callee, bool ignore_return) { + vmIntrinsics::ID id = callee->intrinsic_id(); + vmIntrinsics::PolymorphicPrefix pfx = vmIntrinsics::polymorphic_prefix(id); + assert(pfx != vmIntrinsics::PP_NONE, "must have a polymorphic prefix"); + const int args_base = state()->stack_size() - callee->arg_size(); + int prefix_end = args_base; // incremented over the prefix + assert(!callee->is_static(), ""); + ++prefix_end; // skip over Unsafe receiver object + jint mo = -1; // mo should be MO_PLAIN or the like; use MO_VOLATILE if not constant + jint bt = -1; // bt should be T_BYTE or the like; bail if not constant + jint op = -1; // op should be OP_SWAP or the like; bail if not constant + { + IntConstant* intcon; + switch (pfx) { + case vmIntrinsics::PP_MO: + intcon = state()->stack_at_inc(prefix_end)->type()->as_IntConstant(); + if (intcon != nullptr) mo = intcon->value() & 0xFF; + bt = T_OBJECT; // getReferenceMO etc. + break; + case vmIntrinsics::PP_MO_BT: + intcon = state()->stack_at_inc(prefix_end)->type()->as_IntConstant(); + if (intcon != nullptr) mo = intcon->value() & 0xFF; + intcon = state()->stack_at_inc(prefix_end)->type()->as_IntConstant(); + if (intcon != nullptr) bt = intcon->value() & 0xFF; + if (!vmIntrinsics::is_valid_primitive_type(bt)) return false; + break; + case vmIntrinsics::PP_MO_BT_OP: + intcon = state()->stack_at_inc(prefix_end)->type()->as_IntConstant(); + if (intcon != nullptr) mo = intcon->value() & 0xFF; + intcon = state()->stack_at_inc(prefix_end)->type()->as_IntConstant(); + if (intcon != nullptr) bt = intcon->value() & 0xFF; + intcon = state()->stack_at_inc(prefix_end)->type()->as_IntConstant(); + if (intcon != nullptr) op = intcon->value() & 0xFF; + if (!vmIntrinsics::is_valid_primitive_type(bt)) return false; + if (!vmIntrinsics::is_valid_primitive_bits_op(op)) return false; + break; + default: + ShouldNotReachHere(); + } + } + // after skipping prefix, remember the number of skipped items + const int prefix_size = prefix_end - args_base - 1; //do not count receiver + // if MO is unknown, fall down to MO_VOLATILE + if (!vmIntrinsics::is_valid_memory_order(mo)) mo = vmIntrinsics::MO_VOLATILE; + BasicType t = (BasicType)bt; + assert(bt >= 0 && is_java_type(t), ""); + if (op < 0) op = vmIntrinsics::OP_NONE; + switch(id) { + case vmIntrinsics::_getReferenceMO: + assert(t == T_OBJECT, ""); // and fall through + case vmIntrinsics::_getPrimitiveBitsMO: + append_unsafe_get(callee, (vmIntrinsics::MemoryOrder)mo, t, prefix_size); break; + + case vmIntrinsics::_putReferenceMO: + assert(t == T_OBJECT, ""); // and fall through + case vmIntrinsics::_putPrimitiveBitsMO: + append_unsafe_put(callee, (vmIntrinsics::MemoryOrder)mo, t, prefix_size); break; + + case vmIntrinsics::_compareAndSetReferenceMO: + assert(t == T_OBJECT, ""); // and fall through + case vmIntrinsics::_compareAndSetPrimitiveBitsMO: + if (t == T_INT || t == T_LONG || t == T_OBJECT) { + append_unsafe_CAS(callee, (vmIntrinsics::MemoryOrder)mo, t, + vmIntrinsics::OP_NONE, prefix_size); + return true; + } + // FIXME: detect other combinations supported by platform + return false; + + case vmIntrinsics::_getAndSetReferenceMO: + assert(t == T_OBJECT, ""); + assert(op == vmIntrinsics::OP_NONE, ""); + op = vmIntrinsics::OP_SWAP; + // and fall through + case vmIntrinsics::_getAndOperatePrimitiveBitsMO: + switch (op) { + case vmIntrinsics::OP_ADD: + if (t == T_INT && !VM_Version::supports_atomic_getadd4()) return false; + if (t == T_LONG && !VM_Version::supports_atomic_getadd8()) return false; + break; + case vmIntrinsics::OP_SWAP: + if (t == T_INT && !VM_Version::supports_atomic_getset4()) return false; + if (t == T_LONG && !VM_Version::supports_atomic_getset8()) return false; + break; + default: + return false; + } + // FIXME: Most platforms (including arm64 and x64) support byte + // and short as well, and with all the bitwise combination ops. + if (t == T_INT || t == T_LONG || t == T_OBJECT) { + append_unsafe_CAS(callee, (vmIntrinsics::MemoryOrder)mo, t, + (vmIntrinsics::BitsOperation)op, prefix_size); + return true; + } + return false; // no other combinations supported + + case vmIntrinsics::_compareAndExchangeReferenceMO: + case vmIntrinsics::_compareAndExchangePrimitiveBitsMO: + // FIXME: Most platforms support full cmpxchg in all sizes. + return false; + + default: + return false; + } + return true; +} + + bool GraphBuilder::try_inline_intrinsics(ciMethod* callee, bool ignore_return) { // For calling is_intrinsic_available we need to transition to // the '_thread_in_vm' state because is_intrinsic_available() @@ -3685,11 +3803,19 @@ bool GraphBuilder::try_inline_intrinsics(ciMethod* callee, bool ignore_return) { if (!InlineNatives) { // Return false and also set message that the inlining of // intrinsics has been disabled in general. + // FIXME: It is not clear that InlineNatives is the correct switch for this. + // Some intrinsics are regular non-native methods, yet they get blocked here. INLINE_BAILOUT("intrinsic method inlining disabled"); } else { + // Return false but allow for normal inlining return false; } } + + if (vmIntrinsics::polymorphic_prefix(callee->intrinsic_id()) != vmIntrinsics::PP_NONE) { + return try_inline_polymorphic_intrinsic(callee, ignore_return); + } + build_graph_for_intrinsic(callee, ignore_return); if (_inline_bailout_msg != nullptr) { return false; @@ -3893,7 +4019,10 @@ bool GraphBuilder::try_inline_full(ciMethod* callee, bool holder_known, bool ign // now perform tests that are based on flag settings bool inlinee_by_directive = compilation()->directive()->should_inline(callee); if (callee->force_inline() || inlinee_by_directive) { - if (inline_level() > MaxForceInlineLevel ) INLINE_BAILOUT("MaxForceInlineLevel"); + // Intrinsics do not by themselves cause deep inlining, and they should be + // allowed to inline fully even if calling code is getting close to the limit. + bool check_level = (callee->intrinsic_id() == vmIntrinsics::_none); + if (check_level && inline_level() > MaxForceInlineLevel ) INLINE_BAILOUT("MaxForceInlineLevel"); if (recursive_inline_level(callee) > C1MaxRecursiveInlineLevel) INLINE_BAILOUT("recursive inlining too deep"); const char* msg = ""; @@ -4319,72 +4448,216 @@ void GraphBuilder::pop_scope_for_jsr() { _scope_data = scope_data()->parent(); } -void GraphBuilder::append_unsafe_get(ciMethod* callee, BasicType t, bool is_volatile) { +void GraphBuilder::append_unsafe_get(ciMethod* callee, + vmIntrinsics::MemoryOrder mo, BasicType t, + int prefix_size) { Values* args = state()->pop_arguments(callee->arg_size()); null_check(args->at(0)); - Instruction* offset = args->at(2); -#ifndef _LP64 - offset = append(new Convert(Bytecodes::_l2i, offset, as_ValueType(T_INT))); -#endif - Instruction* op = append(new UnsafeGet(t, args->at(1), offset, is_volatile)); + Value base = args->at(prefix_size + 1); + Value offset = args->at(prefix_size + 2); + offset = adjust_unsafe_offset(offset); + Instruction* op = append(new UnsafeGet(t, base, offset, mo)); + // The underlying get operation is strongly typed, but the result + // must be widened to 64 bits. + op = adjust_unsafe_container(op, t, T_LONG); push(op->type(), op); compilation()->set_has_unsafe_access(true); } -void GraphBuilder::append_unsafe_put(ciMethod* callee, BasicType t, bool is_volatile) { +void GraphBuilder::append_unsafe_put(ciMethod* callee, + vmIntrinsics::MemoryOrder mo, BasicType t, + int prefix_size) { Values* args = state()->pop_arguments(callee->arg_size()); null_check(args->at(0)); - Instruction* offset = args->at(2); -#ifndef _LP64 - offset = append(new Convert(Bytecodes::_l2i, offset, as_ValueType(T_INT))); -#endif - Value val = args->at(3); + Value base = args->at(prefix_size + 1); + Value offset = args->at(prefix_size + 2); + Value val = args->at(prefix_size + 3); + offset = adjust_unsafe_offset(offset); + val = adjust_unsafe_container(val, T_LONG, t); if (t == T_BOOLEAN) { Value mask = append(new Constant(new IntConstant(1))); val = append(new LogicOp(Bytecodes::_iand, val, mask)); } - Instruction* op = append(new UnsafePut(t, args->at(1), offset, val, is_volatile)); + Instruction* op = append(new UnsafePut(t, base, offset, val, mo)); compilation()->set_has_unsafe_access(true); kill_all(); } -void GraphBuilder::append_unsafe_CAS(ciMethod* callee) { +// Append compareAndSet (OP_NONE), getAndSet (OP_SWAP), or setAndFoo (OP_Foo). +// The backend will inspect the intrinsic and prefix to DTRT. +void GraphBuilder::append_unsafe_CAS(ciMethod* callee, + vmIntrinsics::MemoryOrder mo, BasicType t, + vmIntrinsics::BitsOperation op, + int prefix_size) { ValueStack* state_before = copy_state_for_exception(); - ValueType* result_type = as_ValueType(callee->return_type()); - assert(result_type->is_int(), "int result"); + vmIntrinsics::ID id = callee->intrinsic_id(); + // The uit (unsafe type) is long for primitive GAX, else T_OBJECT or T_BOOLEAN + BasicType uit = callee->return_type()->basic_type(); + const bool is_plain_CAS = (op == vmIntrinsics::OP_NONE); // CAS vs. GAS/GAX + ValueType* result_type = as_ValueType(is_plain_CAS ? T_BOOLEAN : t); + assert(is_plain_CAS + ? uit == T_BOOLEAN + : uit == T_OBJECT || uit == T_LONG, "correct polymorphic method return"); Values* args = state()->pop_arguments(callee->arg_size()); // Pop off some args to specially handle, then push back - Value newval = args->pop(); - Value cmpval = args->pop(); + Value newval = is_plain_CAS ? args->pop() : nullptr; + Value cmpval = args->pop(); // also, operand for GAS/GAX Value offset = args->pop(); - Value src = args->pop(); - Value unsafe_obj = args->pop(); + Value base = args->pop(); + assert(args->length() == 1+prefix_size, ""); // do not pop the prefix + Value unsafe_obj = args->at(0); + args->clear(); // discard unsafe obj and MO_BT prefix // Separately handle the unsafe arg. It is not needed for code // generation, but must be null checked null_check(unsafe_obj); -#ifndef _LP64 - offset = append(new Convert(Bytecodes::_l2i, offset, as_ValueType(T_INT))); -#endif + offset = adjust_unsafe_offset(offset); + cmpval = adjust_unsafe_container(cmpval, T_LONG, t); + if (is_plain_CAS) + newval = adjust_unsafe_container(newval, T_LONG, t); - args->push(src); + args->push(base); args->push(offset); - args->push(cmpval); - args->push(newval); + args->push(cmpval); // or operand for GAX, or newval for GAS + if (is_plain_CAS) + args->push(newval); // GAS/GAX will compute the newval // An unsafe CAS can alias with other field accesses, but we don't // know which ones so mark the state as no preserved. This will // cause CSE to invalidate memory across it. - bool preserves_state = false; - Intrinsic* result = new Intrinsic(result_type, callee->intrinsic_id(), args, false, state_before, preserves_state); - append_split(result); - push(result_type, result); + const bool not_ps = false; // preserve_state = false + Intrinsic* result = new Intrinsic(result_type, id, args, false, state_before, not_ps); + append_split(result->with_polymorphic_prefix(mo, t, op)); + switch (uit) { + case T_BOOLEAN: + assert(is_plain_CAS, "plain CAS returns boolean"); + assert(id == vmIntrinsics::_compareAndSetReferenceMO || + id == vmIntrinsics::_compareAndSetPrimitiveBitsMO, ""); + ipush(result); break; + case T_OBJECT: + assert(!is_plain_CAS && op == vmIntrinsics::OP_NONE, "swap returns reference"); + assert(id == vmIntrinsics::_getAndSetReferenceMO || + id == vmIntrinsics::_getAndOperatePrimitiveBitsMO, ""); + apush(result); break; + default: + assert(!is_plain_CAS && uit == T_LONG, "returns primitive bits"); + assert(id == vmIntrinsics::_getAndOperatePrimitiveBitsMO, ""); + // The underlying intrinsic is strongly typed, but the result must + // be widened to 64 bits. + lpush(adjust_unsafe_container(result, t, T_LONG)); break; + } compilation()->set_has_unsafe_access(true); } +// Support for shifting a container type, without changing significant +// bits. It is used by "primitive-bits" methods in the Unsafe API. +// +// If the container is widened, there are no guarantees about what +// high bits are used for padding, as per the Unsafe API javadoc. +// To canonicalize bits, you might use something like _iand or _i2s. +// +// The invariant is that the representational bits of the smaller type +// are preserved exactly. We might execute _i2l or _l2i but never _i2f. +// If we need to deal with floats, we use the bit-conversion intrinsics. +// +// In subwords, there are no guarantees about the value of +// insignificant padding bits -- they do not need to be sign bits (as +// for byte) or zero bits (as for char). Not even _i2b is necessary; +// it would not break the invariant but it would be useless. +// +// As a special case, if either type is T_OBJECT, this is a nop, +// because object pointers are all the same format. The user +// might has asked for T_LONG to be converted to T_OBJECT, +// but such a nonsense request will be quietly refused. +// This makes it easier for common code to handle both +// references and primitives. +Value GraphBuilder::adjust_unsafe_container(Value x, BasicType from, BasicType to) { + if (from == T_OBJECT || to == T_OBJECT) { + // T_OBJECT does not mix with anything else; just return it + assert(x->type()->is_object(), ""); + return x; + } + assert(x->type()->tag() == as_ValueType(from)->tag(), ""); + if (from == to) return x; // no action + assert(is_java_primitive(from) && is_java_primitive(to), ""); + if (is_double_word_type(from) != is_double_word_type(to)) { + // size changes always go through T_INT + if (from == T_INT) { + assert(to == T_LONG, "T_DOUBLE is not a container"); + return append(new Convert(Bytecodes::_i2l, x, as_ValueType(T_LONG))); + } else if (to == T_INT) { + assert(from == T_LONG, "T_DOUBLE is not a container"); + Value v = nullptr; + if (CanonicalizeConversions && + match_conversion(x, &v) == Bytecodes::_i2l) { + return v; // back-to-back i2l->l2i means no change + } + return append(new Convert(Bytecodes::_l2i, x, as_ValueType(T_INT))); + } else { + // use one or two bitwise conversions, one of which is int-vs-long + Value y = adjust_unsafe_container(x, from, T_INT); + Value z = adjust_unsafe_container(y, T_INT, to); + return z; + } + } + if (is_floating_point_type(from) || is_floating_point_type(to)) { + vmIntrinsics::ID id = vmIntrinsics::raw_floating_conversion(from, to); + assert(id != vmIntrinsics::_none, "must be"); + // bitwise same-size raw conversion: int-vs-float or long-vs-double + Value subst = nullptr; + if (CanonicalizeConversions && + match_unary_intrinsic(x, &subst) == vmIntrinsics::raw_floating_conversion(to, from)) { + return subst; // back-to-back inverse means no change + } + const bool ps = true, ct = false; + assert(vmIntrinsics::preserves_state(id) == ps && + vmIntrinsics::can_trap(id) == ct, "must be"); + Values* args = new Values(1); + args->push(x); + return append_split(new Intrinsic(as_ValueType(to), id, args, false, state(), ps, ct)); + } + // nothing left to do, since we do not care about subword padding + assert(!is_double_word_type(from) && !is_double_word_type(to), ""); + assert(from == T_INT || as_ValueType(from)->is_int(), ""); + assert(to == T_INT || as_ValueType(to)->is_int(), ""); + return x; +} + + +Bytecodes::Code GraphBuilder::match_conversion(Value x, Value* arg) { + assert(CanonicalizeNodes, "else do not call"); + Convert* conv = x->as_Convert(); + if (conv != nullptr) { + *arg = conv->value(); + return conv->op(); + } else { + return Bytecodes::_illegal; + } +} + + +vmIntrinsics::ID GraphBuilder::match_unary_intrinsic(Value x, Value* arg) { + assert(CanonicalizeNodes, "else do not call"); + Intrinsic* intr = x->as_Intrinsic(); + if (intr != nullptr && intr->number_of_arguments() == 1) { + *arg = intr->argument_at(0); + return intr->id(); + } else { + return vmIntrinsics::_none; + } +} + + +Value GraphBuilder::adjust_unsafe_offset(Value offset) { + // cut down the offset to machine word size + BasicType T_INTX = LP64_ONLY(T_LONG) NOT_LP64(T_INT); + return adjust_unsafe_container(offset, T_LONG, T_INTX); +} + + void GraphBuilder::append_char_access(ciMethod* callee, bool is_store) { // This intrinsic accesses byte[] array as char[] array. Computing the offsets // correctly requires matched array shapes. @@ -4474,20 +4747,6 @@ void GraphBuilder::print_inlining(ciMethod* callee, const char* msg, bool succes } } -void GraphBuilder::append_unsafe_get_and_set(ciMethod* callee, bool is_add) { - Values* args = state()->pop_arguments(callee->arg_size()); - BasicType t = callee->return_type()->basic_type(); - null_check(args->at(0)); - Instruction* offset = args->at(2); -#ifndef _LP64 - offset = append(new Convert(Bytecodes::_l2i, offset, as_ValueType(T_INT))); -#endif - Instruction* op = append(new UnsafeGetAndSet(t, args->at(1), offset, args->at(3), is_add)); - compilation()->set_has_unsafe_access(true); - kill_all(); - push(op->type(), op); -} - #ifndef PRODUCT void GraphBuilder::print_stats() { if (UseLocalValueNumbering) { diff --git a/src/hotspot/share/c1/c1_GraphBuilder.hpp b/src/hotspot/share/c1/c1_GraphBuilder.hpp index 6eaeda5adcf94..ffaf7e1460ccd 100644 --- a/src/hotspot/share/c1/c1_GraphBuilder.hpp +++ b/src/hotspot/share/c1/c1_GraphBuilder.hpp @@ -351,6 +351,7 @@ class GraphBuilder { // inliners bool try_inline( ciMethod* callee, bool holder_known, bool ignore_return, Bytecodes::Code bc = Bytecodes::_illegal, Value receiver = nullptr); bool try_inline_intrinsics(ciMethod* callee, bool ignore_return = false); + bool try_inline_polymorphic_intrinsic(ciMethod* callee, bool ignore_return = false); bool try_inline_full( ciMethod* callee, bool holder_known, bool ignore_return, Bytecodes::Code bc = Bytecodes::_illegal, Value receiver = nullptr); bool try_inline_jsr(int jsr_dest_bci); @@ -373,10 +374,15 @@ class GraphBuilder { void pop_scope(); void pop_scope_for_jsr(); - void append_unsafe_get(ciMethod* callee, BasicType t, bool is_volatile); - void append_unsafe_put(ciMethod* callee, BasicType t, bool is_volatile); - void append_unsafe_CAS(ciMethod* callee); - void append_unsafe_get_and_set(ciMethod* callee, bool is_add); + void append_unsafe_get(ciMethod* callee, vmIntrinsics::MemoryOrder mo, BasicType t, int prefix_size); + void append_unsafe_put(ciMethod* callee, vmIntrinsics::MemoryOrder mo, BasicType t, int prefix_size); + void append_unsafe_CAS(ciMethod* callee, vmIntrinsics::MemoryOrder mo, BasicType t, + vmIntrinsics::BitsOperation op, int prefix_size); + Value adjust_unsafe_container(Value x, BasicType from, BasicType to); + Value adjust_unsafe_offset(Value offset); + Bytecodes::Code match_conversion( Value x, Value* arg); + vmIntrinsics::ID match_unary_intrinsic(Value x, Value* arg); + void append_char_access(ciMethod* callee, bool is_store); void append_alloc_array_copy(ciMethod* callee); diff --git a/src/hotspot/share/c1/c1_Instruction.cpp b/src/hotspot/share/c1/c1_Instruction.cpp index 3a7edef0088ec..aa75786141834 100644 --- a/src/hotspot/share/c1/c1_Instruction.cpp +++ b/src/hotspot/share/c1/c1_Instruction.cpp @@ -543,6 +543,42 @@ void BlockBegin::set_end(BlockEnd* new_end) { // Assumes that no predecessor of } +Intrinsic* +Intrinsic::with_polymorphic_prefix(vmIntrinsics::MemoryOrder memory_order, + BasicType basic_type, + vmIntrinsics::BitsOperation bits_op) { + vmIntrinsics::PolymorphicPrefix pfx = vmIntrinsics::polymorphic_prefix(id()); + switch (pfx) { + case vmIntrinsics::PP_MO: + assert(memory_order != vmIntrinsics::MO_NONE, ""); + assert(basic_type == T_OBJECT || basic_type == T_ILLEGAL, ""); + assert(bits_op == vmIntrinsics::OP_NONE, ""); + break; + case vmIntrinsics::PP_MO_BT: + assert(memory_order != vmIntrinsics::MO_NONE, ""); + assert(is_java_primitive(basic_type), ""); + assert(bits_op == vmIntrinsics::OP_NONE, ""); + break; + case vmIntrinsics::PP_MO_BT_OP: + assert(memory_order != vmIntrinsics::MO_NONE, ""); + assert(is_java_primitive(basic_type), ""); + assert(bits_op != vmIntrinsics::OP_NONE, ""); + break; + default: + ShouldNotReachHere(); + } + // make sure previous settings are empty + assert(_basic_type == T_ILLEGAL, ""); + assert(_memory_order == vmIntrinsics::MO_NONE, ""); + assert(_bits_op == vmIntrinsics::OP_NONE, ""); + // install the settings; some might still be empty + _memory_order = memory_order; + _basic_type = basic_type; + _bits_op = bits_op; + return this; +} + + void BlockBegin::disconnect_edge(BlockBegin* from, BlockBegin* to) { // disconnect any edges between from and to #ifndef PRODUCT diff --git a/src/hotspot/share/c1/c1_Instruction.hpp b/src/hotspot/share/c1/c1_Instruction.hpp index 6b5838f60621d..383595135c586 100644 --- a/src/hotspot/share/c1/c1_Instruction.hpp +++ b/src/hotspot/share/c1/c1_Instruction.hpp @@ -94,7 +94,6 @@ class Base; class UnsafeOp; class UnsafeGet; class UnsafePut; -class UnsafeGetAndSet; class ProfileCall; class ProfileReturnType; class ProfileInvoke; @@ -188,7 +187,6 @@ class InstructionVisitor: public StackObj { virtual void do_ExceptionObject(ExceptionObject* x) = 0; virtual void do_UnsafeGet (UnsafeGet* x) = 0; virtual void do_UnsafePut (UnsafePut* x) = 0; - virtual void do_UnsafeGetAndSet(UnsafeGetAndSet* x) = 0; virtual void do_ProfileCall (ProfileCall* x) = 0; virtual void do_ProfileReturnType (ProfileReturnType* x) = 0; virtual void do_ProfileInvoke (ProfileInvoke* x) = 0; @@ -1501,7 +1499,6 @@ LEAF(MonitorExit, AccessMonitor) } }; - LEAF(Intrinsic, StateSplit) private: vmIntrinsics::ID _id; @@ -1509,6 +1506,11 @@ LEAF(Intrinsic, StateSplit) Values* _args; Value _recv; + // polymorphic prefix arguments, if applicable, set neutrally if not: + vmIntrinsics::MemoryOrder _memory_order; + BasicType _basic_type; + vmIntrinsics::BitsOperation _bits_op; + public: // preserves_state can be set to true for Intrinsics // which are guaranteed to preserve register state across any slow @@ -1528,6 +1530,9 @@ LEAF(Intrinsic, StateSplit) , _id(id) , _args(args) , _recv(nullptr) + , _memory_order(vmIntrinsics::MO_NONE) + , _basic_type(T_ILLEGAL) + , _bits_op(vmIntrinsics::OP_NONE) { assert(args != nullptr, "args must exist"); ASSERT_VALUES @@ -1567,6 +1572,25 @@ LEAF(Intrinsic, StateSplit) StateSplit::input_values_do(f); for (int i = 0; i < _args->length(); i++) f->visit(_args->adr_at(i)); } + + // prefix arguments + vmIntrinsics::MemoryOrder memory_order() const { + assert(_memory_order != vmIntrinsics::MO_NONE, "must be present"); + return _memory_order; + } + BasicType basic_type() const { + assert(_basic_type != T_ILLEGAL, "must be present"); + return _basic_type; + } + vmIntrinsics::BitsOperation bits_op() const { + assert(_bits_op != vmIntrinsics::OP_NONE, "must be present"); + return _bits_op; + } + // one-time setup after creation, required for polymorphic intrinsics + Intrinsic* with_polymorphic_prefix( + vmIntrinsics::MemoryOrder memory_order, + BasicType basic_type = T_OBJECT, + vmIntrinsics::BitsOperation bits_op = vmIntrinsics::OP_NONE); }; @@ -2139,14 +2163,15 @@ BASE(UnsafeOp, Instruction) private: Value _object; // Object to be fetched from or mutated Value _offset; // Offset within object - bool _is_volatile; // true if volatile - dl/JSR166 + vmIntrinsics::MemoryOrder _memory_order; // MO_PLAIN, MO_VOLATILE - dl/JSR166 BasicType _basic_type; // ValueType can not express byte-sized integers protected: // creation - UnsafeOp(BasicType basic_type, Value object, Value offset, bool is_put, bool is_volatile) + UnsafeOp(BasicType basic_type, Value object, Value offset, bool is_put, + vmIntrinsics::MemoryOrder memory_order) : Instruction(is_put ? voidType : as_ValueType(basic_type)), - _object(object), _offset(offset), _is_volatile(is_volatile), _basic_type(basic_type) + _object(object), _offset(offset), _memory_order(memory_order), _basic_type(basic_type) { //Note: Unsafe ops are not not guaranteed to throw NPE. // Convservatively, Unsafe operations must be pinned though we could be @@ -2159,7 +2184,7 @@ BASE(UnsafeOp, Instruction) BasicType basic_type() { return _basic_type; } Value object() { return _object; } Value offset() { return _offset; } - bool is_volatile() { return _is_volatile; } + vmIntrinsics::MemoryOrder memory_order() { return _memory_order; } // generic virtual void input_values_do(ValueVisitor* f) { f->visit(&_object); @@ -2170,14 +2195,14 @@ LEAF(UnsafeGet, UnsafeOp) private: bool _is_raw; public: - UnsafeGet(BasicType basic_type, Value object, Value offset, bool is_volatile) - : UnsafeOp(basic_type, object, offset, false, is_volatile) + UnsafeGet(BasicType basic_type, Value object, Value offset, vmIntrinsics::MemoryOrder memory_order) + : UnsafeOp(basic_type, object, offset, false, memory_order) { ASSERT_VALUES _is_raw = false; } - UnsafeGet(BasicType basic_type, Value object, Value offset, bool is_volatile, bool is_raw) - : UnsafeOp(basic_type, object, offset, false, is_volatile), _is_raw(is_raw) + UnsafeGet(BasicType basic_type, Value object, Value offset, vmIntrinsics::MemoryOrder memory_order, bool is_raw) + : UnsafeOp(basic_type, object, offset, false, memory_order), _is_raw(is_raw) { ASSERT_VALUES } @@ -2191,36 +2216,14 @@ LEAF(UnsafePut, UnsafeOp) private: Value _value; // Value to be stored public: - UnsafePut(BasicType basic_type, Value object, Value offset, Value value, bool is_volatile) - : UnsafeOp(basic_type, object, offset, true, is_volatile) - , _value(value) - { - ASSERT_VALUES - } - - // accessors - Value value() { return _value; } - - // generic - virtual void input_values_do(ValueVisitor* f) { UnsafeOp::input_values_do(f); - f->visit(&_value); } -}; - -LEAF(UnsafeGetAndSet, UnsafeOp) - private: - Value _value; // Value to be stored - bool _is_add; - public: - UnsafeGetAndSet(BasicType basic_type, Value object, Value offset, Value value, bool is_add) - : UnsafeOp(basic_type, object, offset, false, false) + UnsafePut(BasicType basic_type, Value object, Value offset, Value value, vmIntrinsics::MemoryOrder memory_order) + : UnsafeOp(basic_type, object, offset, true, memory_order) , _value(value) - , _is_add(is_add) { ASSERT_VALUES } // accessors - bool is_add() const { return _is_add; } Value value() { return _value; } // generic diff --git a/src/hotspot/share/c1/c1_InstructionPrinter.cpp b/src/hotspot/share/c1/c1_InstructionPrinter.cpp index 59e24c3e6c52d..fe53ffa5f6cda 100644 --- a/src/hotspot/share/c1/c1_InstructionPrinter.cpp +++ b/src/hotspot/share/c1/c1_InstructionPrinter.cpp @@ -791,13 +791,6 @@ void InstructionPrinter::do_UnsafePut(UnsafePut* x) { output()->put(')'); } -void InstructionPrinter::do_UnsafeGetAndSet(UnsafeGetAndSet* x) { - print_unsafe_op(x, x->is_add()?"UnsafeGetAndSet (add)":"UnsafeGetAndSet"); - output()->print(", value "); - print_value(x->value()); - output()->put(')'); -} - void InstructionPrinter::do_RangeCheckPredicate(RangeCheckPredicate* x) { if (x->x() != nullptr && x->y() != nullptr) { diff --git a/src/hotspot/share/c1/c1_InstructionPrinter.hpp b/src/hotspot/share/c1/c1_InstructionPrinter.hpp index b7bac56132776..ebba82d9afc2b 100644 --- a/src/hotspot/share/c1/c1_InstructionPrinter.hpp +++ b/src/hotspot/share/c1/c1_InstructionPrinter.hpp @@ -122,7 +122,6 @@ class InstructionPrinter: public InstructionVisitor { virtual void do_ExceptionObject(ExceptionObject* x); virtual void do_UnsafeGet (UnsafeGet* x); virtual void do_UnsafePut (UnsafePut* x); - virtual void do_UnsafeGetAndSet(UnsafeGetAndSet* x); virtual void do_ProfileCall (ProfileCall* x); virtual void do_ProfileReturnType (ProfileReturnType* x); virtual void do_ProfileInvoke (ProfileInvoke* x); diff --git a/src/hotspot/share/c1/c1_LIRGenerator.cpp b/src/hotspot/share/c1/c1_LIRGenerator.cpp index f6807abcd7acd..f4792b228e085 100644 --- a/src/hotspot/share/c1/c1_LIRGenerator.cpp +++ b/src/hotspot/share/c1/c1_LIRGenerator.cpp @@ -1492,21 +1492,58 @@ LIR_Opr LIRGenerator::load_constant(LIR_Const* c) { //------------------------field access-------------------------------------- -void LIRGenerator::do_CompareAndSwap(Intrinsic* x, ValueType* type) { +void LIRGenerator::do_CompareAndSwap(Intrinsic* x) { + BasicType type = x->basic_type(); + // ignore memory order -- just do MO_VOLATILE assert(x->number_of_arguments() == 4, "wrong type"); LIRItem obj (x->argument_at(0), this); // object LIRItem offset(x->argument_at(1), this); // offset of field LIRItem cmp (x->argument_at(2), this); // value to compare with field LIRItem val (x->argument_at(3), this); // replace field with val if matches cmp assert(obj.type()->tag() == objectTag, "invalid type"); - assert(cmp.type()->tag() == type->tag(), "invalid type"); - assert(val.type()->tag() == type->tag(), "invalid type"); + ValueType* vt = as_ValueType(type); + assert(cmp.type()->tag() == vt->tag(), "invalid type"); + assert(val.type()->tag() == vt->tag(), "invalid type"); - LIR_Opr result = access_atomic_cmpxchg_at(IN_HEAP, as_BasicType(type), + LIR_Opr result = access_atomic_cmpxchg_at(IN_HEAP, type, obj, offset, cmp, val); set_result(x, result); } +void LIRGenerator::do_GetAndOperate(Intrinsic* x) { + BasicType type = x->basic_type(); + // ignore memory order -- just do MO_VOLATILE + vmIntrinsics::BitsOperation op = x->bits_op(); + assert(x->number_of_arguments() == 3, "wrong type"); + LIRItem obj (x->argument_at(0), this); // object + LIRItem off (x->argument_at(1), this); // offset of field + LIRItem value(x->argument_at(2), this); // operand or swap value + + // fall down to MO_VOLATILE; maybe handle MO_RELEASE, etc., later? + DecoratorSet decorators = IN_HEAP | C1_UNSAFE_ACCESS | MO_SEQ_CST; + + if (is_reference_type(type)) { + decorators |= ON_UNKNOWN_OOP_REF; + } + + assert(obj.type()->tag() == objectTag, "invalid type"); + ValueType* vt = as_ValueType(type); + assert(value.type()->tag() == vt->tag(), "invalid type"); + + LIR_Opr result; + switch (x->bits_op()) { + case vmIntrinsics::OP_ADD: + result = access_atomic_add_at(decorators, type, obj, off, value); + break; + default: + assert(false, "either add or swap, please"); + case vmIntrinsics::OP_SWAP: + result = access_atomic_xchg_at(decorators, type, obj, off, value); + break; + } + set_result(x, result); +} + // Comment copied form templateTable_i486.cpp // ---------------------------------------------------------------------------- // Volatile variables demand their effects be made known to all CPU's in @@ -2040,7 +2077,8 @@ void LIRGenerator::do_UnsafeGet(UnsafeGet* x) { DecoratorSet decorators = IN_HEAP | C1_UNSAFE_ACCESS; - if (x->is_volatile()) { + if (x->memory_order() != vmIntrinsics::MO_PLAIN) { + // fall down to MO_VOLATILE; maybe handle MO_ACQUIRE later? decorators |= MO_SEQ_CST; } if (type == T_BOOLEAN) { @@ -2092,33 +2130,13 @@ void LIRGenerator::do_UnsafePut(UnsafePut* x) { if (is_reference_type(type)) { decorators |= ON_UNKNOWN_OOP_REF; } - if (x->is_volatile()) { + if (x->memory_order() != vmIntrinsics::MO_PLAIN) { + // fall down to MO_VOLATILE; maybe handle MO_RELEASE later? decorators |= MO_SEQ_CST; } access_store_at(decorators, type, src, off.result(), data.result()); } -void LIRGenerator::do_UnsafeGetAndSet(UnsafeGetAndSet* x) { - BasicType type = x->basic_type(); - LIRItem src(x->object(), this); - LIRItem off(x->offset(), this); - LIRItem value(x->value(), this); - - DecoratorSet decorators = IN_HEAP | C1_UNSAFE_ACCESS | MO_SEQ_CST; - - if (is_reference_type(type)) { - decorators |= ON_UNKNOWN_OOP_REF; - } - - LIR_Opr result; - if (x->is_add()) { - result = access_atomic_add_at(decorators, type, src, off, value); - } else { - result = access_atomic_xchg_at(decorators, type, src, off, value); - } - set_result(x, result); -} - void LIRGenerator::do_SwitchRanges(SwitchRangeArray* x, LIR_Opr value, BlockBegin* default_sux) { int lng = x->length(); @@ -2869,14 +2887,14 @@ void LIRGenerator::do_Intrinsic(Intrinsic* x) { do_PreconditionsCheckIndex(x, T_LONG); break; - case vmIntrinsics::_compareAndSetReference: - do_CompareAndSwap(x, objectType); - break; - case vmIntrinsics::_compareAndSetInt: - do_CompareAndSwap(x, intType); + case vmIntrinsics::_compareAndSetReferenceMO: + case vmIntrinsics::_compareAndSetPrimitiveBitsMO: + do_CompareAndSwap(x); break; - case vmIntrinsics::_compareAndSetLong: - do_CompareAndSwap(x, longType); + + case vmIntrinsics::_getAndSetReferenceMO: + case vmIntrinsics::_getAndOperatePrimitiveBitsMO: + do_GetAndOperate(x); break; case vmIntrinsics::_loadFence : @@ -2917,7 +2935,10 @@ void LIRGenerator::do_Intrinsic(Intrinsic* x) { do_blackhole(x); break; - default: ShouldNotReachHere(); break; + default: + guarantee(false, "unhandled do_Intrinsic %d: %s", + (int)x->id(), vmIntrinsics::name_at(x->id())); + break; } } diff --git a/src/hotspot/share/c1/c1_LIRGenerator.hpp b/src/hotspot/share/c1/c1_LIRGenerator.hpp index ec0ea5dc047d3..0bb69f83344d9 100644 --- a/src/hotspot/share/c1/c1_LIRGenerator.hpp +++ b/src/hotspot/share/c1/c1_LIRGenerator.hpp @@ -263,7 +263,8 @@ class LIRGenerator: public InstructionVisitor, public BlockClosure { void do_MathIntrinsic(Intrinsic* x); void do_LibmIntrinsic(Intrinsic* x); void do_ArrayCopy(Intrinsic* x); - void do_CompareAndSwap(Intrinsic* x, ValueType* type); + void do_CompareAndSwap(Intrinsic* x); + void do_GetAndOperate(Intrinsic* x); void do_PreconditionsCheckIndex(Intrinsic* x, BasicType type); void do_FPIntrinsics(Intrinsic* x); void do_Reference_get0(Intrinsic* x); @@ -580,7 +581,6 @@ class LIRGenerator: public InstructionVisitor, public BlockClosure { virtual void do_ExceptionObject(ExceptionObject* x); virtual void do_UnsafeGet (UnsafeGet* x); virtual void do_UnsafePut (UnsafePut* x); - virtual void do_UnsafeGetAndSet(UnsafeGetAndSet* x); virtual void do_ProfileCall (ProfileCall* x); virtual void do_ProfileReturnType (ProfileReturnType* x); virtual void do_ProfileInvoke (ProfileInvoke* x); diff --git a/src/hotspot/share/c1/c1_Optimizer.cpp b/src/hotspot/share/c1/c1_Optimizer.cpp index 0c18694df785b..b7475c2699d02 100644 --- a/src/hotspot/share/c1/c1_Optimizer.cpp +++ b/src/hotspot/share/c1/c1_Optimizer.cpp @@ -579,7 +579,6 @@ class NullCheckVisitor: public InstructionVisitor { void do_ExceptionObject(ExceptionObject* x); void do_UnsafeGet (UnsafeGet* x); void do_UnsafePut (UnsafePut* x); - void do_UnsafeGetAndSet(UnsafeGetAndSet* x); void do_ProfileCall (ProfileCall* x); void do_ProfileReturnType (ProfileReturnType* x); void do_ProfileInvoke (ProfileInvoke* x); @@ -763,7 +762,6 @@ void NullCheckVisitor::do_OsrEntry (OsrEntry* x) {} void NullCheckVisitor::do_ExceptionObject(ExceptionObject* x) { nce()->handle_ExceptionObject(x); } void NullCheckVisitor::do_UnsafeGet (UnsafeGet* x) {} void NullCheckVisitor::do_UnsafePut (UnsafePut* x) {} -void NullCheckVisitor::do_UnsafeGetAndSet(UnsafeGetAndSet* x) {} void NullCheckVisitor::do_ProfileCall (ProfileCall* x) { nce()->clear_last_explicit_null_check(); nce()->handle_ProfileCall(x); } void NullCheckVisitor::do_ProfileReturnType (ProfileReturnType* x) { nce()->handle_ProfileReturnType(x); } diff --git a/src/hotspot/share/c1/c1_RangeCheckElimination.hpp b/src/hotspot/share/c1/c1_RangeCheckElimination.hpp index 833f5dd1e99cb..41a0995f6f844 100644 --- a/src/hotspot/share/c1/c1_RangeCheckElimination.hpp +++ b/src/hotspot/share/c1/c1_RangeCheckElimination.hpp @@ -156,7 +156,6 @@ class RangeCheckEliminator { void do_ExceptionObject(ExceptionObject* x) { /* nothing to do */ }; void do_UnsafePut (UnsafePut* x) { /* nothing to do */ }; void do_UnsafeGet (UnsafeGet* x) { /* nothing to do */ }; - void do_UnsafeGetAndSet(UnsafeGetAndSet* x) { /* nothing to do */ }; void do_ProfileCall (ProfileCall* x) { /* nothing to do */ }; void do_ProfileReturnType (ProfileReturnType* x) { /* nothing to do */ }; void do_ProfileInvoke (ProfileInvoke* x) { /* nothing to do */ }; diff --git a/src/hotspot/share/c1/c1_ValueMap.hpp b/src/hotspot/share/c1/c1_ValueMap.hpp index 6583a07c920d4..ebe2d6fa64869 100644 --- a/src/hotspot/share/c1/c1_ValueMap.hpp +++ b/src/hotspot/share/c1/c1_ValueMap.hpp @@ -154,9 +154,8 @@ class ValueNumberingVisitor: public InstructionVisitor { void do_MonitorExit (MonitorExit* x) { kill_memory(); } void do_Invoke (Invoke* x) { kill_memory(); } void do_UnsafePut (UnsafePut* x) { kill_memory(); } - void do_UnsafeGetAndSet(UnsafeGetAndSet* x) { kill_memory(); } void do_UnsafeGet (UnsafeGet* x) { - if (x->is_volatile()) { // the JMM requires this + if (x->memory_order() != vmIntrinsics::MO_PLAIN) { // the JMM requires this kill_memory(); } } diff --git a/src/hotspot/share/classfile/classFileParser.cpp b/src/hotspot/share/classfile/classFileParser.cpp index c9d9d3632b522..6b9fff7e60eb3 100644 --- a/src/hotspot/share/classfile/classFileParser.cpp +++ b/src/hotspot/share/classfile/classFileParser.cpp @@ -4947,11 +4947,16 @@ static void check_methods_for_intrinsics(const InstanceKlass* ik, if (CheckIntrinsics) { // Check if an intrinsic is defined for method 'method', // but the method is not annotated with @IntrinsicCandidate. - if (method->intrinsic_id() != vmIntrinsics::_none && - !method->intrinsic_candidate()) { - tty->print("Compiler intrinsic is defined for method [%s], " - "but the method is not annotated with @IntrinsicCandidate.%s", + vmIntrinsicID iid = method->intrinsic_id(); + bool is_wrapper = (iid != vmIntrinsics::_none && + vmIntrinsics::flags_for(iid) == vmIntrinsics::F_PW); + bool expect_annotation = (iid != vmIntrinsics::_none && !is_wrapper); + if (method->intrinsic_candidate() != expect_annotation) { + tty->print("Compiler intrinsic%s is defined for method [%s], " + "but the method is %s annotated with @IntrinsicCandidate.%s", + (is_wrapper ? " wrapper" : ""), method->name_and_sig_as_C_string(), + (expect_annotation ? "not" : "incorrectly"), NOT_DEBUG(" Method will not be inlined.") DEBUG_ONLY(" Exiting.") ); tty->cr(); @@ -4980,6 +4985,7 @@ static void check_methods_for_intrinsics(const InstanceKlass* ik, // The check is potentially expensive, therefore it is available // only in debug builds. + int fails = 0; // collect all the failing methods at once for (auto id : EnumRange{}) { if (vmIntrinsics::_compiledLambdaForm == id) { // The _compiledLamdbdaForm intrinsic is a special marker for bytecode @@ -5008,16 +5014,20 @@ static void check_methods_for_intrinsics(const InstanceKlass* ik, if (!match) { char buf[1000]; tty->print("Compiler intrinsic is defined for method [%s], " - "but the method is not available in class [%s].%s", + "but the method is not available in class [%s].", vmIntrinsics::short_name_as_C_string(id, buf, sizeof(buf)), - ik->name()->as_C_string(), - NOT_DEBUG("") DEBUG_ONLY(" Exiting.") + ik->name()->as_C_string() ); tty->cr(); - DEBUG_ONLY(vm_exit(1)); + ++fails; } } } // end for + if (fails > 0) { + tty->print_cr("Compiler intrinsic not available for %d methods.%s", + fails, NOT_DEBUG("") DEBUG_ONLY(" Exiting.")); + DEBUG_ONLY(vm_exit(1)); + } } // CheckIntrinsics #endif // ASSERT } diff --git a/src/hotspot/share/classfile/vmIntrinsics.cpp b/src/hotspot/share/classfile/vmIntrinsics.cpp index f9b12df84caff..dbf84c18c9861 100644 --- a/src/hotspot/share/classfile/vmIntrinsics.cpp +++ b/src/hotspot/share/classfile/vmIntrinsics.cpp @@ -65,6 +65,44 @@ inline bool match_F_SN(u2 flags) { return (flags & (req | neg)) == req; } +inline bool match_F_PW(u2 flags) { + const int req = 0; // the wrapper might be static or not + const int neg = JVM_ACC_SYNCHRONIZED | JVM_ACC_NATIVE; + return (flags & (req | neg)) == req; +} + +inline bool match_F_PI(u2 flags) { + const int req = 0; // the intrinsic is a plain or native method + const int neg = JVM_ACC_SYNCHRONIZED; + return (flags & (req | neg)) == req; +} + +vmIntrinsics::PolymorphicPrefix +vmIntrinsics::polymorphic_prefix(vmIntrinsics::ID id) { + if (flags_for(id) == F_PI) { + switch (id) { + case _getReferenceMO: + case _putReferenceMO: + case _compareAndSetReferenceMO: + case _compareAndExchangeReferenceMO: + return PP_MO; + + case _getPrimitiveBitsMO: + case _putPrimitiveBitsMO: + case _compareAndSetPrimitiveBitsMO: + case _compareAndExchangePrimitiveBitsMO: + case _getAndSetReferenceMO: + return PP_MO_BT; + + case _getAndOperatePrimitiveBitsMO: + return PP_MO_BT_OP; + + default: break; + } + } + return PP_NONE; +} + bool vmIntrinsics::preserves_state(vmIntrinsics::ID id) { assert(id != vmIntrinsics::_none, "must be a VM intrinsic"); switch(id) { @@ -336,6 +374,20 @@ bool vmIntrinsics::disabled_by_jvm_flags(vmIntrinsics::ID id) { case vmIntrinsics::_updateByteBufferCRC32: if (!UseCRC32Intrinsics) return true; break; + case vmIntrinsics::_getPrimitiveBitsMO: + case vmIntrinsics::_putPrimitiveBitsMO: + case vmIntrinsics::_getReferenceMO: + case vmIntrinsics::_putReferenceMO: + case vmIntrinsics::_compareAndSetPrimitiveBitsMO: + case vmIntrinsics::_compareAndExchangePrimitiveBitsMO: + case vmIntrinsics::_compareAndSetReferenceMO: + case vmIntrinsics::_compareAndExchangeReferenceMO: + case vmIntrinsics::_getAndOperatePrimitiveBitsMO: + case vmIntrinsics::_getAndSetReferenceMO: + // these are the polymorphic unsafe primitives + if (!InlineUnsafeOps) return true; + break; + // these are all wrappers; they are not subject to intrinsic expansion per se: case vmIntrinsics::_getReference: case vmIntrinsics::_getBoolean: case vmIntrinsics::_getByte: @@ -354,60 +406,6 @@ bool vmIntrinsics::disabled_by_jvm_flags(vmIntrinsics::ID id) { case vmIntrinsics::_putLong: case vmIntrinsics::_putFloat: case vmIntrinsics::_putDouble: - case vmIntrinsics::_getReferenceVolatile: - case vmIntrinsics::_getBooleanVolatile: - case vmIntrinsics::_getByteVolatile: - case vmIntrinsics::_getShortVolatile: - case vmIntrinsics::_getCharVolatile: - case vmIntrinsics::_getIntVolatile: - case vmIntrinsics::_getLongVolatile: - case vmIntrinsics::_getFloatVolatile: - case vmIntrinsics::_getDoubleVolatile: - case vmIntrinsics::_putReferenceVolatile: - case vmIntrinsics::_putBooleanVolatile: - case vmIntrinsics::_putByteVolatile: - case vmIntrinsics::_putShortVolatile: - case vmIntrinsics::_putCharVolatile: - case vmIntrinsics::_putIntVolatile: - case vmIntrinsics::_putLongVolatile: - case vmIntrinsics::_putFloatVolatile: - case vmIntrinsics::_putDoubleVolatile: - case vmIntrinsics::_getReferenceAcquire: - case vmIntrinsics::_getBooleanAcquire: - case vmIntrinsics::_getByteAcquire: - case vmIntrinsics::_getShortAcquire: - case vmIntrinsics::_getCharAcquire: - case vmIntrinsics::_getIntAcquire: - case vmIntrinsics::_getLongAcquire: - case vmIntrinsics::_getFloatAcquire: - case vmIntrinsics::_getDoubleAcquire: - case vmIntrinsics::_putReferenceRelease: - case vmIntrinsics::_putBooleanRelease: - case vmIntrinsics::_putByteRelease: - case vmIntrinsics::_putShortRelease: - case vmIntrinsics::_putCharRelease: - case vmIntrinsics::_putIntRelease: - case vmIntrinsics::_putLongRelease: - case vmIntrinsics::_putFloatRelease: - case vmIntrinsics::_putDoubleRelease: - case vmIntrinsics::_getReferenceOpaque: - case vmIntrinsics::_getBooleanOpaque: - case vmIntrinsics::_getByteOpaque: - case vmIntrinsics::_getShortOpaque: - case vmIntrinsics::_getCharOpaque: - case vmIntrinsics::_getIntOpaque: - case vmIntrinsics::_getLongOpaque: - case vmIntrinsics::_getFloatOpaque: - case vmIntrinsics::_getDoubleOpaque: - case vmIntrinsics::_putReferenceOpaque: - case vmIntrinsics::_putBooleanOpaque: - case vmIntrinsics::_putByteOpaque: - case vmIntrinsics::_putShortOpaque: - case vmIntrinsics::_putCharOpaque: - case vmIntrinsics::_putIntOpaque: - case vmIntrinsics::_putLongOpaque: - case vmIntrinsics::_putFloatOpaque: - case vmIntrinsics::_putDoubleOpaque: case vmIntrinsics::_getAndAddInt: case vmIntrinsics::_getAndAddLong: case vmIntrinsics::_getAndSetInt: @@ -418,28 +416,13 @@ bool vmIntrinsics::disabled_by_jvm_flags(vmIntrinsics::ID id) { case vmIntrinsics::_fullFence: case vmIntrinsics::_compareAndSetLong: case vmIntrinsics::_weakCompareAndSetLong: - case vmIntrinsics::_weakCompareAndSetLongPlain: - case vmIntrinsics::_weakCompareAndSetLongAcquire: - case vmIntrinsics::_weakCompareAndSetLongRelease: case vmIntrinsics::_compareAndSetInt: case vmIntrinsics::_weakCompareAndSetInt: - case vmIntrinsics::_weakCompareAndSetIntPlain: - case vmIntrinsics::_weakCompareAndSetIntAcquire: - case vmIntrinsics::_weakCompareAndSetIntRelease: case vmIntrinsics::_compareAndSetReference: case vmIntrinsics::_weakCompareAndSetReference: - case vmIntrinsics::_weakCompareAndSetReferencePlain: - case vmIntrinsics::_weakCompareAndSetReferenceAcquire: - case vmIntrinsics::_weakCompareAndSetReferenceRelease: case vmIntrinsics::_compareAndExchangeInt: - case vmIntrinsics::_compareAndExchangeIntAcquire: - case vmIntrinsics::_compareAndExchangeIntRelease: case vmIntrinsics::_compareAndExchangeLong: - case vmIntrinsics::_compareAndExchangeLongAcquire: - case vmIntrinsics::_compareAndExchangeLongRelease: case vmIntrinsics::_compareAndExchangeReference: - case vmIntrinsics::_compareAndExchangeReferenceAcquire: - case vmIntrinsics::_compareAndExchangeReferenceRelease: case vmIntrinsics::_allocateInstance: if (!InlineUnsafeOps) return true; break; @@ -687,6 +670,8 @@ vmIntrinsics::ID vmIntrinsics::find_id(const char* name) { return _none; } +// True if the VM version supports it and it is not flag-disabled. +// (Later, a compiler backend and compiler directives can fine-tune.) bool vmIntrinsics::is_intrinsic_available(vmIntrinsics::ID id) { return VM_Version::is_intrinsic_supported(id) && !is_disabled_by_flags(id); } @@ -806,6 +791,8 @@ const char* vmIntrinsics::short_name_as_C_string(vmIntrinsics::ID id, char* buf, case F_RN: fname = "native "; break; case F_SN: fname = "native static "; break; case F_S: fname = "static "; break; + case F_PI: fname = "/*poly*/ "; break; + case F_PW: fname = "/*poly wrapper*/ "; break; default: break; } const char* kptr = strrchr(kname, JVM_SIGNATURE_SLASH); @@ -823,7 +810,6 @@ const char* vmIntrinsics::short_name_as_C_string(vmIntrinsics::ID id, char* buf, #define ID4(x, y, z, f) ((ID3(x, y, z) << vmIntrinsics::log2_FLAG_LIMIT) | (jlong) (f)) -#ifndef PRODUCT static const jlong intrinsic_info_array[vmIntrinsics::number_of_intrinsics()+1] = { #define VM_INTRINSIC_INFO(ignore_id, klass, name, sig, fcode) \ ID4(SID_ENUM(klass), SID_ENUM(name), SID_ENUM(sig), vmIntrinsics::fcode), @@ -840,29 +826,28 @@ inline jlong intrinsic_info(vmIntrinsics::ID id) { vmSymbolID vmIntrinsics::class_for(vmIntrinsics::ID id) { jlong info = intrinsic_info(id); - int shift = 2*vmSymbols::log2_SID_LIMIT + log2_FLAG_LIMIT, mask = right_n_bits(vmSymbols::log2_SID_LIMIT); - assert(((ID4(1021,1022,1023,7) >> shift) & mask) == 1021, ""); + const int shift = 2*vmSymbols::log2_SID_LIMIT + log2_FLAG_LIMIT, mask = right_n_bits(vmSymbols::log2_SID_LIMIT); + static_assert(((ID4(1021,1022,1023,7) >> shift) & mask) == 1021, ""); return vmSymbols::as_SID( checked_cast((info >> shift) & mask)); } vmSymbolID vmIntrinsics::name_for(vmIntrinsics::ID id) { jlong info = intrinsic_info(id); - int shift = vmSymbols::log2_SID_LIMIT + log2_FLAG_LIMIT, mask = right_n_bits(vmSymbols::log2_SID_LIMIT); - assert(((ID4(1021,1022,1023,7) >> shift) & mask) == 1022, ""); + const int shift = vmSymbols::log2_SID_LIMIT + log2_FLAG_LIMIT, mask = right_n_bits(vmSymbols::log2_SID_LIMIT); + static_assert(((ID4(1021,1022,1023,7) >> shift) & mask) == 1022, ""); return vmSymbols::as_SID( checked_cast((info >> shift) & mask)); } vmSymbolID vmIntrinsics::signature_for(vmIntrinsics::ID id) { jlong info = intrinsic_info(id); - int shift = log2_FLAG_LIMIT, mask = right_n_bits(vmSymbols::log2_SID_LIMIT); - assert(((ID4(1021,1022,1023,7) >> shift) & mask) == 1023, ""); + const int shift = log2_FLAG_LIMIT, mask = right_n_bits(vmSymbols::log2_SID_LIMIT); + static_assert(((ID4(1021,1022,1023,7) >> shift) & mask) == 1023, ""); return vmSymbols::as_SID( checked_cast((info >> shift) & mask)); } vmIntrinsics::Flags vmIntrinsics::flags_for(vmIntrinsics::ID id) { jlong info = intrinsic_info(id); - int shift = 0, mask = right_n_bits(log2_FLAG_LIMIT); - assert(((ID4(1021,1022,1023,7) >> shift) & mask) == 7, ""); + const int shift = 0, mask = right_n_bits(log2_FLAG_LIMIT); + static_assert(((ID4(1021,1022,1023,7) >> shift) & mask) == 7, ""); return Flags( (info >> shift) & mask ); } -#endif // !PRODUCT diff --git a/src/hotspot/share/classfile/vmIntrinsics.hpp b/src/hotspot/share/classfile/vmIntrinsics.hpp index 6f9c2326a45a3..9f22ae22eca6e 100644 --- a/src/hotspot/share/classfile/vmIntrinsics.hpp +++ b/src/hotspot/share/classfile/vmIntrinsics.hpp @@ -29,6 +29,7 @@ #include "memory/allStatic.hpp" #include "utilities/enumIterator.hpp" #include "utilities/globalDefinitions.hpp" +#include "utilities/tribool.hpp" #include "utilities/vmEnums.hpp" class Method; @@ -709,7 +710,40 @@ class methodHandle; /* special marker for blackholed methods: */ \ do_intrinsic(_blackhole, java_lang_Object, blackhole_name, star_name, F_S) \ \ - /* unsafe memory references (there are a lot of them...) */ \ + /* unsafe memory references */ \ + do_intrinsic(_getPrimitiveBitsMO, jdk_internal_misc_Unsafe, getPrimitiveBitsMO_name, getPrimitiveBitsMO_signature, F_PI) \ + do_name( getPrimitiveBitsMO_name, "getPrimitiveBitsMO") \ + do_signature(getPrimitiveBitsMO_signature, "(BBLjava/lang/Object;J)J") \ + do_intrinsic(_putPrimitiveBitsMO, jdk_internal_misc_Unsafe, putPrimitiveBitsMO_name, putPrimitiveBitsMO_signature, F_PI) \ + do_name( putPrimitiveBitsMO_name, "putPrimitiveBitsMO") \ + do_signature(putPrimitiveBitsMO_signature, "(BBLjava/lang/Object;JJ)V") \ + do_intrinsic(_getReferenceMO, jdk_internal_misc_Unsafe, getReferenceMO_name, getReferenceMO_signature, F_PI) \ + do_name( getReferenceMO_name, "getReferenceMO") \ + do_signature(getReferenceMO_signature, "(BLjava/lang/Object;J)Ljava/lang/Object;") \ + do_intrinsic(_putReferenceMO, jdk_internal_misc_Unsafe, putReferenceMO_name, putReferenceMO_signature, F_PI) \ + do_name( putReferenceMO_name, "putReferenceMO") \ + do_signature(putReferenceMO_signature, "(BLjava/lang/Object;JLjava/lang/Object;)V") \ + \ + do_intrinsic(_compareAndSetPrimitiveBitsMO, jdk_internal_misc_Unsafe, compareAndSetPrimitiveBitsMO_name, compareAndSetPrimitiveBitsMO_signature, F_PI) \ + do_name( compareAndSetPrimitiveBitsMO_name, "compareAndSetPrimitiveBitsMO") \ + do_signature(compareAndSetPrimitiveBitsMO_signature, "(BBLjava/lang/Object;JJJ)Z") \ + do_intrinsic(_compareAndExchangePrimitiveBitsMO, jdk_internal_misc_Unsafe, compareAndExchangePrimitiveBitsMO_name, compareAndExchangePrimitiveBitsMO_signature, F_PI) \ + do_name( compareAndExchangePrimitiveBitsMO_name, "compareAndExchangePrimitiveBitsMO") \ + do_signature(compareAndExchangePrimitiveBitsMO_signature, "(BBLjava/lang/Object;JJJ)J") \ + do_intrinsic(_compareAndSetReferenceMO, jdk_internal_misc_Unsafe, compareAndSetReferenceMO_name, compareAndSetReferenceMO_signature, F_PI) \ + do_name( compareAndSetReferenceMO_name, "compareAndSetReferenceMO") \ + do_signature(compareAndSetReferenceMO_signature, "(BLjava/lang/Object;JLjava/lang/Object;Ljava/lang/Object;)Z") \ + do_intrinsic(_compareAndExchangeReferenceMO, jdk_internal_misc_Unsafe, compareAndExchangeReferenceMO_name, compareAndExchangeReferenceMO_signature, F_PI) \ + do_name( compareAndExchangeReferenceMO_name, "compareAndExchangeReferenceMO") \ + do_signature(compareAndExchangeReferenceMO_signature, "(BLjava/lang/Object;JLjava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;") \ + do_intrinsic(_getAndOperatePrimitiveBitsMO, jdk_internal_misc_Unsafe, getAndOperatePrimitiveBitsMO_name, getAndOperatePrimitiveBitsMO_signature, F_PI) \ + do_name( getAndOperatePrimitiveBitsMO_name, "getAndOperatePrimitiveBitsMO") \ + do_signature(getAndOperatePrimitiveBitsMO_signature, "(BBBLjava/lang/Object;JJ)J" ) \ + do_intrinsic(_getAndSetReferenceMO, jdk_internal_misc_Unsafe, getAndSetReferenceMO_name, getAndSetReferenceMO_signature, F_PI) \ + do_name( getAndSetReferenceMO_name, "getAndSetReferenceMO") \ + do_signature(getAndSetReferenceMO_signature, "(BLjava/lang/Object;JLjava/lang/Object;)Ljava/lang/Object;") \ + \ + /* wrappers for polymorphic unsafe intrinsics (there are a lot of them...) */ \ do_signature(getReference_signature, "(Ljava/lang/Object;J)Ljava/lang/Object;") \ do_signature(putReference_signature, "(Ljava/lang/Object;JLjava/lang/Object;)V") \ do_signature(getBoolean_signature, "(Ljava/lang/Object;J)Z") \ @@ -739,125 +773,38 @@ class methodHandle; do_name(getFloat_name,"getFloat") do_name(putFloat_name,"putFloat") \ do_name(getDouble_name,"getDouble") do_name(putDouble_name,"putDouble") \ \ - do_intrinsic(_getReference, jdk_internal_misc_Unsafe, getReference_name, getReference_signature, F_RN) \ - do_intrinsic(_getBoolean, jdk_internal_misc_Unsafe, getBoolean_name, getBoolean_signature, F_RN) \ - do_intrinsic(_getByte, jdk_internal_misc_Unsafe, getByte_name, getByte_signature, F_RN) \ - do_intrinsic(_getShort, jdk_internal_misc_Unsafe, getShort_name, getShort_signature, F_RN) \ - do_intrinsic(_getChar, jdk_internal_misc_Unsafe, getChar_name, getChar_signature, F_RN) \ - do_intrinsic(_getInt, jdk_internal_misc_Unsafe, getInt_name, getInt_signature, F_RN) \ - do_intrinsic(_getLong, jdk_internal_misc_Unsafe, getLong_name, getLong_signature, F_RN) \ - do_intrinsic(_getFloat, jdk_internal_misc_Unsafe, getFloat_name, getFloat_signature, F_RN) \ - do_intrinsic(_getDouble, jdk_internal_misc_Unsafe, getDouble_name, getDouble_signature, F_RN) \ - do_intrinsic(_putReference, jdk_internal_misc_Unsafe, putReference_name, putReference_signature, F_RN) \ - do_intrinsic(_putBoolean, jdk_internal_misc_Unsafe, putBoolean_name, putBoolean_signature, F_RN) \ - do_intrinsic(_putByte, jdk_internal_misc_Unsafe, putByte_name, putByte_signature, F_RN) \ - do_intrinsic(_putShort, jdk_internal_misc_Unsafe, putShort_name, putShort_signature, F_RN) \ - do_intrinsic(_putChar, jdk_internal_misc_Unsafe, putChar_name, putChar_signature, F_RN) \ - do_intrinsic(_putInt, jdk_internal_misc_Unsafe, putInt_name, putInt_signature, F_RN) \ - do_intrinsic(_putLong, jdk_internal_misc_Unsafe, putLong_name, putLong_signature, F_RN) \ - do_intrinsic(_putFloat, jdk_internal_misc_Unsafe, putFloat_name, putFloat_signature, F_RN) \ - do_intrinsic(_putDouble, jdk_internal_misc_Unsafe, putDouble_name, putDouble_signature, F_RN) \ - \ - do_name(getReferenceVolatile_name,"getReferenceVolatile") do_name(putReferenceVolatile_name,"putReferenceVolatile") \ - do_name(getBooleanVolatile_name,"getBooleanVolatile") do_name(putBooleanVolatile_name,"putBooleanVolatile") \ - do_name(getByteVolatile_name,"getByteVolatile") do_name(putByteVolatile_name,"putByteVolatile") \ - do_name(getShortVolatile_name,"getShortVolatile") do_name(putShortVolatile_name,"putShortVolatile") \ - do_name(getCharVolatile_name,"getCharVolatile") do_name(putCharVolatile_name,"putCharVolatile") \ - do_name(getIntVolatile_name,"getIntVolatile") do_name(putIntVolatile_name,"putIntVolatile") \ - do_name(getLongVolatile_name,"getLongVolatile") do_name(putLongVolatile_name,"putLongVolatile") \ - do_name(getFloatVolatile_name,"getFloatVolatile") do_name(putFloatVolatile_name,"putFloatVolatile") \ - do_name(getDoubleVolatile_name,"getDoubleVolatile") do_name(putDoubleVolatile_name,"putDoubleVolatile") \ - \ - do_intrinsic(_getReferenceVolatile, jdk_internal_misc_Unsafe, getReferenceVolatile_name, getReference_signature, F_RN) \ - do_intrinsic(_getBooleanVolatile, jdk_internal_misc_Unsafe, getBooleanVolatile_name, getBoolean_signature, F_RN) \ - do_intrinsic(_getByteVolatile, jdk_internal_misc_Unsafe, getByteVolatile_name, getByte_signature, F_RN) \ - do_intrinsic(_getShortVolatile, jdk_internal_misc_Unsafe, getShortVolatile_name, getShort_signature, F_RN) \ - do_intrinsic(_getCharVolatile, jdk_internal_misc_Unsafe, getCharVolatile_name, getChar_signature, F_RN) \ - do_intrinsic(_getIntVolatile, jdk_internal_misc_Unsafe, getIntVolatile_name, getInt_signature, F_RN) \ - do_intrinsic(_getLongVolatile, jdk_internal_misc_Unsafe, getLongVolatile_name, getLong_signature, F_RN) \ - do_intrinsic(_getFloatVolatile, jdk_internal_misc_Unsafe, getFloatVolatile_name, getFloat_signature, F_RN) \ - do_intrinsic(_getDoubleVolatile, jdk_internal_misc_Unsafe, getDoubleVolatile_name, getDouble_signature, F_RN) \ - do_intrinsic(_putReferenceVolatile, jdk_internal_misc_Unsafe, putReferenceVolatile_name, putReference_signature, F_RN) \ - do_intrinsic(_putBooleanVolatile, jdk_internal_misc_Unsafe, putBooleanVolatile_name, putBoolean_signature, F_RN) \ - do_intrinsic(_putByteVolatile, jdk_internal_misc_Unsafe, putByteVolatile_name, putByte_signature, F_RN) \ - do_intrinsic(_putShortVolatile, jdk_internal_misc_Unsafe, putShortVolatile_name, putShort_signature, F_RN) \ - do_intrinsic(_putCharVolatile, jdk_internal_misc_Unsafe, putCharVolatile_name, putChar_signature, F_RN) \ - do_intrinsic(_putIntVolatile, jdk_internal_misc_Unsafe, putIntVolatile_name, putInt_signature, F_RN) \ - do_intrinsic(_putLongVolatile, jdk_internal_misc_Unsafe, putLongVolatile_name, putLong_signature, F_RN) \ - do_intrinsic(_putFloatVolatile, jdk_internal_misc_Unsafe, putFloatVolatile_name, putFloat_signature, F_RN) \ - do_intrinsic(_putDoubleVolatile, jdk_internal_misc_Unsafe, putDoubleVolatile_name, putDouble_signature, F_RN) \ - \ - do_name(getReferenceOpaque_name,"getReferenceOpaque") do_name(putReferenceOpaque_name,"putReferenceOpaque") \ - do_name(getBooleanOpaque_name,"getBooleanOpaque") do_name(putBooleanOpaque_name,"putBooleanOpaque") \ - do_name(getByteOpaque_name,"getByteOpaque") do_name(putByteOpaque_name,"putByteOpaque") \ - do_name(getShortOpaque_name,"getShortOpaque") do_name(putShortOpaque_name,"putShortOpaque") \ - do_name(getCharOpaque_name,"getCharOpaque") do_name(putCharOpaque_name,"putCharOpaque") \ - do_name(getIntOpaque_name,"getIntOpaque") do_name(putIntOpaque_name,"putIntOpaque") \ - do_name(getLongOpaque_name,"getLongOpaque") do_name(putLongOpaque_name,"putLongOpaque") \ - do_name(getFloatOpaque_name,"getFloatOpaque") do_name(putFloatOpaque_name,"putFloatOpaque") \ - do_name(getDoubleOpaque_name,"getDoubleOpaque") do_name(putDoubleOpaque_name,"putDoubleOpaque") \ - \ - do_intrinsic(_getReferenceOpaque, jdk_internal_misc_Unsafe, getReferenceOpaque_name, getReference_signature, F_R) \ - do_intrinsic(_getBooleanOpaque, jdk_internal_misc_Unsafe, getBooleanOpaque_name, getBoolean_signature, F_R) \ - do_intrinsic(_getByteOpaque, jdk_internal_misc_Unsafe, getByteOpaque_name, getByte_signature, F_R) \ - do_intrinsic(_getShortOpaque, jdk_internal_misc_Unsafe, getShortOpaque_name, getShort_signature, F_R) \ - do_intrinsic(_getCharOpaque, jdk_internal_misc_Unsafe, getCharOpaque_name, getChar_signature, F_R) \ - do_intrinsic(_getIntOpaque, jdk_internal_misc_Unsafe, getIntOpaque_name, getInt_signature, F_R) \ - do_intrinsic(_getLongOpaque, jdk_internal_misc_Unsafe, getLongOpaque_name, getLong_signature, F_R) \ - do_intrinsic(_getFloatOpaque, jdk_internal_misc_Unsafe, getFloatOpaque_name, getFloat_signature, F_R) \ - do_intrinsic(_getDoubleOpaque, jdk_internal_misc_Unsafe, getDoubleOpaque_name, getDouble_signature, F_R) \ - do_intrinsic(_putReferenceOpaque, jdk_internal_misc_Unsafe, putReferenceOpaque_name, putReference_signature, F_R) \ - do_intrinsic(_putBooleanOpaque, jdk_internal_misc_Unsafe, putBooleanOpaque_name, putBoolean_signature, F_R) \ - do_intrinsic(_putByteOpaque, jdk_internal_misc_Unsafe, putByteOpaque_name, putByte_signature, F_R) \ - do_intrinsic(_putShortOpaque, jdk_internal_misc_Unsafe, putShortOpaque_name, putShort_signature, F_R) \ - do_intrinsic(_putCharOpaque, jdk_internal_misc_Unsafe, putCharOpaque_name, putChar_signature, F_R) \ - do_intrinsic(_putIntOpaque, jdk_internal_misc_Unsafe, putIntOpaque_name, putInt_signature, F_R) \ - do_intrinsic(_putLongOpaque, jdk_internal_misc_Unsafe, putLongOpaque_name, putLong_signature, F_R) \ - do_intrinsic(_putFloatOpaque, jdk_internal_misc_Unsafe, putFloatOpaque_name, putFloat_signature, F_R) \ - do_intrinsic(_putDoubleOpaque, jdk_internal_misc_Unsafe, putDoubleOpaque_name, putDouble_signature, F_R) \ - \ - do_name(getReferenceAcquire_name, "getReferenceAcquire") do_name(putReferenceRelease_name, "putReferenceRelease") \ - do_name(getBooleanAcquire_name, "getBooleanAcquire") do_name(putBooleanRelease_name, "putBooleanRelease") \ - do_name(getByteAcquire_name, "getByteAcquire") do_name(putByteRelease_name, "putByteRelease") \ - do_name(getShortAcquire_name, "getShortAcquire") do_name(putShortRelease_name, "putShortRelease") \ - do_name(getCharAcquire_name, "getCharAcquire") do_name(putCharRelease_name, "putCharRelease") \ - do_name(getIntAcquire_name, "getIntAcquire") do_name(putIntRelease_name, "putIntRelease") \ - do_name(getLongAcquire_name, "getLongAcquire") do_name(putLongRelease_name, "putLongRelease") \ - do_name(getFloatAcquire_name, "getFloatAcquire") do_name(putFloatRelease_name, "putFloatRelease") \ - do_name(getDoubleAcquire_name, "getDoubleAcquire") do_name(putDoubleRelease_name, "putDoubleRelease") \ - \ - do_intrinsic(_getReferenceAcquire, jdk_internal_misc_Unsafe, getReferenceAcquire_name, getReference_signature, F_R) \ - do_intrinsic(_getBooleanAcquire, jdk_internal_misc_Unsafe, getBooleanAcquire_name, getBoolean_signature, F_R) \ - do_intrinsic(_getByteAcquire, jdk_internal_misc_Unsafe, getByteAcquire_name, getByte_signature, F_R) \ - do_intrinsic(_getShortAcquire, jdk_internal_misc_Unsafe, getShortAcquire_name, getShort_signature, F_R) \ - do_intrinsic(_getCharAcquire, jdk_internal_misc_Unsafe, getCharAcquire_name, getChar_signature, F_R) \ - do_intrinsic(_getIntAcquire, jdk_internal_misc_Unsafe, getIntAcquire_name, getInt_signature, F_R) \ - do_intrinsic(_getLongAcquire, jdk_internal_misc_Unsafe, getLongAcquire_name, getLong_signature, F_R) \ - do_intrinsic(_getFloatAcquire, jdk_internal_misc_Unsafe, getFloatAcquire_name, getFloat_signature, F_R) \ - do_intrinsic(_getDoubleAcquire, jdk_internal_misc_Unsafe, getDoubleAcquire_name, getDouble_signature, F_R) \ - do_intrinsic(_putReferenceRelease, jdk_internal_misc_Unsafe, putReferenceRelease_name, putReference_signature, F_R) \ - do_intrinsic(_putBooleanRelease, jdk_internal_misc_Unsafe, putBooleanRelease_name, putBoolean_signature, F_R) \ - do_intrinsic(_putByteRelease, jdk_internal_misc_Unsafe, putByteRelease_name, putByte_signature, F_R) \ - do_intrinsic(_putShortRelease, jdk_internal_misc_Unsafe, putShortRelease_name, putShort_signature, F_R) \ - do_intrinsic(_putCharRelease, jdk_internal_misc_Unsafe, putCharRelease_name, putChar_signature, F_R) \ - do_intrinsic(_putIntRelease, jdk_internal_misc_Unsafe, putIntRelease_name, putInt_signature, F_R) \ - do_intrinsic(_putLongRelease, jdk_internal_misc_Unsafe, putLongRelease_name, putLong_signature, F_R) \ - do_intrinsic(_putFloatRelease, jdk_internal_misc_Unsafe, putFloatRelease_name, putFloat_signature, F_R) \ - do_intrinsic(_putDoubleRelease, jdk_internal_misc_Unsafe, putDoubleRelease_name, putDouble_signature, F_R) \ + do_intrinsic(_getReference, jdk_internal_misc_Unsafe, getReference_name, getReference_signature, F_PW) \ + do_intrinsic(_getBoolean, jdk_internal_misc_Unsafe, getBoolean_name, getBoolean_signature, F_PW) \ + do_intrinsic(_getByte, jdk_internal_misc_Unsafe, getByte_name, getByte_signature, F_PW) \ + do_intrinsic(_getShort, jdk_internal_misc_Unsafe, getShort_name, getShort_signature, F_PW) \ + do_intrinsic(_getChar, jdk_internal_misc_Unsafe, getChar_name, getChar_signature, F_PW) \ + do_intrinsic(_getInt, jdk_internal_misc_Unsafe, getInt_name, getInt_signature, F_PW) \ + do_intrinsic(_getLong, jdk_internal_misc_Unsafe, getLong_name, getLong_signature, F_PW) \ + do_intrinsic(_getFloat, jdk_internal_misc_Unsafe, getFloat_name, getFloat_signature, F_PW) \ + do_intrinsic(_getDouble, jdk_internal_misc_Unsafe, getDouble_name, getDouble_signature, F_PW) \ + do_intrinsic(_putReference, jdk_internal_misc_Unsafe, putReference_name, putReference_signature, F_PW) \ + do_intrinsic(_putBoolean, jdk_internal_misc_Unsafe, putBoolean_name, putBoolean_signature, F_PW) \ + do_intrinsic(_putByte, jdk_internal_misc_Unsafe, putByte_name, putByte_signature, F_PW) \ + do_intrinsic(_putShort, jdk_internal_misc_Unsafe, putShort_name, putShort_signature, F_PW) \ + do_intrinsic(_putChar, jdk_internal_misc_Unsafe, putChar_name, putChar_signature, F_PW) \ + do_intrinsic(_putInt, jdk_internal_misc_Unsafe, putInt_name, putInt_signature, F_PW) \ + do_intrinsic(_putLong, jdk_internal_misc_Unsafe, putLong_name, putLong_signature, F_PW) \ + do_intrinsic(_putFloat, jdk_internal_misc_Unsafe, putFloat_name, putFloat_signature, F_PW) \ + do_intrinsic(_putDouble, jdk_internal_misc_Unsafe, putDouble_name, putDouble_signature, F_PW) \ \ do_name(getShortUnaligned_name,"getShortUnaligned") do_name(putShortUnaligned_name,"putShortUnaligned") \ do_name(getCharUnaligned_name,"getCharUnaligned") do_name(putCharUnaligned_name,"putCharUnaligned") \ do_name(getIntUnaligned_name,"getIntUnaligned") do_name(putIntUnaligned_name,"putIntUnaligned") \ do_name(getLongUnaligned_name,"getLongUnaligned") do_name(putLongUnaligned_name,"putLongUnaligned") \ \ - do_intrinsic(_getShortUnaligned, jdk_internal_misc_Unsafe, getShortUnaligned_name, getShort_signature, F_R) \ - do_intrinsic(_getCharUnaligned, jdk_internal_misc_Unsafe, getCharUnaligned_name, getChar_signature, F_R) \ - do_intrinsic(_getIntUnaligned, jdk_internal_misc_Unsafe, getIntUnaligned_name, getInt_signature, F_R) \ - do_intrinsic(_getLongUnaligned, jdk_internal_misc_Unsafe, getLongUnaligned_name, getLong_signature, F_R) \ - do_intrinsic(_putShortUnaligned, jdk_internal_misc_Unsafe, putShortUnaligned_name, putShort_signature, F_R) \ - do_intrinsic(_putCharUnaligned, jdk_internal_misc_Unsafe, putCharUnaligned_name, putChar_signature, F_R) \ - do_intrinsic(_putIntUnaligned, jdk_internal_misc_Unsafe, putIntUnaligned_name, putInt_signature, F_R) \ - do_intrinsic(_putLongUnaligned, jdk_internal_misc_Unsafe, putLongUnaligned_name, putLong_signature, F_R) \ + do_intrinsic(_getShortUnaligned, jdk_internal_misc_Unsafe, getShortUnaligned_name, getShort_signature, F_PW) \ + do_intrinsic(_getCharUnaligned, jdk_internal_misc_Unsafe, getCharUnaligned_name, getChar_signature, F_PW) \ + do_intrinsic(_getIntUnaligned, jdk_internal_misc_Unsafe, getIntUnaligned_name, getInt_signature, F_PW) \ + do_intrinsic(_getLongUnaligned, jdk_internal_misc_Unsafe, getLongUnaligned_name, getLong_signature, F_PW) \ + do_intrinsic(_putShortUnaligned, jdk_internal_misc_Unsafe, putShortUnaligned_name, putShort_signature, F_PW) \ + do_intrinsic(_putCharUnaligned, jdk_internal_misc_Unsafe, putCharUnaligned_name, putChar_signature, F_PW) \ + do_intrinsic(_putIntUnaligned, jdk_internal_misc_Unsafe, putIntUnaligned_name, putInt_signature, F_PW) \ + do_intrinsic(_putLongUnaligned, jdk_internal_misc_Unsafe, putLongUnaligned_name, putLong_signature, F_PW) \ \ do_signature(compareAndSetReference_signature, "(Ljava/lang/Object;JLjava/lang/Object;Ljava/lang/Object;)Z") \ do_signature(compareAndExchangeReference_signature, "(Ljava/lang/Object;JLjava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;") \ @@ -872,115 +819,62 @@ class methodHandle; \ do_name(compareAndSetReference_name, "compareAndSetReference") \ do_name(compareAndExchangeReference_name, "compareAndExchangeReference") \ - do_name(compareAndExchangeReferenceAcquire_name, "compareAndExchangeReferenceAcquire") \ - do_name(compareAndExchangeReferenceRelease_name, "compareAndExchangeReferenceRelease") \ do_name(compareAndSetLong_name, "compareAndSetLong") \ do_name(compareAndExchangeLong_name, "compareAndExchangeLong") \ - do_name(compareAndExchangeLongAcquire_name, "compareAndExchangeLongAcquire") \ - do_name(compareAndExchangeLongRelease_name, "compareAndExchangeLongRelease") \ do_name(compareAndSetInt_name, "compareAndSetInt") \ do_name(compareAndExchangeInt_name, "compareAndExchangeInt") \ - do_name(compareAndExchangeIntAcquire_name, "compareAndExchangeIntAcquire") \ - do_name(compareAndExchangeIntRelease_name, "compareAndExchangeIntRelease") \ do_name(compareAndSetByte_name, "compareAndSetByte") \ do_name(compareAndExchangeByte_name, "compareAndExchangeByte") \ - do_name(compareAndExchangeByteAcquire_name, "compareAndExchangeByteAcquire") \ - do_name(compareAndExchangeByteRelease_name, "compareAndExchangeByteRelease") \ do_name(compareAndSetShort_name, "compareAndSetShort") \ do_name(compareAndExchangeShort_name, "compareAndExchangeShort") \ - do_name(compareAndExchangeShortAcquire_name, "compareAndExchangeShortAcquire") \ - do_name(compareAndExchangeShortRelease_name, "compareAndExchangeShortRelease") \ \ - do_name(weakCompareAndSetReferencePlain_name, "weakCompareAndSetReferencePlain") \ - do_name(weakCompareAndSetReferenceAcquire_name, "weakCompareAndSetReferenceAcquire") \ - do_name(weakCompareAndSetReferenceRelease_name, "weakCompareAndSetReferenceRelease") \ do_name(weakCompareAndSetReference_name, "weakCompareAndSetReference") \ - do_name(weakCompareAndSetLongPlain_name, "weakCompareAndSetLongPlain") \ - do_name(weakCompareAndSetLongAcquire_name, "weakCompareAndSetLongAcquire") \ - do_name(weakCompareAndSetLongRelease_name, "weakCompareAndSetLongRelease") \ do_name(weakCompareAndSetLong_name, "weakCompareAndSetLong") \ - do_name(weakCompareAndSetIntPlain_name, "weakCompareAndSetIntPlain") \ - do_name(weakCompareAndSetIntAcquire_name, "weakCompareAndSetIntAcquire") \ - do_name(weakCompareAndSetIntRelease_name, "weakCompareAndSetIntRelease") \ do_name(weakCompareAndSetInt_name, "weakCompareAndSetInt") \ - do_name(weakCompareAndSetBytePlain_name, "weakCompareAndSetBytePlain") \ - do_name(weakCompareAndSetByteAcquire_name, "weakCompareAndSetByteAcquire") \ - do_name(weakCompareAndSetByteRelease_name, "weakCompareAndSetByteRelease") \ - do_name(weakCompareAndSetByte_name, "weakCompareAndSetByte") \ - do_name(weakCompareAndSetShortPlain_name, "weakCompareAndSetShortPlain") \ - do_name(weakCompareAndSetShortAcquire_name, "weakCompareAndSetShortAcquire") \ - do_name(weakCompareAndSetShortRelease_name, "weakCompareAndSetShortRelease") \ - do_name(weakCompareAndSetShort_name, "weakCompareAndSetShort") \ - \ - do_intrinsic(_compareAndSetReference, jdk_internal_misc_Unsafe, compareAndSetReference_name, compareAndSetReference_signature, F_RN) \ - do_intrinsic(_compareAndExchangeReference, jdk_internal_misc_Unsafe, compareAndExchangeReference_name, compareAndExchangeReference_signature, F_RN) \ - do_intrinsic(_compareAndExchangeReferenceAcquire, jdk_internal_misc_Unsafe, compareAndExchangeReferenceAcquire_name, compareAndExchangeReference_signature, F_R) \ - do_intrinsic(_compareAndExchangeReferenceRelease, jdk_internal_misc_Unsafe, compareAndExchangeReferenceRelease_name, compareAndExchangeReference_signature, F_R) \ - do_intrinsic(_compareAndSetLong, jdk_internal_misc_Unsafe, compareAndSetLong_name, compareAndSetLong_signature, F_RN) \ - do_intrinsic(_compareAndExchangeLong, jdk_internal_misc_Unsafe, compareAndExchangeLong_name, compareAndExchangeLong_signature, F_RN) \ - do_intrinsic(_compareAndExchangeLongAcquire, jdk_internal_misc_Unsafe, compareAndExchangeLongAcquire_name, compareAndExchangeLong_signature, F_R) \ - do_intrinsic(_compareAndExchangeLongRelease, jdk_internal_misc_Unsafe, compareAndExchangeLongRelease_name, compareAndExchangeLong_signature, F_R) \ - do_intrinsic(_compareAndSetInt, jdk_internal_misc_Unsafe, compareAndSetInt_name, compareAndSetInt_signature, F_RN) \ - do_intrinsic(_compareAndExchangeInt, jdk_internal_misc_Unsafe, compareAndExchangeInt_name, compareAndExchangeInt_signature, F_RN) \ - do_intrinsic(_compareAndExchangeIntAcquire, jdk_internal_misc_Unsafe, compareAndExchangeIntAcquire_name, compareAndExchangeInt_signature, F_R) \ - do_intrinsic(_compareAndExchangeIntRelease, jdk_internal_misc_Unsafe, compareAndExchangeIntRelease_name, compareAndExchangeInt_signature, F_R) \ - do_intrinsic(_compareAndSetByte, jdk_internal_misc_Unsafe, compareAndSetByte_name, compareAndSetByte_signature, F_R) \ - do_intrinsic(_compareAndExchangeByte, jdk_internal_misc_Unsafe, compareAndExchangeByte_name, compareAndExchangeByte_signature, F_R) \ - do_intrinsic(_compareAndExchangeByteAcquire, jdk_internal_misc_Unsafe, compareAndExchangeByteAcquire_name, compareAndExchangeByte_signature, F_R) \ - do_intrinsic(_compareAndExchangeByteRelease, jdk_internal_misc_Unsafe, compareAndExchangeByteRelease_name, compareAndExchangeByte_signature, F_R) \ - do_intrinsic(_compareAndSetShort, jdk_internal_misc_Unsafe, compareAndSetShort_name, compareAndSetShort_signature, F_R) \ - do_intrinsic(_compareAndExchangeShort, jdk_internal_misc_Unsafe, compareAndExchangeShort_name, compareAndExchangeShort_signature, F_R) \ - do_intrinsic(_compareAndExchangeShortAcquire, jdk_internal_misc_Unsafe, compareAndExchangeShortAcquire_name, compareAndExchangeShort_signature, F_R) \ - do_intrinsic(_compareAndExchangeShortRelease, jdk_internal_misc_Unsafe, compareAndExchangeShortRelease_name, compareAndExchangeShort_signature, F_R) \ + \ + do_intrinsic(_compareAndSetReference, jdk_internal_misc_Unsafe, compareAndSetReference_name, compareAndSetReference_signature, F_PW) \ + do_intrinsic(_compareAndExchangeReference, jdk_internal_misc_Unsafe, compareAndExchangeReference_name, compareAndExchangeReference_signature, F_PW) \ + do_intrinsic(_compareAndSetLong, jdk_internal_misc_Unsafe, compareAndSetLong_name, compareAndSetLong_signature, F_PW) \ + do_intrinsic(_compareAndExchangeLong, jdk_internal_misc_Unsafe, compareAndExchangeLong_name, compareAndExchangeLong_signature, F_PW) \ + do_intrinsic(_compareAndSetInt, jdk_internal_misc_Unsafe, compareAndSetInt_name, compareAndSetInt_signature, F_PW) \ + do_intrinsic(_compareAndExchangeInt, jdk_internal_misc_Unsafe, compareAndExchangeInt_name, compareAndExchangeInt_signature, F_PW) \ + do_intrinsic(_compareAndSetByte, jdk_internal_misc_Unsafe, compareAndSetByte_name, compareAndSetByte_signature, F_PW) \ + do_intrinsic(_compareAndExchangeByte, jdk_internal_misc_Unsafe, compareAndExchangeByte_name, compareAndExchangeByte_signature, F_PW) \ + do_intrinsic(_compareAndSetShort, jdk_internal_misc_Unsafe, compareAndSetShort_name, compareAndSetShort_signature, F_PW) \ + do_intrinsic(_compareAndExchangeShort, jdk_internal_misc_Unsafe, compareAndExchangeShort_name, compareAndExchangeShort_signature, F_PW) \ \ - do_intrinsic(_weakCompareAndSetReferencePlain, jdk_internal_misc_Unsafe, weakCompareAndSetReferencePlain_name, compareAndSetReference_signature, F_R) \ - do_intrinsic(_weakCompareAndSetReferenceAcquire,jdk_internal_misc_Unsafe, weakCompareAndSetReferenceAcquire_name, compareAndSetReference_signature, F_R) \ - do_intrinsic(_weakCompareAndSetReferenceRelease,jdk_internal_misc_Unsafe, weakCompareAndSetReferenceRelease_name, compareAndSetReference_signature, F_R) \ - do_intrinsic(_weakCompareAndSetReference, jdk_internal_misc_Unsafe, weakCompareAndSetReference_name, compareAndSetReference_signature, F_R) \ - do_intrinsic(_weakCompareAndSetLongPlain, jdk_internal_misc_Unsafe, weakCompareAndSetLongPlain_name, compareAndSetLong_signature, F_R) \ - do_intrinsic(_weakCompareAndSetLongAcquire, jdk_internal_misc_Unsafe, weakCompareAndSetLongAcquire_name, compareAndSetLong_signature, F_R) \ - do_intrinsic(_weakCompareAndSetLongRelease, jdk_internal_misc_Unsafe, weakCompareAndSetLongRelease_name, compareAndSetLong_signature, F_R) \ - do_intrinsic(_weakCompareAndSetLong, jdk_internal_misc_Unsafe, weakCompareAndSetLong_name, compareAndSetLong_signature, F_R) \ - do_intrinsic(_weakCompareAndSetIntPlain, jdk_internal_misc_Unsafe, weakCompareAndSetIntPlain_name, compareAndSetInt_signature, F_R) \ - do_intrinsic(_weakCompareAndSetIntAcquire, jdk_internal_misc_Unsafe, weakCompareAndSetIntAcquire_name, compareAndSetInt_signature, F_R) \ - do_intrinsic(_weakCompareAndSetIntRelease, jdk_internal_misc_Unsafe, weakCompareAndSetIntRelease_name, compareAndSetInt_signature, F_R) \ - do_intrinsic(_weakCompareAndSetInt, jdk_internal_misc_Unsafe, weakCompareAndSetInt_name, compareAndSetInt_signature, F_R) \ - do_intrinsic(_weakCompareAndSetBytePlain, jdk_internal_misc_Unsafe, weakCompareAndSetBytePlain_name, compareAndSetByte_signature, F_R) \ - do_intrinsic(_weakCompareAndSetByteAcquire, jdk_internal_misc_Unsafe, weakCompareAndSetByteAcquire_name, compareAndSetByte_signature, F_R) \ - do_intrinsic(_weakCompareAndSetByteRelease, jdk_internal_misc_Unsafe, weakCompareAndSetByteRelease_name, compareAndSetByte_signature, F_R) \ - do_intrinsic(_weakCompareAndSetByte, jdk_internal_misc_Unsafe, weakCompareAndSetByte_name, compareAndSetByte_signature, F_R) \ - do_intrinsic(_weakCompareAndSetShortPlain, jdk_internal_misc_Unsafe, weakCompareAndSetShortPlain_name, compareAndSetShort_signature, F_R) \ - do_intrinsic(_weakCompareAndSetShortAcquire, jdk_internal_misc_Unsafe, weakCompareAndSetShortAcquire_name, compareAndSetShort_signature, F_R) \ - do_intrinsic(_weakCompareAndSetShortRelease, jdk_internal_misc_Unsafe, weakCompareAndSetShortRelease_name, compareAndSetShort_signature, F_R) \ - do_intrinsic(_weakCompareAndSetShort, jdk_internal_misc_Unsafe, weakCompareAndSetShort_name, compareAndSetShort_signature, F_R) \ + do_intrinsic(_weakCompareAndSetReference, jdk_internal_misc_Unsafe, weakCompareAndSetReference_name, compareAndSetReference_signature, F_PW)\ + do_intrinsic(_weakCompareAndSetLong, jdk_internal_misc_Unsafe, weakCompareAndSetLong_name, compareAndSetLong_signature, F_PW)\ + do_intrinsic(_weakCompareAndSetInt, jdk_internal_misc_Unsafe, weakCompareAndSetInt_name, compareAndSetInt_signature, F_PW)\ \ - do_intrinsic(_getAndAddInt, jdk_internal_misc_Unsafe, getAndAddInt_name, getAndAddInt_signature, F_R) \ + do_intrinsic(_getAndAddInt, jdk_internal_misc_Unsafe, getAndAddInt_name, getAndAddInt_signature, F_PW) \ do_name( getAndAddInt_name, "getAndAddInt") \ do_signature(getAndAddInt_signature, "(Ljava/lang/Object;JI)I" ) \ - do_intrinsic(_getAndAddLong, jdk_internal_misc_Unsafe, getAndAddLong_name, getAndAddLong_signature, F_R) \ + do_intrinsic(_getAndAddLong, jdk_internal_misc_Unsafe, getAndAddLong_name, getAndAddLong_signature, F_PW) \ do_name( getAndAddLong_name, "getAndAddLong") \ do_signature(getAndAddLong_signature, "(Ljava/lang/Object;JJ)J" ) \ - do_intrinsic(_getAndAddByte, jdk_internal_misc_Unsafe, getAndAddByte_name, getAndAddByte_signature, F_R) \ + do_intrinsic(_getAndAddByte, jdk_internal_misc_Unsafe, getAndAddByte_name, getAndAddByte_signature, F_PW) \ do_name( getAndAddByte_name, "getAndAddByte") \ do_signature(getAndAddByte_signature, "(Ljava/lang/Object;JB)B" ) \ - do_intrinsic(_getAndAddShort, jdk_internal_misc_Unsafe, getAndAddShort_name, getAndAddShort_signature, F_R) \ + do_intrinsic(_getAndAddShort, jdk_internal_misc_Unsafe, getAndAddShort_name, getAndAddShort_signature, F_PW) \ do_name( getAndAddShort_name, "getAndAddShort") \ do_signature(getAndAddShort_signature, "(Ljava/lang/Object;JS)S" ) \ - do_intrinsic(_getAndSetInt, jdk_internal_misc_Unsafe, getAndSetInt_name, getAndSetInt_signature, F_R) \ + do_intrinsic(_getAndSetInt, jdk_internal_misc_Unsafe, getAndSetInt_name, getAndSetInt_signature, F_PW) \ do_name( getAndSetInt_name, "getAndSetInt") \ do_alias( getAndSetInt_signature, /*"(Ljava/lang/Object;JI)I"*/ getAndAddInt_signature) \ - do_intrinsic(_getAndSetLong, jdk_internal_misc_Unsafe, getAndSetLong_name, getAndSetLong_signature, F_R) \ + do_intrinsic(_getAndSetLong, jdk_internal_misc_Unsafe, getAndSetLong_name, getAndSetLong_signature, F_PW) \ do_name( getAndSetLong_name, "getAndSetLong") \ do_alias( getAndSetLong_signature, /*"(Ljava/lang/Object;JJ)J"*/ getAndAddLong_signature)\ - do_intrinsic(_getAndSetByte, jdk_internal_misc_Unsafe, getAndSetByte_name, getAndSetByte_signature, F_R) \ + do_intrinsic(_getAndSetByte, jdk_internal_misc_Unsafe, getAndSetByte_name, getAndSetByte_signature, F_PW) \ do_name( getAndSetByte_name, "getAndSetByte") \ do_alias( getAndSetByte_signature, /*"(Ljava/lang/Object;JB)B"*/ getAndAddByte_signature)\ - do_intrinsic(_getAndSetShort, jdk_internal_misc_Unsafe, getAndSetShort_name, getAndSetShort_signature, F_R) \ + do_intrinsic(_getAndSetShort, jdk_internal_misc_Unsafe, getAndSetShort_name, getAndSetShort_signature, F_PW) \ do_name( getAndSetShort_name, "getAndSetShort") \ do_alias( getAndSetShort_signature, /*"(Ljava/lang/Object;JS)S"*/ getAndAddShort_signature) \ - do_intrinsic(_getAndSetReference, jdk_internal_misc_Unsafe, getAndSetReference_name, getAndSetReference_signature, F_R) \ + do_intrinsic(_getAndSetReference, jdk_internal_misc_Unsafe, getAndSetReference_name, getAndSetReference_signature, F_PW) \ do_name( getAndSetReference_name, "getAndSetReference") \ do_signature(getAndSetReference_signature, "(Ljava/lang/Object;JLjava/lang/Object;)Ljava/lang/Object;" ) \ + /* end of wrappers for polymorphic unsafe intrinsics */ \ \ /* Float16Math API intrinsification support */ \ /* Float16 signatures */ \ @@ -1495,6 +1389,8 @@ class vmIntrinsics : AllStatic { F_Y, // !static !native synchronized F_RN, // !static native !synchronized F_SN, // static native !synchronized + F_PI, // polymorphic intrinsic + F_PW, // polymorphic intrinsic wrapper (must inline) FLAG_LIMIT }; @@ -1511,6 +1407,9 @@ class vmIntrinsics : AllStatic { case F_Y: case F_RN: return false; + case F_PI: + case F_PW: + return false; // there is no F_SPI or F_SPW default: ShouldNotReachHere(); return false; @@ -1525,6 +1424,8 @@ class vmIntrinsics : AllStatic { case F_SN: case F_S: case F_R: + case F_PI: + case F_PW: return false; default: ShouldNotReachHere(); @@ -1532,7 +1433,7 @@ class vmIntrinsics : AllStatic { } } - static constexpr bool is_flag_native(Flags flags) { + static constexpr TriBool is_flag_native(Flags flags) { switch (flags) { case F_RN: case F_SN: @@ -1540,7 +1441,10 @@ class vmIntrinsics : AllStatic { case F_S: case F_R: case F_Y: + case F_PW: return false; + case F_PI: // we don't care whether a poly-intrinsic is native or not + return TriBool(); default: ShouldNotReachHere(); return false; @@ -1604,18 +1508,16 @@ class vmIntrinsics : AllStatic { "correct static flag: %s", name_at(id)); assert(is_flag_synchronized(flags_for(id)) == ((flags & JVM_ACC_SYNCHRONIZED) != 0), "correct synchronized flag: %s", name_at(id)); - assert( is_flag_native(flags_for(id)) == ((flags & JVM_ACC_NATIVE) != 0), + assert( is_flag_native(flags_for(id)).match((flags & JVM_ACC_NATIVE) != 0), "correct native flag: %s", name_at(id)); return id; } -#ifndef PRODUCT // Find out the symbols behind an intrinsic: static vmSymbolID class_for(ID id); static vmSymbolID name_for(ID id); static vmSymbolID signature_for(ID id); static Flags flags_for(ID id); -#endif static bool class_has_intrinsics(vmSymbolID holder); @@ -1623,11 +1525,115 @@ class vmIntrinsics : AllStatic { // The methods below provide information related to compiling intrinsics. + // Infrastructure for polymorphic intrinsics. + // These are parameterized by constant prefix arguments, + // which act like instruction operation sub-fields. + // For example: + // - A MO argument selects between get-plain, get-volatile, etc. + // - A BT argument selects between get-int, get-byte, etc. + // - An OP argument selects between get-and-add, get-and-xor, etc. + enum PolymorphicPrefix { + PP_NONE = 0, // regular intrinsic -- not parameterized + PP_MO, // getReferenceMO(byte mo, ...) + PP_MO_BT, // getPrimitiveBitsMO(byte mo, byte bt, ...) + PP_MO_BT_OP, // getAndOperatePrimitiveBitsMO(byte mo, byte bt, byte op, ...) + }; + + // The polymorphism enables the intrinsic to expand to a variable + // range of instructions, rather than a single instruction (or + // single sequence). A polymorphic intrinsic may require some + // or all of its prefix arguments to be compile-time constants, + // if it is to be expanded. For example, some code that performs + // cmpxchg for a byte is different from code that does the same + // operation on a long, but both are covered by the same intrinsic. + // The difference would be T_BYTE vs. T_LONG, as a prefix. + static PolymorphicPrefix polymorphic_prefix(vmIntrinsics::ID id); + + // Memory order codes used by polymorphic intrinsics. + // These are also defined as byte constants in the Unsafe class. + // (See also accessDecorators.hpp for a more detailed API.) + #define VMI_MEMORY_ORDERS_DO(fn) \ + fn(MO_PLAIN, 1) \ + fn(MO_VOLATILE, 2) \ + fn(MO_OPAQUE, 3) \ + fn(MO_ACQUIRE, 4) \ + fn(MO_RELEASE, 8) \ + fn(MO_WEAK_CAS, 16) \ + fn(MO_UNALIGNED, 32) \ + /*end*/ + + enum MemoryOrder : int { + MO_NONE = 0, + #define VMI_MEMORY_ORDER_DEFINE(mo, code) mo = code, + VMI_MEMORY_ORDERS_DO(VMI_MEMORY_ORDER_DEFINE) + #undef VMI_MEMORY_ORDER_DEFINE + // There are important derived combinations, + // such as UNALIGNED+PLAIN, or WEAK_CASE+VOLATILE. + // These are not separately listed, but can appar + // as MO arguments to unsafe access primitives. + }; + static constexpr int MO_MODE_MASK = (MO_PLAIN|MO_VOLATILE|MO_ACQUIRE|MO_RELEASE); + static constexpr int MO_EXTRA_BITS_MASK = (MO_WEAK_CAS|MO_UNALIGNED); + + // These are defined as byte constants in the Unsafe class. + // They are used only by Unsafe.getAndOperatePrimitiveBitsMO. + #define VMI_PRIMITIVE_BITS_OPERATIONS_DO(fn) \ + fn(OP_ADD, '+') \ + fn(OP_BITAND, '&') \ + fn(OP_BITOR, '|') \ + fn(OP_BITXOR, '^') \ + fn(OP_SWAP, '=') \ + /*end*/ + + enum BitsOperation { + OP_NONE = 0, + #define VMI_PRIMITIVE_BITS_OPERATION_DEFINE(op, code) op = code, + VMI_PRIMITIVE_BITS_OPERATIONS_DO(VMI_PRIMITIVE_BITS_OPERATION_DEFINE) + #undef VMI_PRIMITIVE_BITS_OPERATION_DEFINE + }; + // This high level of macro-abstraction is intended assists us in + // validating that the constants are the same in Java as in C++. + + // valid mo constant is one of plain, volatile, acquire, or release + static bool is_valid_memory_order(int mo, int optionally_exclude = -1) { + switch (mo) { + case MO_PLAIN: case MO_VOLATILE: + case MO_ACQUIRE: case MO_RELEASE: case MO_OPAQUE: + return (mo != optionally_exclude); + default: + return false; + } + } + static bool is_valid_primitive_type(int bt) { + // cannot use is_java_primitive because bt might be a weirdo like -1 + return T_BOOLEAN <= bt && bt <= T_LONG; + } + static bool is_valid_primitive_bits_op(int op) { + #define VALID_PRIMITIVE_BITS_CASE(ignore, code) \ + case code: return true; + switch (op) { + VMI_PRIMITIVE_BITS_OPERATIONS_DO(VALID_PRIMITIVE_BITS_CASE) + default: + return false; + } + #undef VALID_PRIMITIVE_BITS_CASE + } + // (1) Information needed by the C1 compiler. static bool preserves_state(vmIntrinsics::ID id); static bool can_trap(vmIntrinsics::ID id); static bool should_be_pinned(vmIntrinsics::ID id); + static ID raw_floating_conversion(BasicType from, BasicType to) { + switch (from) { + case T_INT: if (to == T_FLOAT) return _intBitsToFloat; break; + case T_FLOAT: if (to == T_INT) return _floatToRawIntBits; break; + case T_LONG: if (to == T_DOUBLE) return _longBitsToDouble; break; + case T_DOUBLE: if (to == T_LONG) return _doubleToRawLongBits; break; + default: break; + } + return _none; + } // (2) Information needed by the C2 compiler. @@ -1652,6 +1658,14 @@ class vmIntrinsics : AllStatic { // the corresponding coarse-grained control(2) disables it. static bool is_disabled_by_flags(vmIntrinsics::ID id); + // Returns true if (a) the intrinsic is not disabled by flags, + // and (b) it is supported by the current VM version. + // The method AbstractCompiler::is_intrinsic_available wraps + // this logic further, potentially reducing the availability + // by additional checks, (c) whether the particular compiler + // backend supports the intrinsic, and (d) whether compiler + // directives (perhaps method-specific ones) have disabled + // the intrinsic. static bool is_intrinsic_available(vmIntrinsics::ID id); }; diff --git a/src/hotspot/share/classfile/vmSymbols.cpp b/src/hotspot/share/classfile/vmSymbols.cpp index bd32eac4f341b..c77decf1a4e85 100644 --- a/src/hotspot/share/classfile/vmSymbols.cpp +++ b/src/hotspot/share/classfile/vmSymbols.cpp @@ -77,9 +77,9 @@ static const char* vm_symbol_enum_name(vmSymbolID sid) { static const char* vm_symbol_bodies = VM_SYMBOLS_DO(VM_SYMBOL_BODY, VM_ALIAS_IGNORE); void vmSymbols::initialize() { - assert(SID_LIMIT <= (1< (1< (1<(vmIntrinsics::_none), "do this just once"); const uintptr_t max_id_uint = right_n_bits((int)(sizeof(_intrinsic_id) * BitsPerByte)); - assert((uintptr_t)vmIntrinsics::ID_LIMIT <= max_id_uint, "else fix size"); + static_assert((uintptr_t)vmIntrinsics::ID_LIMIT <= max_id_uint, "else fix size"); assert(intrinsic_id_size_in_bytes() == sizeof(_intrinsic_id), ""); // the klass name is well-known: @@ -1734,11 +1734,39 @@ void Method::init_intrinsic_id(vmSymbolID klass_id) { // ditto for method and signature: vmSymbolID name_id = vmSymbols::find_sid(name()); - if (klass_id != VM_SYMBOL_ENUM_NAME(java_lang_invoke_MethodHandle) - && klass_id != VM_SYMBOL_ENUM_NAME(java_lang_invoke_VarHandle) - && name_id == vmSymbolID::NO_SID) { - return; + switch (klass_id) { + case VM_SYMBOL_ENUM_NAME(java_lang_invoke_MethodHandle): + case VM_SYMBOL_ENUM_NAME(java_lang_invoke_VarHandle): + break; // do not worry about the name yet + + case VM_SYMBOL_ENUM_NAME(jdk_internal_misc_Unsafe): + { + if (!is_native()) break; + const char* native_suffix = "Native"; + if (name_id == vmSymbolID::NO_SID && name()->ends_with(native_suffix)) { + // could be an alias for a non-native intrinsic (a JIT fallback) + ResourceMark rm; + int name_len = name()->utf8_length(); + char* name_str = name()->as_utf8(); + TempNewSymbol trial_name = SymbolTable::probe(name_str, name_len - strlen(native_suffix)); + if (trial_name == nullptr) { + break; // no such symbol + } + Method* method = method_holder()->lookup_method(trial_name, signature()); + if (method == nullptr || method->is_native() || method->is_static() != is_static()) { + break; // no such method + } + name_id = vmSymbols::find_sid(trial_name); + } + } + break; + + default: + if (name_id == vmSymbolID::NO_SID) { + return; + } } + vmSymbolID sig_id = vmSymbols::find_sid(signature()); if (klass_id != VM_SYMBOL_ENUM_NAME(java_lang_invoke_MethodHandle) && klass_id != VM_SYMBOL_ENUM_NAME(java_lang_invoke_VarHandle) diff --git a/src/hotspot/share/opto/c2compiler.cpp b/src/hotspot/share/opto/c2compiler.cpp index ead1b78cdea9b..a7eeed85cdd73 100644 --- a/src/hotspot/share/opto/c2compiler.cpp +++ b/src/hotspot/share/opto/c2compiler.cpp @@ -366,7 +366,7 @@ bool C2Compiler::is_intrinsic_supported(vmIntrinsics::ID id) { break; /* CompareAndSet, Object: */ - case vmIntrinsics::_compareAndSetReference: + case vmIntrinsics::_compareAndSetReferenceMO: #ifdef _LP64 if ( UseCompressedOops && !Matcher::match_rule_supported(Op_CompareAndSwapN)) return false; if (!UseCompressedOops && !Matcher::match_rule_supported(Op_CompareAndSwapP)) return false; @@ -374,65 +374,14 @@ bool C2Compiler::is_intrinsic_supported(vmIntrinsics::ID id) { if (!Matcher::match_rule_supported(Op_CompareAndSwapP)) return false; #endif break; - case vmIntrinsics::_weakCompareAndSetReferencePlain: - case vmIntrinsics::_weakCompareAndSetReferenceAcquire: - case vmIntrinsics::_weakCompareAndSetReferenceRelease: - case vmIntrinsics::_weakCompareAndSetReference: -#ifdef _LP64 - if ( UseCompressedOops && !Matcher::match_rule_supported(Op_WeakCompareAndSwapN)) return false; - if (!UseCompressedOops && !Matcher::match_rule_supported(Op_WeakCompareAndSwapP)) return false; -#else - if (!Matcher::match_rule_supported(Op_WeakCompareAndSwapP)) return false; -#endif - break; - /* CompareAndSet, Long: */ - case vmIntrinsics::_compareAndSetLong: - if (!Matcher::match_rule_supported(Op_CompareAndSwapL)) return false; - break; - case vmIntrinsics::_weakCompareAndSetLongPlain: - case vmIntrinsics::_weakCompareAndSetLongAcquire: - case vmIntrinsics::_weakCompareAndSetLongRelease: - case vmIntrinsics::_weakCompareAndSetLong: - if (!Matcher::match_rule_supported(Op_WeakCompareAndSwapL)) return false; - break; - - /* CompareAndSet, Int: */ - case vmIntrinsics::_compareAndSetInt: - if (!Matcher::match_rule_supported(Op_CompareAndSwapI)) return false; - break; - case vmIntrinsics::_weakCompareAndSetIntPlain: - case vmIntrinsics::_weakCompareAndSetIntAcquire: - case vmIntrinsics::_weakCompareAndSetIntRelease: - case vmIntrinsics::_weakCompareAndSetInt: - if (!Matcher::match_rule_supported(Op_WeakCompareAndSwapI)) return false; - break; - - /* CompareAndSet, Byte: */ - case vmIntrinsics::_compareAndSetByte: - if (!Matcher::match_rule_supported(Op_CompareAndSwapB)) return false; - break; - case vmIntrinsics::_weakCompareAndSetBytePlain: - case vmIntrinsics::_weakCompareAndSetByteAcquire: - case vmIntrinsics::_weakCompareAndSetByteRelease: - case vmIntrinsics::_weakCompareAndSetByte: - if (!Matcher::match_rule_supported(Op_WeakCompareAndSwapB)) return false; - break; - - /* CompareAndSet, Short: */ - case vmIntrinsics::_compareAndSetShort: - if (!Matcher::match_rule_supported(Op_CompareAndSwapS)) return false; - break; - case vmIntrinsics::_weakCompareAndSetShortPlain: - case vmIntrinsics::_weakCompareAndSetShortAcquire: - case vmIntrinsics::_weakCompareAndSetShortRelease: - case vmIntrinsics::_weakCompareAndSetShort: - if (!Matcher::match_rule_supported(Op_WeakCompareAndSwapS)) return false; + /* CompareAndSet, Int/Long: */ + case vmIntrinsics::_compareAndSetPrimitiveBitsMO: + if (!(Matcher::match_rule_supported(Op_CompareAndSwapL) || + Matcher::match_rule_supported(Op_CompareAndSwapI))) return false; break; /* CompareAndExchange, Object: */ - case vmIntrinsics::_compareAndExchangeReference: - case vmIntrinsics::_compareAndExchangeReferenceAcquire: - case vmIntrinsics::_compareAndExchangeReferenceRelease: + case vmIntrinsics::_compareAndExchangeReferenceMO: #ifdef _LP64 if ( UseCompressedOops && !Matcher::match_rule_supported(Op_CompareAndExchangeN)) return false; if (!UseCompressedOops && !Matcher::match_rule_supported(Op_CompareAndExchangeP)) return false; @@ -441,60 +390,20 @@ bool C2Compiler::is_intrinsic_supported(vmIntrinsics::ID id) { #endif break; - /* CompareAndExchange, Long: */ - case vmIntrinsics::_compareAndExchangeLong: - case vmIntrinsics::_compareAndExchangeLongAcquire: - case vmIntrinsics::_compareAndExchangeLongRelease: - if (!Matcher::match_rule_supported(Op_CompareAndExchangeL)) return false; + /* CompareAndExchange, Int/Long: */ + case vmIntrinsics::_compareAndExchangePrimitiveBitsMO: + if (!(Matcher::match_rule_supported(Op_CompareAndExchangeL) || + Matcher::match_rule_supported(Op_CompareAndExchangeI))) return false; break; - /* CompareAndExchange, Int: */ - case vmIntrinsics::_compareAndExchangeInt: - case vmIntrinsics::_compareAndExchangeIntAcquire: - case vmIntrinsics::_compareAndExchangeIntRelease: - if (!Matcher::match_rule_supported(Op_CompareAndExchangeI)) return false; - break; - - /* CompareAndExchange, Byte: */ - case vmIntrinsics::_compareAndExchangeByte: - case vmIntrinsics::_compareAndExchangeByteAcquire: - case vmIntrinsics::_compareAndExchangeByteRelease: - if (!Matcher::match_rule_supported(Op_CompareAndExchangeB)) return false; - break; - - /* CompareAndExchange, Short: */ - case vmIntrinsics::_compareAndExchangeShort: - case vmIntrinsics::_compareAndExchangeShortAcquire: - case vmIntrinsics::_compareAndExchangeShortRelease: - if (!Matcher::match_rule_supported(Op_CompareAndExchangeS)) return false; - break; - - case vmIntrinsics::_getAndAddByte: - if (!Matcher::match_rule_supported(Op_GetAndAddB)) return false; - break; - case vmIntrinsics::_getAndAddShort: - if (!Matcher::match_rule_supported(Op_GetAndAddS)) return false; - break; - case vmIntrinsics::_getAndAddInt: - if (!Matcher::match_rule_supported(Op_GetAndAddI)) return false; - break; - case vmIntrinsics::_getAndAddLong: - if (!Matcher::match_rule_supported(Op_GetAndAddL)) return false; - break; - - case vmIntrinsics::_getAndSetByte: - if (!Matcher::match_rule_supported(Op_GetAndSetB)) return false; - break; - case vmIntrinsics::_getAndSetShort: - if (!Matcher::match_rule_supported(Op_GetAndSetS)) return false; - break; - case vmIntrinsics::_getAndSetInt: - if (!Matcher::match_rule_supported(Op_GetAndSetI)) return false; - break; - case vmIntrinsics::_getAndSetLong: - if (!Matcher::match_rule_supported(Op_GetAndSetL)) return false; + /* GetAndSet/GetAndAdd, Int/Long: */ + case vmIntrinsics::_getAndOperatePrimitiveBitsMO: + if (!(Matcher::match_rule_supported(Op_GetAndAddI) || + Matcher::match_rule_supported(Op_GetAndAddL) || + Matcher::match_rule_supported(Op_GetAndSetI) || + Matcher::match_rule_supported(Op_GetAndSetL))) return false; break; - case vmIntrinsics::_getAndSetReference: + case vmIntrinsics::_getAndSetReferenceMO: #ifdef _LP64 if (!UseCompressedOops && !Matcher::match_rule_supported(Op_GetAndSetP)) return false; if (UseCompressedOops && !Matcher::match_rule_supported(Op_GetAndSetN)) return false; @@ -658,86 +567,10 @@ bool C2Compiler::is_intrinsic_supported(vmIntrinsics::ID id) { case vmIntrinsics::_getCharsStringU: case vmIntrinsics::_getCharStringU: case vmIntrinsics::_putCharStringU: - case vmIntrinsics::_getReference: - case vmIntrinsics::_getBoolean: - case vmIntrinsics::_getByte: - case vmIntrinsics::_getShort: - case vmIntrinsics::_getChar: - case vmIntrinsics::_getInt: - case vmIntrinsics::_getLong: - case vmIntrinsics::_getFloat: - case vmIntrinsics::_getDouble: - case vmIntrinsics::_putReference: - case vmIntrinsics::_putBoolean: - case vmIntrinsics::_putByte: - case vmIntrinsics::_putShort: - case vmIntrinsics::_putChar: - case vmIntrinsics::_putInt: - case vmIntrinsics::_putLong: - case vmIntrinsics::_putFloat: - case vmIntrinsics::_putDouble: - case vmIntrinsics::_getReferenceVolatile: - case vmIntrinsics::_getBooleanVolatile: - case vmIntrinsics::_getByteVolatile: - case vmIntrinsics::_getShortVolatile: - case vmIntrinsics::_getCharVolatile: - case vmIntrinsics::_getIntVolatile: - case vmIntrinsics::_getLongVolatile: - case vmIntrinsics::_getFloatVolatile: - case vmIntrinsics::_getDoubleVolatile: - case vmIntrinsics::_putReferenceVolatile: - case vmIntrinsics::_putBooleanVolatile: - case vmIntrinsics::_putByteVolatile: - case vmIntrinsics::_putShortVolatile: - case vmIntrinsics::_putCharVolatile: - case vmIntrinsics::_putIntVolatile: - case vmIntrinsics::_putLongVolatile: - case vmIntrinsics::_putFloatVolatile: - case vmIntrinsics::_putDoubleVolatile: - case vmIntrinsics::_getReferenceAcquire: - case vmIntrinsics::_getBooleanAcquire: - case vmIntrinsics::_getByteAcquire: - case vmIntrinsics::_getShortAcquire: - case vmIntrinsics::_getCharAcquire: - case vmIntrinsics::_getIntAcquire: - case vmIntrinsics::_getLongAcquire: - case vmIntrinsics::_getFloatAcquire: - case vmIntrinsics::_getDoubleAcquire: - case vmIntrinsics::_putReferenceRelease: - case vmIntrinsics::_putBooleanRelease: - case vmIntrinsics::_putByteRelease: - case vmIntrinsics::_putShortRelease: - case vmIntrinsics::_putCharRelease: - case vmIntrinsics::_putIntRelease: - case vmIntrinsics::_putLongRelease: - case vmIntrinsics::_putFloatRelease: - case vmIntrinsics::_putDoubleRelease: - case vmIntrinsics::_getReferenceOpaque: - case vmIntrinsics::_getBooleanOpaque: - case vmIntrinsics::_getByteOpaque: - case vmIntrinsics::_getShortOpaque: - case vmIntrinsics::_getCharOpaque: - case vmIntrinsics::_getIntOpaque: - case vmIntrinsics::_getLongOpaque: - case vmIntrinsics::_getFloatOpaque: - case vmIntrinsics::_getDoubleOpaque: - case vmIntrinsics::_putReferenceOpaque: - case vmIntrinsics::_putBooleanOpaque: - case vmIntrinsics::_putByteOpaque: - case vmIntrinsics::_putShortOpaque: - case vmIntrinsics::_putCharOpaque: - case vmIntrinsics::_putIntOpaque: - case vmIntrinsics::_putLongOpaque: - case vmIntrinsics::_putFloatOpaque: - case vmIntrinsics::_putDoubleOpaque: - case vmIntrinsics::_getShortUnaligned: - case vmIntrinsics::_getCharUnaligned: - case vmIntrinsics::_getIntUnaligned: - case vmIntrinsics::_getLongUnaligned: - case vmIntrinsics::_putShortUnaligned: - case vmIntrinsics::_putCharUnaligned: - case vmIntrinsics::_putIntUnaligned: - case vmIntrinsics::_putLongUnaligned: + case vmIntrinsics::_getReferenceMO: + case vmIntrinsics::_putReferenceMO: + case vmIntrinsics::_getPrimitiveBitsMO: + case vmIntrinsics::_putPrimitiveBitsMO: case vmIntrinsics::_loadFence: case vmIntrinsics::_storeFence: case vmIntrinsics::_storeStoreFence: diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp index 2263fa720ce3f..c010875aa804a 100644 --- a/src/hotspot/share/opto/library_call.cpp +++ b/src/hotspot/share/opto/library_call.cpp @@ -235,7 +235,52 @@ bool LibraryCallKit::try_to_inline(int predicate) { } assert(merged_memory(), ""); - switch (intrinsic_id()) { + vmIntrinsics::ID id = intrinsic_id(); + + vmIntrinsics::PolymorphicPrefix pfx = vmIntrinsics::polymorphic_prefix(id); + // possible decoded prefix values: + int prefix_size = 0; + vmIntrinsics::MemoryOrder mo = vmIntrinsics::MO_NONE; + BasicType bt = T_ILLEGAL; + vmIntrinsics::BitsOperation op = vmIntrinsics::OP_NONE; + if (pfx != vmIntrinsics::PP_NONE) { + // Decode some leading constant arguments... + const int prefix_base = callee()->is_static() ? 0 : 1; + int next_arg = prefix_base; + jint moc = -1; // mo should be MO_PLAIN or the like; use MO_VOLATILE if not constant + jint btc = -1; // bt should be T_BYTE or the like; bail if not constant + jint opc = -1; // op should be OP_SWAP or the like; bail if not constant + switch (pfx) { + case vmIntrinsics::PP_MO: + moc = argument(next_arg++)->find_int_con(-1); + bt = T_OBJECT; // getReferenceMO etc. + break; + case vmIntrinsics::PP_MO_BT: + moc = argument(next_arg++)->find_int_con(-1); + btc = argument(next_arg++)->find_int_con(-1); + if (!vmIntrinsics::is_valid_primitive_type(btc)) return false; + bt = (BasicType)btc; + break; + case vmIntrinsics::PP_MO_BT_OP: + moc = argument(next_arg++)->find_int_con(-1); + btc = argument(next_arg++)->find_int_con(-1); + opc = argument(next_arg++)->find_int_con(-1); + if (!vmIntrinsics::is_valid_primitive_type(btc)) return false; + if (!vmIntrinsics::is_valid_primitive_bits_op(opc)) return false; + bt = (BasicType)btc; + op = (vmIntrinsics::BitsOperation)opc; + break; + default: + ShouldNotReachHere(); + } + if (!vmIntrinsics::is_valid_memory_order(moc & ~vmIntrinsics::MO_EXTRA_BITS_MASK)) + moc = vmIntrinsics::MO_VOLATILE; + mo = (vmIntrinsics::MemoryOrder)moc; + prefix_size = next_arg - prefix_base; + } + // done examining prefix; components are present for intrinsics that need them + + switch (id) { case vmIntrinsics::_hashCode: return inline_native_hashcode(intrinsic()->is_virtual(), !is_static); case vmIntrinsics::_identityHashCode: return inline_native_hashcode(/*!virtual*/ false, is_static); case vmIntrinsics::_getClass: return inline_native_getClass(); @@ -319,149 +364,19 @@ bool LibraryCallKit::try_to_inline(int predicate) { case vmIntrinsics::_inflateStringC: case vmIntrinsics::_inflateStringB: return inline_string_copy(!is_compress); - case vmIntrinsics::_getReference: return inline_unsafe_access(!is_store, T_OBJECT, Relaxed, false); - case vmIntrinsics::_getBoolean: return inline_unsafe_access(!is_store, T_BOOLEAN, Relaxed, false); - case vmIntrinsics::_getByte: return inline_unsafe_access(!is_store, T_BYTE, Relaxed, false); - case vmIntrinsics::_getShort: return inline_unsafe_access(!is_store, T_SHORT, Relaxed, false); - case vmIntrinsics::_getChar: return inline_unsafe_access(!is_store, T_CHAR, Relaxed, false); - case vmIntrinsics::_getInt: return inline_unsafe_access(!is_store, T_INT, Relaxed, false); - case vmIntrinsics::_getLong: return inline_unsafe_access(!is_store, T_LONG, Relaxed, false); - case vmIntrinsics::_getFloat: return inline_unsafe_access(!is_store, T_FLOAT, Relaxed, false); - case vmIntrinsics::_getDouble: return inline_unsafe_access(!is_store, T_DOUBLE, Relaxed, false); - - case vmIntrinsics::_putReference: return inline_unsafe_access( is_store, T_OBJECT, Relaxed, false); - case vmIntrinsics::_putBoolean: return inline_unsafe_access( is_store, T_BOOLEAN, Relaxed, false); - case vmIntrinsics::_putByte: return inline_unsafe_access( is_store, T_BYTE, Relaxed, false); - case vmIntrinsics::_putShort: return inline_unsafe_access( is_store, T_SHORT, Relaxed, false); - case vmIntrinsics::_putChar: return inline_unsafe_access( is_store, T_CHAR, Relaxed, false); - case vmIntrinsics::_putInt: return inline_unsafe_access( is_store, T_INT, Relaxed, false); - case vmIntrinsics::_putLong: return inline_unsafe_access( is_store, T_LONG, Relaxed, false); - case vmIntrinsics::_putFloat: return inline_unsafe_access( is_store, T_FLOAT, Relaxed, false); - case vmIntrinsics::_putDouble: return inline_unsafe_access( is_store, T_DOUBLE, Relaxed, false); - - case vmIntrinsics::_getReferenceVolatile: return inline_unsafe_access(!is_store, T_OBJECT, Volatile, false); - case vmIntrinsics::_getBooleanVolatile: return inline_unsafe_access(!is_store, T_BOOLEAN, Volatile, false); - case vmIntrinsics::_getByteVolatile: return inline_unsafe_access(!is_store, T_BYTE, Volatile, false); - case vmIntrinsics::_getShortVolatile: return inline_unsafe_access(!is_store, T_SHORT, Volatile, false); - case vmIntrinsics::_getCharVolatile: return inline_unsafe_access(!is_store, T_CHAR, Volatile, false); - case vmIntrinsics::_getIntVolatile: return inline_unsafe_access(!is_store, T_INT, Volatile, false); - case vmIntrinsics::_getLongVolatile: return inline_unsafe_access(!is_store, T_LONG, Volatile, false); - case vmIntrinsics::_getFloatVolatile: return inline_unsafe_access(!is_store, T_FLOAT, Volatile, false); - case vmIntrinsics::_getDoubleVolatile: return inline_unsafe_access(!is_store, T_DOUBLE, Volatile, false); - - case vmIntrinsics::_putReferenceVolatile: return inline_unsafe_access( is_store, T_OBJECT, Volatile, false); - case vmIntrinsics::_putBooleanVolatile: return inline_unsafe_access( is_store, T_BOOLEAN, Volatile, false); - case vmIntrinsics::_putByteVolatile: return inline_unsafe_access( is_store, T_BYTE, Volatile, false); - case vmIntrinsics::_putShortVolatile: return inline_unsafe_access( is_store, T_SHORT, Volatile, false); - case vmIntrinsics::_putCharVolatile: return inline_unsafe_access( is_store, T_CHAR, Volatile, false); - case vmIntrinsics::_putIntVolatile: return inline_unsafe_access( is_store, T_INT, Volatile, false); - case vmIntrinsics::_putLongVolatile: return inline_unsafe_access( is_store, T_LONG, Volatile, false); - case vmIntrinsics::_putFloatVolatile: return inline_unsafe_access( is_store, T_FLOAT, Volatile, false); - case vmIntrinsics::_putDoubleVolatile: return inline_unsafe_access( is_store, T_DOUBLE, Volatile, false); - - case vmIntrinsics::_getShortUnaligned: return inline_unsafe_access(!is_store, T_SHORT, Relaxed, true); - case vmIntrinsics::_getCharUnaligned: return inline_unsafe_access(!is_store, T_CHAR, Relaxed, true); - case vmIntrinsics::_getIntUnaligned: return inline_unsafe_access(!is_store, T_INT, Relaxed, true); - case vmIntrinsics::_getLongUnaligned: return inline_unsafe_access(!is_store, T_LONG, Relaxed, true); - - case vmIntrinsics::_putShortUnaligned: return inline_unsafe_access( is_store, T_SHORT, Relaxed, true); - case vmIntrinsics::_putCharUnaligned: return inline_unsafe_access( is_store, T_CHAR, Relaxed, true); - case vmIntrinsics::_putIntUnaligned: return inline_unsafe_access( is_store, T_INT, Relaxed, true); - case vmIntrinsics::_putLongUnaligned: return inline_unsafe_access( is_store, T_LONG, Relaxed, true); - - case vmIntrinsics::_getReferenceAcquire: return inline_unsafe_access(!is_store, T_OBJECT, Acquire, false); - case vmIntrinsics::_getBooleanAcquire: return inline_unsafe_access(!is_store, T_BOOLEAN, Acquire, false); - case vmIntrinsics::_getByteAcquire: return inline_unsafe_access(!is_store, T_BYTE, Acquire, false); - case vmIntrinsics::_getShortAcquire: return inline_unsafe_access(!is_store, T_SHORT, Acquire, false); - case vmIntrinsics::_getCharAcquire: return inline_unsafe_access(!is_store, T_CHAR, Acquire, false); - case vmIntrinsics::_getIntAcquire: return inline_unsafe_access(!is_store, T_INT, Acquire, false); - case vmIntrinsics::_getLongAcquire: return inline_unsafe_access(!is_store, T_LONG, Acquire, false); - case vmIntrinsics::_getFloatAcquire: return inline_unsafe_access(!is_store, T_FLOAT, Acquire, false); - case vmIntrinsics::_getDoubleAcquire: return inline_unsafe_access(!is_store, T_DOUBLE, Acquire, false); - - case vmIntrinsics::_putReferenceRelease: return inline_unsafe_access( is_store, T_OBJECT, Release, false); - case vmIntrinsics::_putBooleanRelease: return inline_unsafe_access( is_store, T_BOOLEAN, Release, false); - case vmIntrinsics::_putByteRelease: return inline_unsafe_access( is_store, T_BYTE, Release, false); - case vmIntrinsics::_putShortRelease: return inline_unsafe_access( is_store, T_SHORT, Release, false); - case vmIntrinsics::_putCharRelease: return inline_unsafe_access( is_store, T_CHAR, Release, false); - case vmIntrinsics::_putIntRelease: return inline_unsafe_access( is_store, T_INT, Release, false); - case vmIntrinsics::_putLongRelease: return inline_unsafe_access( is_store, T_LONG, Release, false); - case vmIntrinsics::_putFloatRelease: return inline_unsafe_access( is_store, T_FLOAT, Release, false); - case vmIntrinsics::_putDoubleRelease: return inline_unsafe_access( is_store, T_DOUBLE, Release, false); - - case vmIntrinsics::_getReferenceOpaque: return inline_unsafe_access(!is_store, T_OBJECT, Opaque, false); - case vmIntrinsics::_getBooleanOpaque: return inline_unsafe_access(!is_store, T_BOOLEAN, Opaque, false); - case vmIntrinsics::_getByteOpaque: return inline_unsafe_access(!is_store, T_BYTE, Opaque, false); - case vmIntrinsics::_getShortOpaque: return inline_unsafe_access(!is_store, T_SHORT, Opaque, false); - case vmIntrinsics::_getCharOpaque: return inline_unsafe_access(!is_store, T_CHAR, Opaque, false); - case vmIntrinsics::_getIntOpaque: return inline_unsafe_access(!is_store, T_INT, Opaque, false); - case vmIntrinsics::_getLongOpaque: return inline_unsafe_access(!is_store, T_LONG, Opaque, false); - case vmIntrinsics::_getFloatOpaque: return inline_unsafe_access(!is_store, T_FLOAT, Opaque, false); - case vmIntrinsics::_getDoubleOpaque: return inline_unsafe_access(!is_store, T_DOUBLE, Opaque, false); - - case vmIntrinsics::_putReferenceOpaque: return inline_unsafe_access( is_store, T_OBJECT, Opaque, false); - case vmIntrinsics::_putBooleanOpaque: return inline_unsafe_access( is_store, T_BOOLEAN, Opaque, false); - case vmIntrinsics::_putByteOpaque: return inline_unsafe_access( is_store, T_BYTE, Opaque, false); - case vmIntrinsics::_putShortOpaque: return inline_unsafe_access( is_store, T_SHORT, Opaque, false); - case vmIntrinsics::_putCharOpaque: return inline_unsafe_access( is_store, T_CHAR, Opaque, false); - case vmIntrinsics::_putIntOpaque: return inline_unsafe_access( is_store, T_INT, Opaque, false); - case vmIntrinsics::_putLongOpaque: return inline_unsafe_access( is_store, T_LONG, Opaque, false); - case vmIntrinsics::_putFloatOpaque: return inline_unsafe_access( is_store, T_FLOAT, Opaque, false); - case vmIntrinsics::_putDoubleOpaque: return inline_unsafe_access( is_store, T_DOUBLE, Opaque, false); - - case vmIntrinsics::_compareAndSetReference: return inline_unsafe_load_store(T_OBJECT, LS_cmp_swap, Volatile); - case vmIntrinsics::_compareAndSetByte: return inline_unsafe_load_store(T_BYTE, LS_cmp_swap, Volatile); - case vmIntrinsics::_compareAndSetShort: return inline_unsafe_load_store(T_SHORT, LS_cmp_swap, Volatile); - case vmIntrinsics::_compareAndSetInt: return inline_unsafe_load_store(T_INT, LS_cmp_swap, Volatile); - case vmIntrinsics::_compareAndSetLong: return inline_unsafe_load_store(T_LONG, LS_cmp_swap, Volatile); - - case vmIntrinsics::_weakCompareAndSetReferencePlain: return inline_unsafe_load_store(T_OBJECT, LS_cmp_swap_weak, Relaxed); - case vmIntrinsics::_weakCompareAndSetReferenceAcquire: return inline_unsafe_load_store(T_OBJECT, LS_cmp_swap_weak, Acquire); - case vmIntrinsics::_weakCompareAndSetReferenceRelease: return inline_unsafe_load_store(T_OBJECT, LS_cmp_swap_weak, Release); - case vmIntrinsics::_weakCompareAndSetReference: return inline_unsafe_load_store(T_OBJECT, LS_cmp_swap_weak, Volatile); - case vmIntrinsics::_weakCompareAndSetBytePlain: return inline_unsafe_load_store(T_BYTE, LS_cmp_swap_weak, Relaxed); - case vmIntrinsics::_weakCompareAndSetByteAcquire: return inline_unsafe_load_store(T_BYTE, LS_cmp_swap_weak, Acquire); - case vmIntrinsics::_weakCompareAndSetByteRelease: return inline_unsafe_load_store(T_BYTE, LS_cmp_swap_weak, Release); - case vmIntrinsics::_weakCompareAndSetByte: return inline_unsafe_load_store(T_BYTE, LS_cmp_swap_weak, Volatile); - case vmIntrinsics::_weakCompareAndSetShortPlain: return inline_unsafe_load_store(T_SHORT, LS_cmp_swap_weak, Relaxed); - case vmIntrinsics::_weakCompareAndSetShortAcquire: return inline_unsafe_load_store(T_SHORT, LS_cmp_swap_weak, Acquire); - case vmIntrinsics::_weakCompareAndSetShortRelease: return inline_unsafe_load_store(T_SHORT, LS_cmp_swap_weak, Release); - case vmIntrinsics::_weakCompareAndSetShort: return inline_unsafe_load_store(T_SHORT, LS_cmp_swap_weak, Volatile); - case vmIntrinsics::_weakCompareAndSetIntPlain: return inline_unsafe_load_store(T_INT, LS_cmp_swap_weak, Relaxed); - case vmIntrinsics::_weakCompareAndSetIntAcquire: return inline_unsafe_load_store(T_INT, LS_cmp_swap_weak, Acquire); - case vmIntrinsics::_weakCompareAndSetIntRelease: return inline_unsafe_load_store(T_INT, LS_cmp_swap_weak, Release); - case vmIntrinsics::_weakCompareAndSetInt: return inline_unsafe_load_store(T_INT, LS_cmp_swap_weak, Volatile); - case vmIntrinsics::_weakCompareAndSetLongPlain: return inline_unsafe_load_store(T_LONG, LS_cmp_swap_weak, Relaxed); - case vmIntrinsics::_weakCompareAndSetLongAcquire: return inline_unsafe_load_store(T_LONG, LS_cmp_swap_weak, Acquire); - case vmIntrinsics::_weakCompareAndSetLongRelease: return inline_unsafe_load_store(T_LONG, LS_cmp_swap_weak, Release); - case vmIntrinsics::_weakCompareAndSetLong: return inline_unsafe_load_store(T_LONG, LS_cmp_swap_weak, Volatile); - - case vmIntrinsics::_compareAndExchangeReference: return inline_unsafe_load_store(T_OBJECT, LS_cmp_exchange, Volatile); - case vmIntrinsics::_compareAndExchangeReferenceAcquire: return inline_unsafe_load_store(T_OBJECT, LS_cmp_exchange, Acquire); - case vmIntrinsics::_compareAndExchangeReferenceRelease: return inline_unsafe_load_store(T_OBJECT, LS_cmp_exchange, Release); - case vmIntrinsics::_compareAndExchangeByte: return inline_unsafe_load_store(T_BYTE, LS_cmp_exchange, Volatile); - case vmIntrinsics::_compareAndExchangeByteAcquire: return inline_unsafe_load_store(T_BYTE, LS_cmp_exchange, Acquire); - case vmIntrinsics::_compareAndExchangeByteRelease: return inline_unsafe_load_store(T_BYTE, LS_cmp_exchange, Release); - case vmIntrinsics::_compareAndExchangeShort: return inline_unsafe_load_store(T_SHORT, LS_cmp_exchange, Volatile); - case vmIntrinsics::_compareAndExchangeShortAcquire: return inline_unsafe_load_store(T_SHORT, LS_cmp_exchange, Acquire); - case vmIntrinsics::_compareAndExchangeShortRelease: return inline_unsafe_load_store(T_SHORT, LS_cmp_exchange, Release); - case vmIntrinsics::_compareAndExchangeInt: return inline_unsafe_load_store(T_INT, LS_cmp_exchange, Volatile); - case vmIntrinsics::_compareAndExchangeIntAcquire: return inline_unsafe_load_store(T_INT, LS_cmp_exchange, Acquire); - case vmIntrinsics::_compareAndExchangeIntRelease: return inline_unsafe_load_store(T_INT, LS_cmp_exchange, Release); - case vmIntrinsics::_compareAndExchangeLong: return inline_unsafe_load_store(T_LONG, LS_cmp_exchange, Volatile); - case vmIntrinsics::_compareAndExchangeLongAcquire: return inline_unsafe_load_store(T_LONG, LS_cmp_exchange, Acquire); - case vmIntrinsics::_compareAndExchangeLongRelease: return inline_unsafe_load_store(T_LONG, LS_cmp_exchange, Release); - - case vmIntrinsics::_getAndAddByte: return inline_unsafe_load_store(T_BYTE, LS_get_add, Volatile); - case vmIntrinsics::_getAndAddShort: return inline_unsafe_load_store(T_SHORT, LS_get_add, Volatile); - case vmIntrinsics::_getAndAddInt: return inline_unsafe_load_store(T_INT, LS_get_add, Volatile); - case vmIntrinsics::_getAndAddLong: return inline_unsafe_load_store(T_LONG, LS_get_add, Volatile); - - case vmIntrinsics::_getAndSetByte: return inline_unsafe_load_store(T_BYTE, LS_get_set, Volatile); - case vmIntrinsics::_getAndSetShort: return inline_unsafe_load_store(T_SHORT, LS_get_set, Volatile); - case vmIntrinsics::_getAndSetInt: return inline_unsafe_load_store(T_INT, LS_get_set, Volatile); - case vmIntrinsics::_getAndSetLong: return inline_unsafe_load_store(T_LONG, LS_get_set, Volatile); - case vmIntrinsics::_getAndSetReference: return inline_unsafe_load_store(T_OBJECT, LS_get_set, Volatile); + case vmIntrinsics::_getReferenceMO: return inline_unsafe_access(!is_store, mo, bt, prefix_size); + case vmIntrinsics::_getPrimitiveBitsMO: return inline_unsafe_access(!is_store, mo, bt, prefix_size); + case vmIntrinsics::_putReferenceMO: return inline_unsafe_access( is_store, mo, bt, prefix_size); + case vmIntrinsics::_putPrimitiveBitsMO: return inline_unsafe_access( is_store, mo, bt, prefix_size); + + case vmIntrinsics::_compareAndSetReferenceMO: return inline_unsafe_load_store(LS_cmp_swap, mo, bt, op, prefix_size); + case vmIntrinsics::_compareAndSetPrimitiveBitsMO: return inline_unsafe_load_store(LS_cmp_swap, mo, bt, op, prefix_size); + + case vmIntrinsics::_compareAndExchangeReferenceMO: return inline_unsafe_load_store(LS_cmp_exchange, mo, bt, op, prefix_size); + case vmIntrinsics::_compareAndExchangePrimitiveBitsMO:return inline_unsafe_load_store(LS_cmp_exchange, mo, bt, op, prefix_size); + + case vmIntrinsics::_getAndSetReferenceMO: return inline_unsafe_load_store(LS_get_set, mo, bt, op, prefix_size); + case vmIntrinsics::_getAndOperatePrimitiveBitsMO: return inline_unsafe_load_store(LS_get_set, mo, bt, op, prefix_size); case vmIntrinsics::_loadFence: case vmIntrinsics::_storeFence: @@ -2370,17 +2285,17 @@ const TypeOopPtr* LibraryCallKit::sharpen_unsafe_type(Compile::AliasType* alias_ return result; } -DecoratorSet LibraryCallKit::mo_decorator_for_access_kind(AccessKind kind) { - switch (kind) { - case Relaxed: - return MO_UNORDERED; - case Opaque: - return MO_RELAXED; - case Acquire: +DecoratorSet LibraryCallKit::mo_decorator_for_access_kind(vmIntrinsics::MemoryOrder mo) { + switch (mo) { + case vmIntrinsics::MO_PLAIN: + return MO_UNORDERED; //meaning: ops may be reordered by both HW and JIT + case vmIntrinsics::MO_OPAQUE: + return MO_RELAXED; //meaning: ops may be reordered by HW but not JIT + case vmIntrinsics::MO_ACQUIRE: return MO_ACQUIRE; - case Release: + case vmIntrinsics::MO_RELEASE: return MO_RELEASE; - case Volatile: + case vmIntrinsics::MO_VOLATILE: return MO_SEQ_CST; default: ShouldNotReachHere(); @@ -2429,13 +2344,21 @@ void LibraryCallKit::SavedState::discard() { _discarded = true; } -bool LibraryCallKit::inline_unsafe_access(bool is_store, const BasicType type, const AccessKind kind, const bool unaligned) { +bool LibraryCallKit::inline_unsafe_access(bool is_store, + vmIntrinsics::MemoryOrder mo, + BasicType type, + int prefix_size) { if (callee()->is_static()) return false; // caller must have the capability! + const bool unaligned = (mo & vmIntrinsics::MO_UNALIGNED) != 0; + mo = (vmIntrinsics::MemoryOrder)(mo & vmIntrinsics::MO_MODE_MASK); DecoratorSet decorators = C2_UNSAFE_ACCESS; - guarantee(!is_store || kind != Acquire, "Acquire accesses can be produced only for loads"); - guarantee( is_store || kind != Release, "Release accesses can be produced only for stores"); + guarantee(!is_store || mo != vmIntrinsics::MO_ACQUIRE, "Acquire accesses can be produced only for loads"); + guarantee( is_store || mo != vmIntrinsics::MO_RELEASE, "Release accesses can be produced only for stores"); assert(type != T_OBJECT || !unaligned, "unaligned access not supported with object type"); + BasicType utype = type == T_OBJECT ? T_OBJECT : T_LONG; + const bool type_is_narrow_int = type != T_OBJECT && !is_double_word_type(type); + if (is_reference_type(type)) { decorators |= ON_UNKNOWN_OOP_REF; } @@ -2450,21 +2373,25 @@ bool LibraryCallKit::inline_unsafe_access(bool is_store, const BasicType type, c // Check the signatures. ciSignature* sig = callee()->signature(); #ifdef ASSERT + for (int i = 0; i < prefix_size; i++) + assert(sig->type_at(i)->basic_type() == T_BYTE, "prefix args are bytes"); if (!is_store) { - // Object getReference(Object base, int/long offset), etc. + // Object getReferenceMO(byte mo, Object base, int/long offset) + // long getPrimitiveBitsMO(byte mo, byte bt, Object base, long offset) BasicType rtype = sig->return_type()->basic_type(); - assert(rtype == type, "getter must return the expected value"); - assert(sig->count() == 2, "oop getter has 2 arguments"); - assert(sig->type_at(0)->basic_type() == T_OBJECT, "getter base is object"); - assert(sig->type_at(1)->basic_type() == T_LONG, "getter offset is correct"); + assert(rtype == utype, "getter must return the expected value"); + assert(sig->count() == prefix_size+2, "oop getter has 2 arguments"); + assert(sig->type_at(prefix_size+0)->basic_type() == T_OBJECT, "getter base is object"); + assert(sig->type_at(prefix_size+1)->basic_type() == T_LONG, "getter offset is correct"); } else { - // void putReference(Object base, int/long offset, Object x), etc. + // void putReference(Object base, int/long offset, Object x) + // void putPrimitiveBitsMO(byte mo, byte bt, Object base, long offset, long x) assert(sig->return_type()->basic_type() == T_VOID, "putter must not return a value"); - assert(sig->count() == 3, "oop putter has 3 arguments"); - assert(sig->type_at(0)->basic_type() == T_OBJECT, "putter base is object"); - assert(sig->type_at(1)->basic_type() == T_LONG, "putter offset is correct"); + assert(sig->count() == prefix_size+3, "oop putter has 3 arguments"); + assert(sig->type_at(prefix_size+0)->basic_type() == T_OBJECT, "putter base is object"); + assert(sig->type_at(prefix_size+1)->basic_type() == T_LONG, "putter offset is correct"); BasicType vtype = sig->type_at(sig->count()-1)->basic_type(); - assert(vtype == type, "putter must accept the expected value"); + assert(vtype == utype, "putter must accept the expected value"); } #endif // ASSERT } @@ -2478,9 +2405,9 @@ bool LibraryCallKit::inline_unsafe_access(bool is_store, const BasicType type, c Node* heap_base_oop = top(); // The base is either a Java object or a value produced by Unsafe.staticFieldBase - Node* base = argument(1); // type: oop + Node* base = argument(prefix_size+1); // type: oop // The offset is a value produced by Unsafe.staticFieldOffset or Unsafe.objectFieldOffset - Node* offset = argument(2); // type: long + Node* offset = argument(prefix_size+2); // type: long // We currently rely on the cookies produced by Unsafe.xxxFieldOffset // to be plain byte offsets, which are also the same as those accepted // by oopDesc::field_addr. @@ -2492,7 +2419,7 @@ bool LibraryCallKit::inline_unsafe_access(bool is_store, const BasicType type, c // Save state and restore on bailout SavedState old_state(this); - Node* adr = make_unsafe_address(base, offset, type, kind == Relaxed); + Node* adr = make_unsafe_address(base, offset, type, mo == vmIntrinsics::MO_PLAIN); assert(!stopped(), "Inlining of unsafe access failed: address construction stopped unexpectedly"); if (_gvn.type(base->uncast())->isa_ptr() == TypePtr::NULL_PTR) { @@ -2512,7 +2439,18 @@ bool LibraryCallKit::inline_unsafe_access(bool is_store, const BasicType type, c decorators |= IN_HEAP; } - Node* val = is_store ? argument(4) : nullptr; + Node* val = nullptr; + if (is_store) { + val = argument(prefix_size+4); // passed as a 64-bit long + if (type_is_narrow_int) { + val = ConvL2I(val); + } + switch (type) { + case T_FLOAT: val = _gvn.transform(new MoveI2FNode(val)); break; + case T_DOUBLE: val = _gvn.transform(new MoveL2DNode(val)); break; + default: break; + } + } const TypePtr* adr_type = _gvn.type(adr)->isa_ptr(); if (adr_type == TypePtr::NULL_PTR) { @@ -2561,7 +2499,7 @@ bool LibraryCallKit::inline_unsafe_access(bool is_store, const BasicType type, c const Type *value_type = Type::get_const_basic_type(type); // Figure out the memory ordering. - decorators |= mo_decorator_for_access_kind(kind); + decorators |= mo_decorator_for_access_kind(mo); if (!is_store && type == T_OBJECT) { const TypeOopPtr* tjp = sharpen_unsafe_type(alias_type, adr_type); @@ -2590,31 +2528,16 @@ bool LibraryCallKit::inline_unsafe_access(bool is_store, const BasicType type, c if (p == nullptr) { // Could not constant fold the load p = access_load_at(heap_base_oop, adr, adr_type, value_type, type, decorators); - // Normalize the value returned by getBoolean in the following cases - if (type == T_BOOLEAN && - (mismatched || - heap_base_oop == top() || // - heap_base_oop is null or - (can_access_non_heap && field == nullptr)) // - heap_base_oop is potentially null - // and the unsafe access is made to large offset - // (i.e., larger than the maximum offset necessary for any - // field access) - ) { - IdealKit ideal = IdealKit(this); -#define __ ideal. - IdealVariable normalized_result(ideal); - __ declarations_done(); - __ set(normalized_result, p); - __ if_then(p, BoolTest::ne, ideal.ConI(0)); - __ set(normalized_result, ideal.ConI(1)); - ideal.end_if(); - final_sync(ideal); - p = __ value(normalized_result); -#undef __ - } + // Note: T_BOOLEAN normalization is handled by the Unsafe wrappers. + // At this point, just load a byte, word, or whatever from memory. + } + switch (type) { + case T_FLOAT: p = _gvn.transform(new MoveF2INode(p)); break; + case T_DOUBLE: p = _gvn.transform(new MoveD2LNode(p)); break; + default: break; } - if (type == T_ADDRESS) { - p = gvn().transform(new CastP2XNode(nullptr, p)); - p = ConvX2UL(p); + if (type_is_narrow_int) { + p = ConvI2L(p); // convert to utype = T_LONG } // The load node has the control of the preceding MemBarCPUOrder. All // following nodes will have the control of the MemBarCPUOrder inserted at @@ -2622,11 +2545,6 @@ bool LibraryCallKit::inline_unsafe_access(bool is_store, const BasicType type, c // point is fine. set_result(p); } else { - if (bt == T_ADDRESS) { - // Repackage the long as a pointer. - val = ConvL2X(val); - val = gvn().transform(new CastX2PNode(val)); - } access_store_at(heap_base_oop, adr, adr_type, val, value_type, type, decorators); } @@ -2637,54 +2555,25 @@ bool LibraryCallKit::inline_unsafe_access(bool is_store, const BasicType type, c // This method serves a couple of different customers (depending on LoadStoreKind): // // LS_cmp_swap: -// -// boolean compareAndSetReference(Object o, long offset, Object expected, Object x); -// boolean compareAndSetInt( Object o, long offset, int expected, int x); -// boolean compareAndSetLong( Object o, long offset, long expected, long x); -// // LS_cmp_swap_weak: -// -// boolean weakCompareAndSetReference( Object o, long offset, Object expected, Object x); -// boolean weakCompareAndSetReferencePlain( Object o, long offset, Object expected, Object x); -// boolean weakCompareAndSetReferenceAcquire(Object o, long offset, Object expected, Object x); -// boolean weakCompareAndSetReferenceRelease(Object o, long offset, Object expected, Object x); -// -// boolean weakCompareAndSetInt( Object o, long offset, int expected, int x); -// boolean weakCompareAndSetIntPlain( Object o, long offset, int expected, int x); -// boolean weakCompareAndSetIntAcquire( Object o, long offset, int expected, int x); -// boolean weakCompareAndSetIntRelease( Object o, long offset, int expected, int x); -// -// boolean weakCompareAndSetLong( Object o, long offset, long expected, long x); -// boolean weakCompareAndSetLongPlain( Object o, long offset, long expected, long x); -// boolean weakCompareAndSetLongAcquire( Object o, long offset, long expected, long x); -// boolean weakCompareAndSetLongRelease( Object o, long offset, long expected, long x); +// boolean compareAndSetReferenceMO( MO, Object,long, Object e, Object x); +// boolean compareAndSetPrimitiveBitsMO(MO,BT, Object,long, long e, long x); // // LS_cmp_exchange: -// -// Object compareAndExchangeReferenceVolatile(Object o, long offset, Object expected, Object x); -// Object compareAndExchangeReferenceAcquire( Object o, long offset, Object expected, Object x); -// Object compareAndExchangeReferenceRelease( Object o, long offset, Object expected, Object x); -// -// Object compareAndExchangeIntVolatile( Object o, long offset, Object expected, Object x); -// Object compareAndExchangeIntAcquire( Object o, long offset, Object expected, Object x); -// Object compareAndExchangeIntRelease( Object o, long offset, Object expected, Object x); -// -// Object compareAndExchangeLongVolatile( Object o, long offset, Object expected, Object x); -// Object compareAndExchangeLongAcquire( Object o, long offset, Object expected, Object x); -// Object compareAndExchangeLongRelease( Object o, long offset, Object expected, Object x); -// -// LS_get_add: -// -// int getAndAddInt( Object o, long offset, int delta) -// long getAndAddLong(Object o, long offset, long delta) +// Object compareAndExchangeReferenceMO( MO, Object,long, Object e, Object x); +// Object compareAndExchangePrimitiveBitsMO(MO,BT, Object,long, Object e, Object x); // // LS_get_set: +// int getAndSetReferenceMO( MO, Object,long, Object newValue) +// LS_get_set: +// LS_get_add: +// int getAndOperatePrimitiveBitsMO(MO,BT, Object,long, long newValueOrDelta) // -// int getAndSet(Object o, long offset, int newValue) -// long getAndSet(Object o, long offset, long newValue) -// Object getAndSet(Object o, long offset, Object newValue) -// -bool LibraryCallKit::inline_unsafe_load_store(const BasicType type, const LoadStoreKind kind, const AccessKind access_kind) { +bool LibraryCallKit::inline_unsafe_load_store(LoadStoreKind kind, + vmIntrinsics::MemoryOrder mo, + BasicType type, + vmIntrinsics::BitsOperation op, + int prefix_size) { // This basic scheme here is the same as inline_unsafe_access, but // differs in enough details that combining them would make the code // overly confusing. (This is a true fact! I originally combined @@ -2693,9 +2582,47 @@ bool LibraryCallKit::inline_unsafe_load_store(const BasicType type, const LoadSt // the correspondences clearer. - dl if (callee()->is_static()) return false; // caller must have the capability! + const bool weak_cas = (mo & vmIntrinsics::MO_WEAK_CAS) != 0; + mo = (vmIntrinsics::MemoryOrder)(mo & vmIntrinsics::MO_MODE_MASK); + + assert(type == T_OBJECT || is_integral_type(type), "Unsafe.java wrapper responsibility"); + BasicType utype = type == T_OBJECT ? T_OBJECT : T_LONG; + + // FIXME: we need to ask the backend if 16-bit and 8-bit types are supported. + // Now that the intrinsics are simplified, we need to directly ask about + // match rules, or (better yet) build matcher queries that can express + // not only an intrinsic (or match rule) but some poly-prefix parameters. + if (is_subword_type(type)) return false; //FIXME + + bool returns_boolean = false; + switch (kind) { + case LS_cmp_swap: + returns_boolean = true; + if (weak_cas) + kind = LS_cmp_swap_weak; // user supplied optional MO_WEAK_CAS bit + break; + case LS_get_set: + switch (op) { + case vmIntrinsics::OP_SWAP: + case vmIntrinsics::OP_NONE: + break; // user requested op'=' or (for ref) no op + case vmIntrinsics::OP_ADD: + kind = LS_get_add; // user requested op'+' + break; + default: + // FIXME: intrinsic expansion NYI for bitswise and/or/xor + // The matcher should support queries like "can I do a GAX + // where X is bitwise-or and T is T_BYTE?". The old system + // of intrinsic codes was never expressive enough to ask + // such questions. + return false; + } + default: + break; + } DecoratorSet decorators = C2_UNSAFE_ACCESS; - decorators |= mo_decorator_for_access_kind(access_kind); + decorators |= mo_decorator_for_access_kind(mo); #ifndef PRODUCT BasicType rtype; @@ -2703,18 +2630,19 @@ bool LibraryCallKit::inline_unsafe_load_store(const BasicType type, const LoadSt ResourceMark rm; // Check the signatures. ciSignature* sig = callee()->signature(); + for (int i = 0; i < prefix_size; i++) + assert(sig->type_at(i)->basic_type() == T_BYTE, "prefix args are bytes"); rtype = sig->return_type()->basic_type(); switch(kind) { case LS_get_add: case LS_get_set: { // Check the signatures. #ifdef ASSERT - assert(rtype == type, "get and set must return the expected type"); - assert(sig->count() == 3, "get and set has 3 arguments"); - assert(sig->type_at(0)->basic_type() == T_OBJECT, "get and set base is object"); - assert(sig->type_at(1)->basic_type() == T_LONG, "get and set offset is long"); - assert(sig->type_at(2)->basic_type() == type, "get and set must take expected type as new value/delta"); - assert(access_kind == Volatile, "mo is not passed to intrinsic nodes in current implementation"); + assert(rtype == utype, "get and set must return the expected type"); + assert(sig->count() == prefix_size+3, "get and set has 3 arguments"); + assert(sig->type_at(prefix_size+0)->basic_type() == T_OBJECT, "get and set base is object"); + assert(sig->type_at(prefix_size+1)->basic_type() == T_LONG, "get and set offset is long"); + assert(sig->type_at(prefix_size+2)->basic_type() == utype, "get and set must take expected type as new value/delta"); #endif // ASSERT break; } @@ -2723,19 +2651,23 @@ bool LibraryCallKit::inline_unsafe_load_store(const BasicType type, const LoadSt // Check the signatures. #ifdef ASSERT assert(rtype == T_BOOLEAN, "CAS must return boolean"); - assert(sig->count() == 4, "CAS has 4 arguments"); - assert(sig->type_at(0)->basic_type() == T_OBJECT, "CAS base is object"); - assert(sig->type_at(1)->basic_type() == T_LONG, "CAS offset is long"); + assert(sig->count() == prefix_size+4, "CAS has 4 arguments"); + assert(sig->type_at(prefix_size+0)->basic_type() == T_OBJECT, "CAS base is object"); + assert(sig->type_at(prefix_size+1)->basic_type() == T_LONG, "CAS offset is long"); + assert(sig->type_at(prefix_size+2)->basic_type() == utype, "CAS e value is utype"); + assert(sig->type_at(prefix_size+3)->basic_type() == utype, "CAS x value is utype"); #endif // ASSERT break; } case LS_cmp_exchange: { // Check the signatures. #ifdef ASSERT - assert(rtype == type, "CAS must return the expected type"); - assert(sig->count() == 4, "CAS has 4 arguments"); - assert(sig->type_at(0)->basic_type() == T_OBJECT, "CAS base is object"); - assert(sig->type_at(1)->basic_type() == T_LONG, "CAS offset is long"); + assert(rtype == utype, "CAS must return the expected type"); + assert(sig->count() == prefix_size+4, "CAS has 4 arguments"); + assert(sig->type_at(prefix_size+0)->basic_type() == T_OBJECT, "CAS base is object"); + assert(sig->type_at(prefix_size+1)->basic_type() == T_LONG, "CAS offset is long"); + assert(sig->type_at(prefix_size+2)->basic_type() == utype, "CAS e value is utype"); + assert(sig->type_at(prefix_size+3)->basic_type() == utype, "CAS x value is utype"); #endif // ASSERT break; } @@ -2748,7 +2680,7 @@ bool LibraryCallKit::inline_unsafe_load_store(const BasicType type, const LoadSt C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe". // Get arguments: - Node* receiver = nullptr; + Node* receiver = argument(0); // type: oop Node* base = nullptr; Node* offset = nullptr; Node* oldval = nullptr; @@ -2757,21 +2689,19 @@ bool LibraryCallKit::inline_unsafe_load_store(const BasicType type, const LoadSt case LS_cmp_swap: case LS_cmp_swap_weak: case LS_cmp_exchange: { - const bool two_slot_type = type2size[type] == 2; - receiver = argument(0); // type: oop - base = argument(1); // type: oop - offset = argument(2); // type: long - oldval = argument(4); // type: oop, int, or long - newval = argument(two_slot_type ? 6 : 5); // type: oop, int, or long + const bool two_slot_utype = type2size[utype] == 2; + base = argument(prefix_size+1); // type: oop + offset = argument(prefix_size+2); // type: long + oldval = argument(prefix_size+4); // type: oop or long + newval = argument(prefix_size+(two_slot_utype ? 6 : 5)); // type: oop or long break; } case LS_get_add: case LS_get_set: { - receiver = argument(0); // type: oop - base = argument(1); // type: oop - offset = argument(2); // type: long + base = argument(prefix_size+1); // type: oop + offset = argument(prefix_size+2); // type: long oldval = nullptr; - newval = argument(4); // type: oop, int, or long + newval = argument(prefix_size+4); // type: ooop or long break; } default: @@ -2787,7 +2717,7 @@ bool LibraryCallKit::inline_unsafe_load_store(const BasicType type, const LoadSt offset = ConvL2X(offset); // Save state and restore on bailout SavedState old_state(this); - Node* adr = make_unsafe_address(base, offset,type, false); + Node* adr = make_unsafe_address(base, offset, type, false); const TypePtr *adr_type = _gvn.type(adr)->isa_ptr(); Compile::AliasType* alias_type = C->alias_type(adr_type); @@ -2832,6 +2762,7 @@ bool LibraryCallKit::inline_unsafe_load_store(const BasicType type, const LoadSt int alias_idx = C->get_alias_index(adr_type); + bool type_is_narrow_int = false; if (is_reference_type(type)) { decorators |= IN_HEAP | ON_UNKNOWN_OOP_REF; @@ -2845,6 +2776,11 @@ bool LibraryCallKit::inline_unsafe_load_store(const BasicType type, const LoadSt // Refine the value to a null constant, when it is known to be null oldval = _gvn.makecon(TypePtr::NULL_PTR); } + } else if (!is_double_word_type(type)) { + type_is_narrow_int = true; // utype is T_LONG, type is T_INT, etc. + newval = ConvL2I(newval); + if (oldval != nullptr) + oldval = ConvL2I(oldval); } Node* result = nullptr; @@ -2875,6 +2811,13 @@ bool LibraryCallKit::inline_unsafe_load_store(const BasicType type, const LoadSt ShouldNotReachHere(); } + if (!returns_boolean && type_is_narrow_int) { + // Intrinsic is one of _compareAndExchangePrimitiveBitsMO (etc.) + // and type is one of T_INT, T_SHORT, T_BYTE. + assert(type2size[result->bottom_type()->basic_type()] == 1, "result type should match"); + result = ConvI2L(result); // convert to utype = T_LONG + } + assert(type2size[result->bottom_type()->basic_type()] == type2size[rtype], "result type should match"); set_result(result); return true; diff --git a/src/hotspot/share/opto/library_call.hpp b/src/hotspot/share/opto/library_call.hpp index bfe29814dec17..a5800bdc7b843 100644 --- a/src/hotspot/share/opto/library_call.hpp +++ b/src/hotspot/share/opto/library_call.hpp @@ -253,9 +253,8 @@ class LibraryCallKit : public GraphKit { int classify_unsafe_addr(Node* &base, Node* &offset, BasicType type); Node* make_unsafe_address(Node*& base, Node* offset, BasicType type = T_ILLEGAL, bool can_cast = false); - typedef enum { Relaxed, Opaque, Volatile, Acquire, Release } AccessKind; - DecoratorSet mo_decorator_for_access_kind(AccessKind kind); - bool inline_unsafe_access(bool is_store, BasicType type, AccessKind kind, bool is_unaligned); + DecoratorSet mo_decorator_for_access_kind(vmIntrinsics::MemoryOrder mo); + bool inline_unsafe_access(bool is_store, vmIntrinsics::MemoryOrder mo, BasicType type, int prefix_size); static bool klass_needs_init_guard(Node* kls); bool inline_unsafe_allocate(); bool inline_unsafe_newArray(bool uninitialized); @@ -317,7 +316,7 @@ class LibraryCallKit : public GraphKit { bool inline_array_sort(); bool inline_array_partition(); typedef enum { LS_get_add, LS_get_set, LS_cmp_swap, LS_cmp_swap_weak, LS_cmp_exchange } LoadStoreKind; - bool inline_unsafe_load_store(BasicType type, LoadStoreKind kind, AccessKind access_kind); + bool inline_unsafe_load_store(LoadStoreKind kind, vmIntrinsics::MemoryOrder mo, BasicType type, vmIntrinsics::BitsOperation op, int prefix_size); bool inline_unsafe_fence(vmIntrinsics::ID id); bool inline_onspinwait(); bool inline_fp_conversions(vmIntrinsics::ID id); diff --git a/src/hotspot/share/prims/unsafe.cpp b/src/hotspot/share/prims/unsafe.cpp index c950690e8ab88..bd6dc246a15d1 100644 --- a/src/hotspot/share/prims/unsafe.cpp +++ b/src/hotspot/share/prims/unsafe.cpp @@ -26,6 +26,7 @@ #include "classfile/classLoader.hpp" #include "classfile/classLoadInfo.hpp" #include "classfile/javaClasses.inline.hpp" +#include "classfile/symbolTable.hpp" #include "classfile/systemDictionary.hpp" #include "classfile/vmSymbols.hpp" #include "jfr/jfrEvents.hpp" @@ -42,6 +43,7 @@ #include "oops/typeArrayOop.inline.hpp" #include "prims/jvmtiExport.hpp" #include "prims/unsafe.hpp" +#include "runtime/fieldDescriptor.inline.hpp" #include "runtime/globals.hpp" #include "runtime/handles.inline.hpp" #include "runtime/interfaceSupport.inline.hpp" @@ -65,7 +67,6 @@ * Implementation of the jdk.internal.misc.Unsafe class */ - #define MAX_OBJECT_SIZE \ ( arrayOopDesc::base_offset_in_bytes(T_DOUBLE) \ + ((julong)max_jint * sizeof(double)) ) @@ -77,6 +78,7 @@ JVM_LEAF(static result_type, header) // All memory access methods (e.g. getInt, copyMemory) must use this macro. +// (Except, methods which read or write managed pointers use another path.) // We call these methods "scoped" methods, as access to these methods is // typically governed by a "scope" (a MemorySessionImpl object), and no // access is allowed when the scope is no longer alive. @@ -190,17 +192,56 @@ class GuardUnsafeAccess { } }; +/** + * Helper macro to implement variable-size operations in polymorphic + * methods that manipulate primitive values for Unsafe. + * Executes the body (whatever that may be) with var_t defined as + * an unsigned integral type of size 1, 2, 4, or 8, and the same + * bit-size as the basic type bt. + * + * The processing for T_BYTE and T_BOOLEAN are the same, as are + * T_LONG and T_DOUBLE, T_INT and T_FLOAT, and TSHORT and T_CHAR. + * It is up to the caller to ensure that no other T-values appear + * here, and that special handling of types (e.g., boolean fixups) + * is performed elsewhere. + */ +#define TYPE_SIZE_SWITCH(bt, var_t, body) { \ + switch ((bt) & UNSAFE_PRIMITIVE_SIZE_MASK) { \ + case 0: { using var_t = jubyte; {body;} break; } \ + case 1: { using var_t = jushort; {body;} break; } \ + case 2: { using var_t = juint; {body;} break; } \ + default: { using var_t = julong; {body;} break; } \ + } \ + } /*end*/ + +static size_t bt_size(BasicType bt) { + return 1 << (bt & UNSAFE_PRIMITIVE_SIZE_MASK); +} + +template +static julong maybe_pad_with_garbage(val_t v) { + julong bits = v; +#ifdef ASSERT + // inject some garbage as padding, to stress-test surrounding layers + // e.g., 0x42 pads up as 0xFFFFFFCE00000042 + if (sizeof(val_t) <= sizeof(bits)/2) { + bits ^= ~bits << (sizeof(bits)/2 * BitsPerByte); + } +#endif //ASSERT + return bits; +} + /** * Helper class for accessing memory. * * Normalizes values and wraps accesses in * JavaThread::doing_unsafe_access() if needed. */ -template class MemoryAccess : StackObj { JavaThread* _thread; oop _obj; ptrdiff_t _offset; + BasicType _basic_type; // Resolves and returns the address of the memory access. // This raw memory access may fault, so we make sure it happens within the @@ -208,142 +249,212 @@ class MemoryAccess : StackObj { // of Thread::set_doing_unsafe_access() is also volatile, these accesses // can not be reordered by the compiler. Therefore, if the access triggers // a fault, we will know that Thread::doing_unsafe_access() returns true. + template volatile T* addr() { void* addr = index_oop_from_field_offset_long(_obj, _offset); return static_cast(addr); } - template - U normalize_for_write(U x) { - return x; + static julong get_via_bytes(size_t size, address addr) { + switch (size) { + case 1: return *(jubyte*) addr; + case 2: return Bytes::get_native_u2(addr); + case 4: return Bytes::get_native_u4(addr); + default: return Bytes::get_native_u8(addr); + } } - jboolean normalize_for_write(jboolean x) { - return x & 1; + static void put_via_bytes(size_t size, address addr, julong x) { + switch (size) { + case 1: *(jubyte*) addr = (jubyte) x; + case 2: Bytes::put_native_u2(addr, (jushort)x); + case 4: Bytes::put_native_u4(addr, (juint) x); + default: Bytes::put_native_u8(addr, x); + } } - template - U normalize_for_read(U x) { - return x; - } - - jboolean normalize_for_read(jboolean x) { - return x != 0; - } + // Note: We do not normalize booleans at this level. That is done + // by strongly-typed VM access methods like oopDesc::bool_field, but + // not by this C++ code, because it is not strongly typed. Instead, + // the next layer up, the Java class Unsafe, handles the sanitizing + // of booleans. See bool2byte and byte2bool in that class. With + // this division of labor, the unsafe native layer (with related JIT + // intrinsics) can concentrate on correctly sized and sequenced + // access, without adding in extra data type requirements. public: - MemoryAccess(JavaThread* thread, jobject obj, jlong offset) - : _thread(thread), _obj(JNIHandles::resolve(obj)), _offset((ptrdiff_t)offset) { - assert_field_offset_sane(_obj, offset); + MemoryAccess(JavaThread* thread, jobject obj, jlong offset, int basic_type) + : _thread(thread), + _obj(JNIHandles::resolve(obj)), _offset((ptrdiff_t)offset), + _basic_type((BasicType)basic_type) + { + //assert_field_offset_sane(_obj, offset); -- done later in addr() + assert(is_java_primitive(_basic_type), "caller resp"); + assert((1 << ((int)_basic_type & UNSAFE_PRIMITIVE_SIZE_MASK)) + == type2aelembytes(_basic_type), "must be"); } - T get() { + julong get() { GuardUnsafeAccess guard(_thread); - return normalize_for_read(*addr()); + TYPE_SIZE_SWITCH(_basic_type, val_t, { + val_t v = *addr(); + return maybe_pad_with_garbage(v); + }); } // we use this method at some places for writing to 0 e.g. to cause a crash; // ubsan does not know that this is the desired behavior ATTRIBUTE_NO_UBSAN - void put(T x) { + void put(julong x) { GuardUnsafeAccess guard(_thread); - *addr() = normalize_for_write(x); + TYPE_SIZE_SWITCH(_basic_type, val_t, { + *addr() = (val_t)x; + }); } + julong get_unaligned() { + GuardUnsafeAccess guard(_thread); + TYPE_SIZE_SWITCH(_basic_type, val_t, { + address va = (address)addr(); + val_t v = get_via_bytes(sizeof(val_t), va); + return maybe_pad_with_garbage(v); + }); + } - T get_volatile() { + void put_unaligned(julong x) { GuardUnsafeAccess guard(_thread); - volatile T ret = RawAccess::load(addr()); - return normalize_for_read(ret); + TYPE_SIZE_SWITCH(_basic_type, val_t, { + address va = (address)addr(); + put_via_bytes(sizeof(val_t), va, x); + }); } - void put_volatile(T x) { + julong get_volatile() { GuardUnsafeAccess guard(_thread); - RawAccess::store(addr(), normalize_for_write(x)); + TYPE_SIZE_SWITCH(_basic_type, val_t, { + val_t v = RawAccess::load(addr()); + return maybe_pad_with_garbage(v); + }); + } + + void put_volatile(julong x) { + GuardUnsafeAccess guard(_thread); + TYPE_SIZE_SWITCH(_basic_type, val_t, { + RawAccess::store(addr(), (val_t)x); + }); } }; // These functions allow a null base pointer with an arbitrary address. // But if the base pointer is non-null, the offset should make some sense. // That is, it should be in the range [0, MAX_OBJECT_SIZE]. -UNSAFE_ENTRY(jobject, Unsafe_GetReference(JNIEnv *env, jobject unsafe, jobject obj, jlong offset)) { +UNSAFE_ENTRY(jobject, Unsafe_GetReferenceMO(JNIEnv *env, jobject unsafe, + jbyte memory_order, + jobject obj, jlong offset)) { oop p = JNIHandles::resolve(obj); assert_field_offset_sane(p, offset); - oop v = HeapAccess::oop_load_at(p, offset); + assert(vmIntrinsics::is_valid_memory_order(memory_order, + vmIntrinsics::MO_RELEASE), + "bad MO bits from Java: 0x%02x", memory_order & 0xFF); + oop v; + switch (memory_order) { + case vmIntrinsics::MO_PLAIN: + v = HeapAccess::oop_load_at(p, offset); + break; + + default: + // MO_VOLATILE is a conservative approximation for acquire & release + v = HeapAccess::oop_load_at(p, offset); + break; + } return JNIHandles::make_local(THREAD, v); } UNSAFE_END -UNSAFE_ENTRY(void, Unsafe_PutReference(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jobject x_h)) { +UNSAFE_ENTRY(void, Unsafe_PutReferenceMO(JNIEnv *env, jobject unsafe, + jbyte memory_order, + jobject obj, jlong offset, jobject x_h)) { oop x = JNIHandles::resolve(x_h); oop p = JNIHandles::resolve(obj); assert_field_offset_sane(p, offset); - HeapAccess::oop_store_at(p, offset, x); + assert(vmIntrinsics::is_valid_memory_order(memory_order, + vmIntrinsics::MO_ACQUIRE), + "bad MO bits from Java: 0x%02x", memory_order & 0xFF); + switch (memory_order) { + case vmIntrinsics::MO_PLAIN: + HeapAccess::oop_store_at(p, offset, x); + break; + + default: + // MO_VOLATILE is a conservative approximation for acquire & release + HeapAccess::oop_store_at(p, offset, x); + break; + } } UNSAFE_END -UNSAFE_ENTRY(jobject, Unsafe_GetReferenceVolatile(JNIEnv *env, jobject unsafe, jobject obj, jlong offset)) { - oop p = JNIHandles::resolve(obj); - assert_field_offset_sane(p, offset); - oop v = HeapAccess::oop_load_at(p, offset); +UNSAFE_ENTRY(jobject, Unsafe_GetUncompressedObject(JNIEnv *env, jobject unsafe, + jlong addr)) { + oop v = *(oop*) (address) addr; return JNIHandles::make_local(THREAD, v); } UNSAFE_END -UNSAFE_ENTRY(void, Unsafe_PutReferenceVolatile(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jobject x_h)) { - oop x = JNIHandles::resolve(x_h); - oop p = JNIHandles::resolve(obj); - assert_field_offset_sane(p, offset); - HeapAccess::oop_store_at(p, offset, x); -} UNSAFE_END +UNSAFE_ENTRY_SCOPED(jlong, Unsafe_GetPrimitiveBitsMO(JNIEnv *env, jobject unsafe, + jbyte memory_order, jbyte basic_type, + jobject obj, jlong offset)) { + assert(vmIntrinsics::is_valid_memory_order(memory_order & ~vmIntrinsics::MO_UNALIGNED, + vmIntrinsics::MO_RELEASE), + "bad MO bits from Java: 0x%02x", memory_order & 0xFF); + assert(vmIntrinsics::is_valid_primitive_type(basic_type), + "bad BT bits from Java: 0x%02x", basic_type & 0xFF); + julong result; + auto ma = MemoryAccess(thread, obj, offset, basic_type); + switch (memory_order) { + case vmIntrinsics::MO_PLAIN | vmIntrinsics::MO_UNALIGNED: + if (!UseUnalignedAccesses && (offset & (bt_size((BasicType)basic_type) - 1)) != 0) { + result = ma.get_unaligned(); + break; + } + // else fall through -UNSAFE_ENTRY(jobject, Unsafe_GetUncompressedObject(JNIEnv *env, jobject unsafe, jlong addr)) { - oop v = *(oop*) (address) addr; - return JNIHandles::make_local(THREAD, v); + case vmIntrinsics::MO_PLAIN: + // Note: This says, "plain" but there is in fact a volatile load inside. + result = ma.get(); + break; + + default: + // MO_VOLATILE is a conservative approximation for acquire & release + result = ma.get_volatile(); + } + return result; } UNSAFE_END -#define DEFINE_GETSETOOP(java_type, Type) \ - \ -UNSAFE_ENTRY_SCOPED(java_type, Unsafe_Get##Type(JNIEnv *env, jobject unsafe, jobject obj, jlong offset)) { \ - return MemoryAccess(thread, obj, offset).get(); \ -} UNSAFE_END \ - \ -UNSAFE_ENTRY_SCOPED(void, Unsafe_Put##Type(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, java_type x)) { \ - MemoryAccess(thread, obj, offset).put(x); \ -} UNSAFE_END \ - \ -// END DEFINE_GETSETOOP. - -DEFINE_GETSETOOP(jboolean, Boolean) -DEFINE_GETSETOOP(jbyte, Byte) -DEFINE_GETSETOOP(jshort, Short); -DEFINE_GETSETOOP(jchar, Char); -DEFINE_GETSETOOP(jint, Int); -DEFINE_GETSETOOP(jlong, Long); -DEFINE_GETSETOOP(jfloat, Float); -DEFINE_GETSETOOP(jdouble, Double); - -#undef DEFINE_GETSETOOP - -#define DEFINE_GETSETOOP_VOLATILE(java_type, Type) \ - \ -UNSAFE_ENTRY_SCOPED(java_type, Unsafe_Get##Type##Volatile(JNIEnv *env, jobject unsafe, jobject obj, jlong offset)) { \ - return MemoryAccess(thread, obj, offset).get_volatile(); \ -} UNSAFE_END \ - \ -UNSAFE_ENTRY_SCOPED(void, Unsafe_Put##Type##Volatile(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, java_type x)) { \ - MemoryAccess(thread, obj, offset).put_volatile(x); \ -} UNSAFE_END \ - \ -// END DEFINE_GETSETOOP_VOLATILE. - -DEFINE_GETSETOOP_VOLATILE(jboolean, Boolean) -DEFINE_GETSETOOP_VOLATILE(jbyte, Byte) -DEFINE_GETSETOOP_VOLATILE(jshort, Short); -DEFINE_GETSETOOP_VOLATILE(jchar, Char); -DEFINE_GETSETOOP_VOLATILE(jint, Int); -DEFINE_GETSETOOP_VOLATILE(jlong, Long); -DEFINE_GETSETOOP_VOLATILE(jfloat, Float); -DEFINE_GETSETOOP_VOLATILE(jdouble, Double); - -#undef DEFINE_GETSETOOP_VOLATILE +UNSAFE_ENTRY_SCOPED(void, Unsafe_PutPrimitiveBitsMO(JNIEnv *env, jobject unsafe, + jbyte memory_order, jbyte basic_type, + jobject obj, jlong offset, jlong x)) { + assert(vmIntrinsics::is_valid_memory_order(memory_order & ~vmIntrinsics::MO_UNALIGNED, + vmIntrinsics::MO_ACQUIRE), + "bad MO bits from Java: 0x%02x", memory_order & 0xFF); + assert(vmIntrinsics::is_valid_primitive_type(basic_type), + "bad BT bits from Java: 0x%02x", basic_type & 0xFF); + auto ma = MemoryAccess(thread, obj, offset, basic_type); + switch (memory_order) { + case vmIntrinsics::MO_PLAIN | vmIntrinsics::MO_UNALIGNED: + if (!UseUnalignedAccesses && (offset & (bt_size((BasicType)basic_type) - 1)) != 0) { + ma.put_unaligned(x); + break; + } + // else fall through + + case vmIntrinsics::MO_PLAIN: + // Note: This says, "plain" but there is in fact a volatile store inside. + ma.put(x); + break; + + default: + // MO_VOLATILE is a conservative approximation for acquire & release + ma.put_volatile(x); + break; + } +} UNSAFE_END UNSAFE_LEAF(void, Unsafe_FullFence(JNIEnv *env, jobject unsafe)) { OrderAccess::fence(); @@ -725,46 +836,70 @@ UNSAFE_ENTRY(void, Unsafe_ThrowException(JNIEnv *env, jobject unsafe, jthrowable // JSR166 ------------------------------------------------------------------ -UNSAFE_ENTRY(jobject, Unsafe_CompareAndExchangeReference(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jobject e_h, jobject x_h)) { +UNSAFE_ENTRY(jobject, Unsafe_CompareAndExchangeReferenceMO(JNIEnv *env, jobject unsafe, + jbyte memory_order, + jobject obj, jlong offset, jobject e_h, jobject x_h)) { + assert(vmIntrinsics::is_valid_memory_order(memory_order), + "bad MO bits from Java: 0x%02x", memory_order & 0xFF); oop x = JNIHandles::resolve(x_h); oop e = JNIHandles::resolve(e_h); oop p = JNIHandles::resolve(obj); assert_field_offset_sane(p, offset); + // just use MO_VOLATILE for all MO inputs oop res = HeapAccess::oop_atomic_cmpxchg_at(p, (ptrdiff_t)offset, e, x); return JNIHandles::make_local(THREAD, res); } UNSAFE_END -UNSAFE_ENTRY_SCOPED(jint, Unsafe_CompareAndExchangeInt(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jint e, jint x)) { - oop p = JNIHandles::resolve(obj); - volatile jint* addr = (volatile jint*)index_oop_from_field_offset_long(p, offset); - return AtomicAccess::cmpxchg(addr, e, x); -} UNSAFE_END - -UNSAFE_ENTRY_SCOPED(jlong, Unsafe_CompareAndExchangeLong(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jlong e, jlong x)) { +UNSAFE_ENTRY_SCOPED(jlong, Unsafe_CompareAndExchangePrimitiveBitsMO(JNIEnv *env, jobject unsafe, + jbyte memory_order, jbyte basic_type, + jobject obj, jlong offset, + jlong e, jlong x)) { + assert(vmIntrinsics::is_valid_memory_order(memory_order), + "bad MO bits from Java: 0x%02x", memory_order & 0xFF); + assert(vmIntrinsics::is_valid_primitive_type(basic_type), + "bad BT bits from Java: 0x%02x", basic_type & 0xFF); oop p = JNIHandles::resolve(obj); - volatile jlong* addr = (volatile jlong*)index_oop_from_field_offset_long(p, offset); - return AtomicAccess::cmpxchg(addr, e, x); + auto addr = index_oop_from_field_offset_long(p, offset); + // just use MO_VOLATILE for all MO inputs + TYPE_SIZE_SWITCH(basic_type, val_t, { + auto expect = static_cast(e); + auto update = static_cast(x); + return AtomicAccess::cmpxchg(static_cast(addr), expect, update); + }); } UNSAFE_END -UNSAFE_ENTRY(jboolean, Unsafe_CompareAndSetReference(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jobject e_h, jobject x_h)) { +UNSAFE_ENTRY(jboolean, Unsafe_CompareAndSetReferenceMO(JNIEnv *env, jobject unsafe, + jbyte memory_order, + jobject obj, jlong offset, jobject e_h, jobject x_h)) { + // ignore MO_WEAK_CAS here; the JIT might use it + assert(vmIntrinsics::is_valid_memory_order(memory_order & ~vmIntrinsics::MO_WEAK_CAS), + "bad MO bits from Java: 0x%02x", memory_order & 0xFF); oop x = JNIHandles::resolve(x_h); oop e = JNIHandles::resolve(e_h); oop p = JNIHandles::resolve(obj); assert_field_offset_sane(p, offset); + // just use MO_VOLATILE for all MO inputs oop ret = HeapAccess::oop_atomic_cmpxchg_at(p, (ptrdiff_t)offset, e, x); - return ret == e; + return ret == e; } UNSAFE_END -UNSAFE_ENTRY_SCOPED(jboolean, Unsafe_CompareAndSetInt(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jint e, jint x)) { +UNSAFE_ENTRY_SCOPED(jboolean, Unsafe_CompareAndSetPrimitiveBitsMO(JNIEnv *env, jobject unsafe, + jbyte memory_order, jbyte basic_type, + jobject obj, jlong offset, jlong e, jlong x)) { + // ignore MO_WEAK_CAS here; the JIT might use it + assert(vmIntrinsics::is_valid_memory_order(memory_order & ~vmIntrinsics::MO_WEAK_CAS), + "bad MO bits from Java: 0x%02x", memory_order & 0xFF); + assert(vmIntrinsics::is_valid_primitive_type(basic_type), + "bad BT bits from Java: 0x%02x", basic_type & 0xFF); oop p = JNIHandles::resolve(obj); - volatile jint* addr = (volatile jint*)index_oop_from_field_offset_long(p, offset); - return AtomicAccess::cmpxchg(addr, e, x) == e; -} UNSAFE_END - -UNSAFE_ENTRY_SCOPED(jboolean, Unsafe_CompareAndSetLong(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jlong e, jlong x)) { - oop p = JNIHandles::resolve(obj); - volatile jlong* addr = (volatile jlong*)index_oop_from_field_offset_long(p, offset); - return AtomicAccess::cmpxchg(addr, e, x) == e; + auto addr = index_oop_from_field_offset_long(p, offset); + // just use MO_VOLATILE for all MO inputs + TYPE_SIZE_SWITCH(basic_type, val_t, { + auto expect = static_cast(e); + auto update = static_cast(x); + auto actual = AtomicAccess::cmpxchg(static_cast(addr), expect, update); + return actual == expect; + }); } UNSAFE_END static void post_thread_park_event(EventThreadPark* event, const oop obj, jlong timeout_nanos, jlong until_epoch_millis) { @@ -859,29 +994,19 @@ UNSAFE_ENTRY(jint, Unsafe_GetLoadAverage0(JNIEnv *env, jobject unsafe, jdoubleAr #define CC (char*) /*cast a literal from (const char*)*/ #define FN_PTR(f) CAST_FROM_FN_PTR(void*, &f) -#define DECLARE_GETPUTOOP(Type, Desc) \ - {CC "get" #Type, CC "(" OBJ "J)" #Desc, FN_PTR(Unsafe_Get##Type)}, \ - {CC "put" #Type, CC "(" OBJ "J" #Desc ")V", FN_PTR(Unsafe_Put##Type)}, \ - {CC "get" #Type "Volatile", CC "(" OBJ "J)" #Desc, FN_PTR(Unsafe_Get##Type##Volatile)}, \ - {CC "put" #Type "Volatile", CC "(" OBJ "J" #Desc ")V", FN_PTR(Unsafe_Put##Type##Volatile)} - - static JNINativeMethod jdk_internal_misc_Unsafe_methods[] = { - {CC "getReference", CC "(" OBJ "J)" OBJ "", FN_PTR(Unsafe_GetReference)}, - {CC "putReference", CC "(" OBJ "J" OBJ ")V", FN_PTR(Unsafe_PutReference)}, - {CC "getReferenceVolatile", CC "(" OBJ "J)" OBJ, FN_PTR(Unsafe_GetReferenceVolatile)}, - {CC "putReferenceVolatile", CC "(" OBJ "J" OBJ ")V", FN_PTR(Unsafe_PutReferenceVolatile)}, + {CC "getReferenceMO", CC "(B" OBJ "J)" OBJ "", FN_PTR(Unsafe_GetReferenceMO)}, + {CC "getPrimitiveBitsMONative", CC "(BB" OBJ "J)" "J", FN_PTR(Unsafe_GetPrimitiveBitsMO)}, + {CC "putReferenceMO", CC "(B" OBJ "J" OBJ ")V", FN_PTR(Unsafe_PutReferenceMO)}, + {CC "putPrimitiveBitsMONative", CC "(BB" OBJ "J" "J" ")V", FN_PTR(Unsafe_PutPrimitiveBitsMO)}, - {CC "getUncompressedObject", CC "(" ADR ")" OBJ, FN_PTR(Unsafe_GetUncompressedObject)}, + {CC "compareAndSetReferenceMO", CC "(B" OBJ "J" OBJ "" OBJ ")" "Z", FN_PTR(Unsafe_CompareAndSetReferenceMO)}, + {CC "compareAndSetPrimitiveBitsMONative", CC "(BB" OBJ "J" "J" "J" ")" "Z", FN_PTR(Unsafe_CompareAndSetPrimitiveBitsMO)}, + {CC "compareAndExchangeReferenceMO", CC "(B" OBJ "J" OBJ "" OBJ ")" OBJ, FN_PTR(Unsafe_CompareAndExchangeReferenceMO)}, + {CC "compareAndExchangePrimitiveBitsMONative", CC "(BB" OBJ "J" "J" "J" ")" "J", FN_PTR(Unsafe_CompareAndExchangePrimitiveBitsMO)}, + // "getAndOperatePrimitiveBitsMO" has a portable fallback coded in Java - DECLARE_GETPUTOOP(Boolean, Z), - DECLARE_GETPUTOOP(Byte, B), - DECLARE_GETPUTOOP(Short, S), - DECLARE_GETPUTOOP(Char, C), - DECLARE_GETPUTOOP(Int, I), - DECLARE_GETPUTOOP(Long, J), - DECLARE_GETPUTOOP(Float, F), - DECLARE_GETPUTOOP(Double, D), + {CC "getUncompressedObject", CC "(" ADR ")" OBJ, FN_PTR(Unsafe_GetUncompressedObject)}, {CC "allocateMemory0", CC "(J)" ADR, FN_PTR(Unsafe_AllocateMemory0)}, {CC "reallocateMemory0", CC "(" ADR "J)" ADR, FN_PTR(Unsafe_ReallocateMemory0)}, @@ -898,12 +1023,6 @@ static JNINativeMethod jdk_internal_misc_Unsafe_methods[] = { {CC "defineClass0", CC "(" DC_Args ")" CLS, FN_PTR(Unsafe_DefineClass0)}, {CC "allocateInstance", CC "(" CLS ")" OBJ, FN_PTR(Unsafe_AllocateInstance)}, {CC "throwException", CC "(" THR ")V", FN_PTR(Unsafe_ThrowException)}, - {CC "compareAndSetReference",CC "(" OBJ "J" OBJ "" OBJ ")Z", FN_PTR(Unsafe_CompareAndSetReference)}, - {CC "compareAndSetInt", CC "(" OBJ "J""I""I"")Z", FN_PTR(Unsafe_CompareAndSetInt)}, - {CC "compareAndSetLong", CC "(" OBJ "J""J""J"")Z", FN_PTR(Unsafe_CompareAndSetLong)}, - {CC "compareAndExchangeReference", CC "(" OBJ "J" OBJ "" OBJ ")" OBJ, FN_PTR(Unsafe_CompareAndExchangeReference)}, - {CC "compareAndExchangeInt", CC "(" OBJ "J""I""I"")I", FN_PTR(Unsafe_CompareAndExchangeInt)}, - {CC "compareAndExchangeLong", CC "(" OBJ "J""J""J"")J", FN_PTR(Unsafe_CompareAndExchangeLong)}, {CC "park", CC "(ZJ)V", FN_PTR(Unsafe_Park)}, {CC "unpark", CC "(" OBJ ")V", FN_PTR(Unsafe_Unpark)}, @@ -942,9 +1061,56 @@ static JNINativeMethod jdk_internal_misc_Unsafe_methods[] = { // The optimizer looks at names and signatures to recognize // individual functions. +static void check_static_constant(JavaThread* thread, InstanceKlass* uk, + const char* name, int value) { + int fieldcv = value == -1 ? 0 : -1; // force mismatch if not changed + TempNewSymbol fname = SymbolTable::probe(name, strlen(name)); + if (fname != nullptr) { + fieldDescriptor fd; + if (uk->find_local_field(fname, vmSymbols::byte_signature(), &fd)) { + if (fd.has_initial_value()) { + fieldcv = fd.int_initial_value(); + } + } + } + guarantee(fieldcv == value, + "mismatch on Unsafe.%s, %d vs. %d", name, value, fieldcv); +} + +static void check_unsafe_constants(JavaThread* thread, jclass unsafeclass) { + InstanceKlass* uk = java_lang_Class::as_InstanceKlass(JNIHandles::resolve_non_null(unsafeclass)); + + #define UNSAFE_BASIC_TYPE_CHECK(bt) \ + check_static_constant(thread, uk, "B" #bt, bt) + UNSAFE_BASIC_TYPE_CHECK(T_BYTE); + UNSAFE_BASIC_TYPE_CHECK(T_BOOLEAN); + UNSAFE_BASIC_TYPE_CHECK(T_CHAR); + UNSAFE_BASIC_TYPE_CHECK(T_FLOAT); + UNSAFE_BASIC_TYPE_CHECK(T_DOUBLE); + UNSAFE_BASIC_TYPE_CHECK(T_BYTE); + UNSAFE_BASIC_TYPE_CHECK(T_SHORT); + UNSAFE_BASIC_TYPE_CHECK(T_INT); + UNSAFE_BASIC_TYPE_CHECK(T_LONG); + + #define VMI_MEMORY_ORDER_CHECK(mo, ignore) \ + check_static_constant(thread, uk, #mo, vmIntrinsics::mo); + VMI_MEMORY_ORDERS_DO(VMI_MEMORY_ORDER_CHECK) + + #define VMI_PRIMITIVE_BITS_OPERATION_CHECK(op, ignore) \ + check_static_constant(thread, uk, #op, vmIntrinsics::op); + VMI_PRIMITIVE_BITS_OPERATIONS_DO(VMI_PRIMITIVE_BITS_OPERATION_CHECK) + + check_static_constant(thread, uk, "PRIMITIVE_SIZE_MASK", UNSAFE_PRIMITIVE_SIZE_MASK); +} + JVM_ENTRY(void, JVM_RegisterJDKInternalMiscUnsafeMethods(JNIEnv *env, jclass unsafeclass)) { ThreadToNativeFromVM ttnfv(thread); int ok = env->RegisterNatives(unsafeclass, jdk_internal_misc_Unsafe_methods, sizeof(jdk_internal_misc_Unsafe_methods)/sizeof(JNINativeMethod)); guarantee(ok == 0, "register jdk.internal.misc.Unsafe natives"); + + { + ThreadInVMfromNative tivfn(thread); + check_unsafe_constants(thread, unsafeclass); // do this bit in VM mode + } } JVM_END diff --git a/src/hotspot/share/prims/unsafe.hpp b/src/hotspot/share/prims/unsafe.hpp index 017b4ce49bb86..67fc78fc43bae 100644 --- a/src/hotspot/share/prims/unsafe.hpp +++ b/src/hotspot/share/prims/unsafe.hpp @@ -36,4 +36,46 @@ jlong Unsafe_field_offset_to_byte_offset(jlong field_offset); jlong Unsafe_field_offset_from_byte_offset(jlong byte_offset); +// The low three bits of the 8 primitive BasicType values encode size. +constexpr int UNSAFE_PRIMITIVE_SIZE_MASK = 3; + +// These are defined as byte constants in the Unsafe class. +// They are used as leading arguments to Unsafe.getReferenceMO, etc. +#define UNSAFE_MEMORY_ORDERS_DO(fn) \ + fn(MO_PLAIN, 1) \ + fn(MO_VOLATILE, 2) \ + fn(MO_ACQUIRE, 4) \ + fn(MO_RELEASE, 8) \ + fn(MO_WEAK_CAS, 16) \ + fn(MO_UNALIGNED, 32) \ + fn(MO_OPAQUE, 3) /*plain+voltile*/ \ + /*end*/ + +enum UnsafeMemoryOrder { + #define UNSAFE_MEMORY_ORDER_DEFINE(mo, code) \ + UNSAFE_ ## mo = code, + UNSAFE_MEMORY_ORDERS_DO(UNSAFE_MEMORY_ORDER_DEFINE) + #undef UNSAFE_MEMORY_ORDER_DEFINE + UNSAFE_MO_MODE_MASK = UNSAFE_MO_PLAIN|UNSAFE_MO_VOLATILE|UNSAFE_MO_ACQUIRE|UNSAFE_MO_RELEASE, +}; + +// These are defined as byte constants in the Unsafe class. +// They are used only by Unsafe.getAndOperatePrimitiveBitsMO. +#define UNSAFE_PRIMITIVE_BITS_OPERATIONS_DO(fn) \ + fn(OP_ADD, '+') \ + fn(OP_BITAND, '&') \ + fn(OP_BITOR, '|') \ + fn(OP_BITXOR, '^') \ + fn(OP_SWAP, '=') \ + /*end*/ + +enum UnsafePrimitiveBitsOperation { + #define UNSAFE_PRIMITIVE_BITS_OPERATION_DEFINE(op, code) \ + UNSAFE_ ## op = code, + UNSAFE_PRIMITIVE_BITS_OPERATIONS_DO(UNSAFE_PRIMITIVE_BITS_OPERATION_DEFINE) + #undef UNSAFE_PRIMITIVE_BITS_OPERATION_DEFINE +}; +// The high level of macro-abstraction is intended to assist us in +// validating that the constants are the same in Java as in C++. + #endif // SHARE_PRIMS_UNSAFE_HPP diff --git a/src/hotspot/share/runtime/atomicAccess.hpp b/src/hotspot/share/runtime/atomicAccess.hpp index c9a2dfb9383a7..39d5d6484a8cc 100644 --- a/src/hotspot/share/runtime/atomicAccess.hpp +++ b/src/hotspot/share/runtime/atomicAccess.hpp @@ -367,7 +367,7 @@ class AtomicAccess : AllStatic { struct CmpxchgImpl; // Platform-specific implementation of cmpxchg. Support for sizes - // of 1, 4, and 8 are required. The class is a function object that + // of 1, 2, 4, and 8 are required. The class is a function object that // must be default constructable, with these requirements: // // - dest is of type T*. @@ -404,7 +404,8 @@ class AtomicAccess : AllStatic { // accesses for 1-byte and 8-byte widths. To use, derive PlatformCmpxchg<1>, // PlatformAdd, PlatformXchg from these classes. public: // Temporary, can't be private: C++03 11.4/2. Fixed by C++11. - struct CmpxchgByteUsingInt; + template + struct CmpxchgSubwordUsingInt; template struct XchgUsingCmpxchg; template @@ -419,7 +420,7 @@ class AtomicAccess : AllStatic { struct XchgImpl; // Platform-specific implementation of xchg. Support for sizes - // of 1, 4, and 8 are required. The class is a function object + // of 1, 2, 4, and 8 are required. The class is a function object // that must be default constructable, with these requirements: // // - dest is of type T*. @@ -671,9 +672,12 @@ struct AtomicAccess::PlatformCmpxchg { // Define the class before including platform file, which may use this // as a base class, requiring it be complete. The definition is later // in this file, near the other definitions related to cmpxchg. -struct AtomicAccess::CmpxchgByteUsingInt { - static uint8_t get_byte_in_int(uint32_t n, uint32_t idx); - static uint32_t set_byte_in_int(uint32_t n, uint8_t b, uint32_t idx); +template +struct AtomicAccess::CmpxchgSubwordUsingInt { + template + static T get_subword_in_int(uint32_t n, uint32_t idx); + template + static uint32_t set_subword_in_int(uint32_t n, T sw, uint32_t idx); template T operator()(T volatile* dest, T compare_value, @@ -718,6 +722,7 @@ class AtomicAccess::AddUsingCmpxchg { old_value = AtomicAccess::load(dest); new_value = old_value + add_value; } while (old_value != AtomicAccess::cmpxchg(dest, old_value, new_value, order)); + STATIC_ASSERT(!std::is_floating_point::value); //FP messes up while(x!=y) return old_value; } }; @@ -1092,28 +1097,36 @@ inline T AtomicAccess::cmpxchg_using_helper(Fn fn, PrimitiveConversions::cast(compare_value))); } -inline uint32_t AtomicAccess::CmpxchgByteUsingInt::set_byte_in_int(uint32_t n, - uint8_t b, - uint32_t idx) { +template +template +inline uint32_t AtomicAccess::CmpxchgSubwordUsingInt::set_subword_in_int(uint32_t n, + T sw, + uint32_t idx) { + STATIC_ASSERT(sizeof(T) == byte_size); uint32_t bitsIdx = BitsPerByte * idx; - return (n & ~(static_cast(0xff) << bitsIdx)) - | (static_cast(b) << bitsIdx); + uint32_t bitSize = BitsPerByte * byte_size; + return (n & ~(static_cast(right_n_bits(bitSize)) << bitsIdx)) + | (static_cast(sw) << bitsIdx); } -inline uint8_t AtomicAccess::CmpxchgByteUsingInt::get_byte_in_int(uint32_t n, - uint32_t idx) { +template +template +inline T AtomicAccess::CmpxchgSubwordUsingInt::get_subword_in_int(uint32_t n, + uint32_t idx) { + STATIC_ASSERT(sizeof(T) == byte_size); uint32_t bitsIdx = BitsPerByte * idx; - return (uint8_t)(n >> bitsIdx); + return (T)(n >> bitsIdx); } +template template -inline T AtomicAccess::CmpxchgByteUsingInt::operator()(T volatile* dest, - T compare_value, - T exchange_value, - atomic_memory_order order) const { - STATIC_ASSERT(sizeof(T) == sizeof(uint8_t)); - uint8_t canon_exchange_value = exchange_value; - uint8_t canon_compare_value = compare_value; +inline T AtomicAccess::CmpxchgSubwordUsingInt::operator()(T volatile* dest, + T compare_value, + T exchange_value, + atomic_memory_order order) const { + STATIC_ASSERT(sizeof(T) == byte_size); + T canon_exchange_value = exchange_value; + T canon_compare_value = compare_value; volatile uint32_t* aligned_dest = reinterpret_cast(align_down(dest, sizeof(uint32_t))); uint32_t offset = checked_cast(pointer_delta(dest, aligned_dest, 1)); @@ -1124,14 +1137,14 @@ inline T AtomicAccess::CmpxchgByteUsingInt::operator()(T volatile* dest, // current value may not be what we are looking for, so force it // to that value so the initial cmpxchg will fail if it is different - uint32_t cur = set_byte_in_int(AtomicAccess::load(aligned_dest), canon_compare_value, idx); + uint32_t cur = set_subword_in_int(AtomicAccess::load(aligned_dest), canon_compare_value, idx); // always execute a real cmpxchg so that we get the required memory // barriers even on initial failure do { // value to swap in matches current value - // except for the one byte we want to update - uint32_t new_value = set_byte_in_int(cur, canon_exchange_value, idx); + // except for the one or two bytes we want to update + uint32_t new_value = set_subword_in_int(cur, canon_exchange_value, idx); uint32_t res = cmpxchg(aligned_dest, cur, new_value, order); if (res == cur) break; // success @@ -1140,9 +1153,10 @@ inline T AtomicAccess::CmpxchgByteUsingInt::operator()(T volatile* dest, // our view of the current int cur = res; // if our byte is still as cur we loop and try again - } while (get_byte_in_int(cur, idx) == canon_compare_value); + } while (get_subword_in_int(cur, idx) == canon_compare_value); - return PrimitiveConversions::cast(get_byte_in_int(cur, idx)); + STATIC_ASSERT(!std::is_floating_point::value); //FP messes up while(x!=y) + return get_subword_in_int(cur, idx); } // Handle xchg for integral types. @@ -1226,6 +1240,7 @@ inline T AtomicAccess::XchgUsingCmpxchg::operator()(T volatile* dest, do { old_value = AtomicAccess::load(dest); } while (old_value != AtomicAccess::cmpxchg(dest, old_value, exchange_value, order)); + STATIC_ASSERT(!std::is_floating_point::value); //FP messes up while(x!=y) return old_value; } diff --git a/src/hotspot/share/utilities/tribool.hpp b/src/hotspot/share/utilities/tribool.hpp index c61ab1d4d4441..984fa3b37d332 100644 --- a/src/hotspot/share/utilities/tribool.hpp +++ b/src/hotspot/share/utilities/tribool.hpp @@ -39,12 +39,12 @@ class TriBool{ explicit TriBool(u1 raw) : _value(raw & 3) {} public: - TriBool() : _value(0) {} - TriBool(bool value) : _value(value) { + constexpr TriBool() : _value(0) {} + constexpr TriBool(bool value) : _value(value) { // set to not-default in separate step to avoid conversion warnings _value |= 2; } - TriBool(const TriBool& o): _value(o._value) {} + constexpr TriBool(const TriBool& o): _value(o._value) {} TriBool& operator=(bool value) { _value = value; @@ -64,6 +64,10 @@ class TriBool{ /*explicit*/ operator bool() const { return (_value & 1); } + + bool match(bool value) { + return is_default() || value == (bool)*this; + } }; // compacted array of TriBool diff --git a/src/java.base/share/classes/java/io/File.java b/src/java.base/share/classes/java/io/File.java index 494dac7f51e23..e7bc4da48feca 100644 --- a/src/java.base/share/classes/java/io/File.java +++ b/src/java.base/share/classes/java/io/File.java @@ -35,6 +35,7 @@ import java.security.SecureRandom; import java.util.ArrayList; import java.util.List; +import jdk.internal.misc.Unsafe; import jdk.internal.util.StaticProperty; /** @@ -2057,11 +2058,11 @@ private synchronized void readObject(java.io.ObjectInputStream s) pathField = pathField.replace(sep, separatorChar); String path = FS.normalize(pathField); UNSAFE.putReference(this, PATH_OFFSET, path); - UNSAFE.putIntVolatile(this, PREFIX_LENGTH_OFFSET, FS.prefixLength(path)); + UNSAFE.putIntMO(Unsafe.MO_VOLATILE, this, PREFIX_LENGTH_OFFSET, FS.prefixLength(path)); } private static final jdk.internal.misc.Unsafe UNSAFE - = jdk.internal.misc.Unsafe.getUnsafe(); + = Unsafe.getUnsafe(); private static final long PATH_OFFSET = UNSAFE.objectFieldOffset(File.class, "path"); private static final long PREFIX_LENGTH_OFFSET diff --git a/src/java.base/share/classes/java/lang/Module.java b/src/java.base/share/classes/java/lang/Module.java index cd2b8095ee48e..bb7b14a5cb78b 100644 --- a/src/java.base/share/classes/java/lang/Module.java +++ b/src/java.base/share/classes/java/lang/Module.java @@ -273,7 +273,7 @@ private EnableNativeAccess() {} private static final long FIELD_OFFSET = UNSAFE.objectFieldOffset(Module.class, "enableNativeAccess"); private static boolean isNativeAccessEnabled(Module target) { - return UNSAFE.getBooleanVolatile(target, FIELD_OFFSET); + return UNSAFE.getBooleanMO(Unsafe.MO_VOLATILE, target, FIELD_OFFSET); } // Atomically sets enableNativeAccess if not already set @@ -350,7 +350,7 @@ private static final class EnableFinalMutation { UNSAFE.objectFieldOffset(Module.class, "enableFinalMutation"); private static boolean isEnableFinalMutation(Module module) { - return UNSAFE.getBooleanVolatile(module, ENABLE_FINAL_MUTATION_OFFSET); + return UNSAFE.getBooleanMO(Unsafe.MO_VOLATILE, module, ENABLE_FINAL_MUTATION_OFFSET); } private static boolean tryEnableFinalMutation(Module module) { diff --git a/src/java.base/share/classes/java/lang/invoke/CallSite.java b/src/java.base/share/classes/java/lang/invoke/CallSite.java index e9e3477f96fd8..d73afbbe61669 100644 --- a/src/java.base/share/classes/java/lang/invoke/CallSite.java +++ b/src/java.base/share/classes/java/lang/invoke/CallSite.java @@ -29,6 +29,7 @@ import static java.lang.invoke.MethodHandles.Lookup.IMPL_LOOKUP; import jdk.internal.vm.annotation.Stable; +import jdk.internal.misc.Unsafe; /** * A {@code CallSite} is a holder for a variable {@link MethodHandle}, @@ -287,7 +288,7 @@ final void setTargetNormal(MethodHandle newTarget) { /*package-private*/ final MethodHandle getTargetVolatile() { - return (MethodHandle) UNSAFE.getReferenceVolatile(this, getTargetOffset()); + return (MethodHandle) UNSAFE.getReferenceMO(Unsafe.MO_VOLATILE, this, getTargetOffset()); } /*package-private*/ diff --git a/src/java.base/share/classes/java/lang/invoke/DirectMethodHandle.java b/src/java.base/share/classes/java/lang/invoke/DirectMethodHandle.java index b5738ad6fba81..f1ee2bda52dcf 100644 --- a/src/java.base/share/classes/java/lang/invoke/DirectMethodHandle.java +++ b/src/java.base/share/classes/java/lang/invoke/DirectMethodHandle.java @@ -737,7 +737,7 @@ private static String unsafeMethodName(boolean isGetter, boolean isVolatile, Wra .append(isGetter ? "get" : "put") .append(name); if (isVolatile) { - sb.append("Volatile"); + sb.append("MO"); } return sb.toString(); } @@ -761,6 +761,10 @@ static LambdaForm makePreparedFieldLambdaForm(byte formOp, boolean isVolatile, i linkerType = MethodType.methodType(ft, Object.class, long.class); else linkerType = MethodType.methodType(void.class, Object.class, long.class, ft); + final int prefixLen = isVolatile ? 1 : 0; + if (prefixLen > 0) { + linkerType = linkerType.insertParameterTypes(0, byte.class); + } MemberName linker = new MemberName(Unsafe.class, unsafeMethodName, linkerType, REF_invokeVirtual); try { linker = IMPL_NAMES.resolveOrFail(REF_invokeVirtual, linker, null, LM_TRUSTED, @@ -803,15 +807,19 @@ static LambdaForm makePreparedFieldLambdaForm(byte formOp, boolean isVolatile, i Object[] outArgs = new Object[1 + linkerType.parameterCount()]; assert(outArgs.length == (isGetter ? 3 : 4)); outArgs[0] = names[U_HOLDER] = new Name(getFunction(NF_UNSAFE)); + if (prefixLen > 0) { + outArgs[1] = (int)(isVolatile ? Unsafe.MO_VOLATILE : Unsafe.MO_PLAIN); + // For some other uses, a second prefix argument might be BT_INT, BT_BYTE, etc. + } if (isStatic) { - outArgs[1] = names[F_HOLDER] = new Name(getFunction(NF_staticBase), names[DMH_THIS]); - outArgs[2] = names[F_OFFSET] = new Name(getFunction(NF_staticOffset), names[DMH_THIS]); + outArgs[prefixLen+1] = names[F_HOLDER] = new Name(getFunction(NF_staticBase), names[DMH_THIS]); + outArgs[prefixLen+2] = names[F_OFFSET] = new Name(getFunction(NF_staticOffset), names[DMH_THIS]); } else { - outArgs[1] = names[OBJ_CHECK] = new Name(getFunction(NF_checkBase), names[OBJ_BASE]); - outArgs[2] = names[F_OFFSET] = new Name(getFunction(NF_fieldOffset), names[DMH_THIS]); + outArgs[prefixLen+1] = names[OBJ_CHECK] = new Name(getFunction(NF_checkBase), names[OBJ_BASE]); + outArgs[prefixLen+2] = names[F_OFFSET] = new Name(getFunction(NF_fieldOffset), names[DMH_THIS]); } if (!isGetter) { - outArgs[3] = (needsCast ? names[PRE_CAST] : names[SET_VALUE]); + outArgs[prefixLen+3] = (needsCast ? names[PRE_CAST] : names[SET_VALUE]); } for (Object a : outArgs) assert(a != null); names[LINKER_CALL] = new Name(linker, outArgs); diff --git a/src/java.base/share/classes/java/lang/invoke/MethodHandle.java b/src/java.base/share/classes/java/lang/invoke/MethodHandle.java index 99d6dbe19a5a1..de26b07bd81db 100644 --- a/src/java.base/share/classes/java/lang/invoke/MethodHandle.java +++ b/src/java.base/share/classes/java/lang/invoke/MethodHandle.java @@ -27,6 +27,7 @@ import jdk.internal.loader.ClassLoaders; +import jdk.internal.misc.Unsafe; import jdk.internal.vm.annotation.AOTSafeClassInitializer; import jdk.internal.vm.annotation.DontInline; import jdk.internal.vm.annotation.ForceInline; @@ -1893,7 +1894,7 @@ void updateForm(Function updater) { if (oldForm != newForm) { assert (newForm.customized == null || newForm.customized == this); newForm.prepare(); // as in MethodHandle. - UNSAFE.putReferenceRelease(this, FORM_OFFSET, newForm); // properly publish newForm + UNSAFE.putReferenceMO(Unsafe.MO_RELEASE, this, FORM_OFFSET, newForm); // properly publish newForm } } finally { updateInProgress = false; diff --git a/src/java.base/share/classes/java/lang/invoke/X-VarHandle.java.template b/src/java.base/share/classes/java/lang/invoke/X-VarHandle.java.template index 4bc9daf84335e..024413b0f481a 100644 --- a/src/java.base/share/classes/java/lang/invoke/X-VarHandle.java.template +++ b/src/java.base/share/classes/java/lang/invoke/X-VarHandle.java.template @@ -24,6 +24,7 @@ */ package java.lang.invoke; +import jdk.internal.misc.Unsafe; import jdk.internal.util.Preconditions; import jdk.internal.vm.annotation.ForceInline; @@ -100,21 +101,21 @@ final class VarHandle$Type$s { @ForceInline static $type$ getVolatile(VarHandle ob, Object holder) { FieldInstanceReadOnly handle = (FieldInstanceReadOnly)ob; - return UNSAFE.get$Type$Volatile(Objects.requireNonNull(handle.receiverType.cast(holder)), + return UNSAFE.get$Type$MO(Unsafe.MO_VOLATILE, Objects.requireNonNull(handle.receiverType.cast(holder)), handle.fieldOffset); } @ForceInline static $type$ getOpaque(VarHandle ob, Object holder) { FieldInstanceReadOnly handle = (FieldInstanceReadOnly)ob; - return UNSAFE.get$Type$Opaque(Objects.requireNonNull(handle.receiverType.cast(holder)), + return UNSAFE.get$Type$MO(Unsafe.MO_OPAQUE, Objects.requireNonNull(handle.receiverType.cast(holder)), handle.fieldOffset); } @ForceInline static $type$ getAcquire(VarHandle ob, Object holder) { FieldInstanceReadOnly handle = (FieldInstanceReadOnly)ob; - return UNSAFE.get$Type$Acquire(Objects.requireNonNull(handle.receiverType.cast(holder)), + return UNSAFE.get$Type$MO(Unsafe.MO_ACQUIRE, Objects.requireNonNull(handle.receiverType.cast(holder)), handle.fieldOffset); } @@ -157,7 +158,7 @@ final class VarHandle$Type$s { @ForceInline static void setVolatile(VarHandle ob, Object holder, $type$ value) { FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob; - UNSAFE.put$Type$Volatile(Objects.requireNonNull(handle.receiverType.cast(holder)), + UNSAFE.put$Type$MO(Unsafe.MO_VOLATILE, Objects.requireNonNull(handle.receiverType.cast(holder)), handle.fieldOffset, {#if[Object]?handle.fieldType.cast(value):value}); } @@ -165,7 +166,7 @@ final class VarHandle$Type$s { @ForceInline static void setOpaque(VarHandle ob, Object holder, $type$ value) { FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob; - UNSAFE.put$Type$Opaque(Objects.requireNonNull(handle.receiverType.cast(holder)), + UNSAFE.put$Type$MO(Unsafe.MO_OPAQUE, Objects.requireNonNull(handle.receiverType.cast(holder)), handle.fieldOffset, {#if[Object]?handle.fieldType.cast(value):value}); } @@ -173,7 +174,7 @@ final class VarHandle$Type$s { @ForceInline static void setRelease(VarHandle ob, Object holder, $type$ value) { FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob; - UNSAFE.put$Type$Release(Objects.requireNonNull(handle.receiverType.cast(holder)), + UNSAFE.put$Type$MO(Unsafe.MO_RELEASE, Objects.requireNonNull(handle.receiverType.cast(holder)), handle.fieldOffset, {#if[Object]?handle.fieldType.cast(value):value}); } @@ -200,7 +201,8 @@ final class VarHandle$Type$s { @ForceInline static $type$ compareAndExchangeAcquire(VarHandle ob, Object holder, $type$ expected, $type$ value) { FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob; - return UNSAFE.compareAndExchange$Type$Acquire(Objects.requireNonNull(handle.receiverType.cast(holder)), + return UNSAFE.compareAndExchange$Type$MO(Unsafe.MO_ACQUIRE, + Objects.requireNonNull(handle.receiverType.cast(holder)), handle.fieldOffset, {#if[Object]?handle.fieldType.cast(expected):expected}, {#if[Object]?handle.fieldType.cast(value):value}); @@ -209,7 +211,8 @@ final class VarHandle$Type$s { @ForceInline static $type$ compareAndExchangeRelease(VarHandle ob, Object holder, $type$ expected, $type$ value) { FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob; - return UNSAFE.compareAndExchange$Type$Release(Objects.requireNonNull(handle.receiverType.cast(holder)), + return UNSAFE.compareAndExchange$Type$MO(Unsafe.MO_RELEASE, + Objects.requireNonNull(handle.receiverType.cast(holder)), handle.fieldOffset, {#if[Object]?handle.fieldType.cast(expected):expected}, {#if[Object]?handle.fieldType.cast(value):value}); @@ -218,7 +221,8 @@ final class VarHandle$Type$s { @ForceInline static boolean weakCompareAndSetPlain(VarHandle ob, Object holder, $type$ expected, $type$ value) { FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob; - return UNSAFE.weakCompareAndSet$Type$Plain(Objects.requireNonNull(handle.receiverType.cast(holder)), + return UNSAFE.compareAndSet$Type$MO(Unsafe.MO_WEAK_CAS_PLAIN, + Objects.requireNonNull(handle.receiverType.cast(holder)), handle.fieldOffset, {#if[Object]?handle.fieldType.cast(expected):expected}, {#if[Object]?handle.fieldType.cast(value):value}); @@ -227,7 +231,8 @@ final class VarHandle$Type$s { @ForceInline static boolean weakCompareAndSet(VarHandle ob, Object holder, $type$ expected, $type$ value) { FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob; - return UNSAFE.weakCompareAndSet$Type$(Objects.requireNonNull(handle.receiverType.cast(holder)), + return UNSAFE.compareAndSet$Type$MO(Unsafe.MO_WEAK_CAS_VOLATILE, + Objects.requireNonNull(handle.receiverType.cast(holder)), handle.fieldOffset, {#if[Object]?handle.fieldType.cast(expected):expected}, {#if[Object]?handle.fieldType.cast(value):value}); @@ -236,7 +241,8 @@ final class VarHandle$Type$s { @ForceInline static boolean weakCompareAndSetAcquire(VarHandle ob, Object holder, $type$ expected, $type$ value) { FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob; - return UNSAFE.weakCompareAndSet$Type$Acquire(Objects.requireNonNull(handle.receiverType.cast(holder)), + return UNSAFE.compareAndSet$Type$MO(Unsafe.MO_WEAK_CAS_ACQUIRE, + Objects.requireNonNull(handle.receiverType.cast(holder)), handle.fieldOffset, {#if[Object]?handle.fieldType.cast(expected):expected}, {#if[Object]?handle.fieldType.cast(value):value}); @@ -245,7 +251,8 @@ final class VarHandle$Type$s { @ForceInline static boolean weakCompareAndSetRelease(VarHandle ob, Object holder, $type$ expected, $type$ value) { FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob; - return UNSAFE.weakCompareAndSet$Type$Release(Objects.requireNonNull(handle.receiverType.cast(holder)), + return UNSAFE.compareAndSet$Type$MO(Unsafe.MO_WEAK_CAS_RELEASE, + Objects.requireNonNull(handle.receiverType.cast(holder)), handle.fieldOffset, {#if[Object]?handle.fieldType.cast(expected):expected}, {#if[Object]?handle.fieldType.cast(value):value}); @@ -262,7 +269,8 @@ final class VarHandle$Type$s { @ForceInline static $type$ getAndSetAcquire(VarHandle ob, Object holder, $type$ value) { FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob; - return UNSAFE.getAndSet$Type$Acquire(Objects.requireNonNull(handle.receiverType.cast(holder)), + return UNSAFE.getAndSet$Type$MO(Unsafe.MO_ACQUIRE, + Objects.requireNonNull(handle.receiverType.cast(holder)), handle.fieldOffset, {#if[Object]?handle.fieldType.cast(value):value}); } @@ -270,7 +278,8 @@ final class VarHandle$Type$s { @ForceInline static $type$ getAndSetRelease(VarHandle ob, Object holder, $type$ value) { FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob; - return UNSAFE.getAndSet$Type$Release(Objects.requireNonNull(handle.receiverType.cast(holder)), + return UNSAFE.getAndSet$Type$MO(Unsafe.MO_RELEASE, + Objects.requireNonNull(handle.receiverType.cast(holder)), handle.fieldOffset, {#if[Object]?handle.fieldType.cast(value):value}); } @@ -288,7 +297,8 @@ final class VarHandle$Type$s { @ForceInline static $type$ getAndAddAcquire(VarHandle ob, Object holder, $type$ value) { FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob; - return UNSAFE.getAndAdd$Type$Acquire(Objects.requireNonNull(handle.receiverType.cast(holder)), + return UNSAFE.getAndAdd$Type$MO(Unsafe.MO_ACQUIRE, + Objects.requireNonNull(handle.receiverType.cast(holder)), handle.fieldOffset, value); } @@ -296,7 +306,8 @@ final class VarHandle$Type$s { @ForceInline static $type$ getAndAddRelease(VarHandle ob, Object holder, $type$ value) { FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob; - return UNSAFE.getAndAdd$Type$Release(Objects.requireNonNull(handle.receiverType.cast(holder)), + return UNSAFE.getAndAdd$Type$MO(Unsafe.MO_RELEASE, + Objects.requireNonNull(handle.receiverType.cast(holder)), handle.fieldOffset, value); } @@ -315,7 +326,8 @@ final class VarHandle$Type$s { @ForceInline static $type$ getAndBitwiseOrRelease(VarHandle ob, Object holder, $type$ value) { FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob; - return UNSAFE.getAndBitwiseOr$Type$Release(Objects.requireNonNull(handle.receiverType.cast(holder)), + return UNSAFE.getAndBitwiseOr$Type$MO(Unsafe.MO_RELEASE, + Objects.requireNonNull(handle.receiverType.cast(holder)), handle.fieldOffset, value); } @@ -323,7 +335,8 @@ final class VarHandle$Type$s { @ForceInline static $type$ getAndBitwiseOrAcquire(VarHandle ob, Object holder, $type$ value) { FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob; - return UNSAFE.getAndBitwiseOr$Type$Acquire(Objects.requireNonNull(handle.receiverType.cast(holder)), + return UNSAFE.getAndBitwiseOr$Type$MO(Unsafe.MO_ACQUIRE, + Objects.requireNonNull(handle.receiverType.cast(holder)), handle.fieldOffset, value); } @@ -339,7 +352,8 @@ final class VarHandle$Type$s { @ForceInline static $type$ getAndBitwiseAndRelease(VarHandle ob, Object holder, $type$ value) { FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob; - return UNSAFE.getAndBitwiseAnd$Type$Release(Objects.requireNonNull(handle.receiverType.cast(holder)), + return UNSAFE.getAndBitwiseAnd$Type$MO(Unsafe.MO_RELEASE, + Objects.requireNonNull(handle.receiverType.cast(holder)), handle.fieldOffset, value); } @@ -347,7 +361,8 @@ final class VarHandle$Type$s { @ForceInline static $type$ getAndBitwiseAndAcquire(VarHandle ob, Object holder, $type$ value) { FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob; - return UNSAFE.getAndBitwiseAnd$Type$Acquire(Objects.requireNonNull(handle.receiverType.cast(holder)), + return UNSAFE.getAndBitwiseAnd$Type$MO(Unsafe.MO_ACQUIRE, + Objects.requireNonNull(handle.receiverType.cast(holder)), handle.fieldOffset, value); } @@ -363,7 +378,8 @@ final class VarHandle$Type$s { @ForceInline static $type$ getAndBitwiseXorRelease(VarHandle ob, Object holder, $type$ value) { FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob; - return UNSAFE.getAndBitwiseXor$Type$Release(Objects.requireNonNull(handle.receiverType.cast(holder)), + return UNSAFE.getAndBitwiseXor$Type$MO(Unsafe.MO_RELEASE, + Objects.requireNonNull(handle.receiverType.cast(holder)), handle.fieldOffset, value); } @@ -371,7 +387,8 @@ final class VarHandle$Type$s { @ForceInline static $type$ getAndBitwiseXorAcquire(VarHandle ob, Object holder, $type$ value) { FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob; - return UNSAFE.getAndBitwiseXor$Type$Acquire(Objects.requireNonNull(handle.receiverType.cast(holder)), + return UNSAFE.getAndBitwiseXor$Type$MO(Unsafe.MO_ACQUIRE, + Objects.requireNonNull(handle.receiverType.cast(holder)), handle.fieldOffset, value); } @@ -448,21 +465,21 @@ final class VarHandle$Type$s { @ForceInline static $type$ getVolatile(VarHandle ob) { FieldStaticReadOnly handle = (FieldStaticReadOnly) ob.target(); - return UNSAFE.get$Type$Volatile(handle.base, + return UNSAFE.get$Type$MO(Unsafe.MO_VOLATILE, handle.base, handle.fieldOffset); } @ForceInline static $type$ getOpaque(VarHandle ob) { FieldStaticReadOnly handle = (FieldStaticReadOnly) ob.target(); - return UNSAFE.get$Type$Opaque(handle.base, + return UNSAFE.get$Type$MO(Unsafe.MO_OPAQUE, handle.base, handle.fieldOffset); } @ForceInline static $type$ getAcquire(VarHandle ob) { FieldStaticReadOnly handle = (FieldStaticReadOnly) ob.target(); - return UNSAFE.get$Type$Acquire(handle.base, + return UNSAFE.get$Type$MO(Unsafe.MO_ACQUIRE, handle.base, handle.fieldOffset); } @@ -505,7 +522,7 @@ final class VarHandle$Type$s { @ForceInline static void setVolatile(VarHandle ob, $type$ value) { FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target(); - UNSAFE.put$Type$Volatile(handle.base, + UNSAFE.put$Type$MO(Unsafe.MO_VOLATILE, handle.base, handle.fieldOffset, {#if[Object]?handle.fieldType.cast(value):value}); } @@ -513,7 +530,7 @@ final class VarHandle$Type$s { @ForceInline static void setOpaque(VarHandle ob, $type$ value) { FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target(); - UNSAFE.put$Type$Opaque(handle.base, + UNSAFE.put$Type$MO(Unsafe.MO_OPAQUE, handle.base, handle.fieldOffset, {#if[Object]?handle.fieldType.cast(value):value}); } @@ -521,7 +538,7 @@ final class VarHandle$Type$s { @ForceInline static void setRelease(VarHandle ob, $type$ value) { FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target(); - UNSAFE.put$Type$Release(handle.base, + UNSAFE.put$Type$MO(Unsafe.MO_RELEASE, handle.base, handle.fieldOffset, {#if[Object]?handle.fieldType.cast(value):value}); } @@ -549,7 +566,8 @@ final class VarHandle$Type$s { @ForceInline static $type$ compareAndExchangeAcquire(VarHandle ob, $type$ expected, $type$ value) { FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target(); - return UNSAFE.compareAndExchange$Type$Acquire(handle.base, + return UNSAFE.compareAndExchange$Type$MO(Unsafe.MO_ACQUIRE, + handle.base, handle.fieldOffset, {#if[Object]?handle.fieldType.cast(expected):expected}, {#if[Object]?handle.fieldType.cast(value):value}); @@ -558,7 +576,8 @@ final class VarHandle$Type$s { @ForceInline static $type$ compareAndExchangeRelease(VarHandle ob, $type$ expected, $type$ value) { FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target(); - return UNSAFE.compareAndExchange$Type$Release(handle.base, + return UNSAFE.compareAndExchange$Type$MO(Unsafe.MO_RELEASE, + handle.base, handle.fieldOffset, {#if[Object]?handle.fieldType.cast(expected):expected}, {#if[Object]?handle.fieldType.cast(value):value}); @@ -567,7 +586,8 @@ final class VarHandle$Type$s { @ForceInline static boolean weakCompareAndSetPlain(VarHandle ob, $type$ expected, $type$ value) { FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target(); - return UNSAFE.weakCompareAndSet$Type$Plain(handle.base, + return UNSAFE.compareAndSet$Type$MO(Unsafe.MO_WEAK_CAS_PLAIN, + handle.base, handle.fieldOffset, {#if[Object]?handle.fieldType.cast(expected):expected}, {#if[Object]?handle.fieldType.cast(value):value}); @@ -576,7 +596,8 @@ final class VarHandle$Type$s { @ForceInline static boolean weakCompareAndSet(VarHandle ob, $type$ expected, $type$ value) { FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target(); - return UNSAFE.weakCompareAndSet$Type$(handle.base, + return UNSAFE.compareAndSet$Type$MO(Unsafe.MO_WEAK_CAS_VOLATILE, + handle.base, handle.fieldOffset, {#if[Object]?handle.fieldType.cast(expected):expected}, {#if[Object]?handle.fieldType.cast(value):value}); @@ -585,7 +606,8 @@ final class VarHandle$Type$s { @ForceInline static boolean weakCompareAndSetAcquire(VarHandle ob, $type$ expected, $type$ value) { FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target(); - return UNSAFE.weakCompareAndSet$Type$Acquire(handle.base, + return UNSAFE.compareAndSet$Type$MO(Unsafe.MO_WEAK_CAS_ACQUIRE, + handle.base, handle.fieldOffset, {#if[Object]?handle.fieldType.cast(expected):expected}, {#if[Object]?handle.fieldType.cast(value):value}); @@ -594,7 +616,8 @@ final class VarHandle$Type$s { @ForceInline static boolean weakCompareAndSetRelease(VarHandle ob, $type$ expected, $type$ value) { FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target(); - return UNSAFE.weakCompareAndSet$Type$Release(handle.base, + return UNSAFE.compareAndSet$Type$MO(Unsafe.MO_WEAK_CAS_RELEASE, + handle.base, handle.fieldOffset, {#if[Object]?handle.fieldType.cast(expected):expected}, {#if[Object]?handle.fieldType.cast(value):value}); @@ -611,7 +634,8 @@ final class VarHandle$Type$s { @ForceInline static $type$ getAndSetAcquire(VarHandle ob, $type$ value) { FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target(); - return UNSAFE.getAndSet$Type$Acquire(handle.base, + return UNSAFE.getAndSet$Type$MO(Unsafe.MO_ACQUIRE, + handle.base, handle.fieldOffset, {#if[Object]?handle.fieldType.cast(value):value}); } @@ -619,7 +643,8 @@ final class VarHandle$Type$s { @ForceInline static $type$ getAndSetRelease(VarHandle ob, $type$ value) { FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target(); - return UNSAFE.getAndSet$Type$Release(handle.base, + return UNSAFE.getAndSet$Type$MO(Unsafe.MO_RELEASE, + handle.base, handle.fieldOffset, {#if[Object]?handle.fieldType.cast(value):value}); } @@ -637,7 +662,8 @@ final class VarHandle$Type$s { @ForceInline static $type$ getAndAddAcquire(VarHandle ob, $type$ value) { FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target(); - return UNSAFE.getAndAdd$Type$Acquire(handle.base, + return UNSAFE.getAndAdd$Type$MO(Unsafe.MO_ACQUIRE, + handle.base, handle.fieldOffset, value); } @@ -645,7 +671,8 @@ final class VarHandle$Type$s { @ForceInline static $type$ getAndAddRelease(VarHandle ob, $type$ value) { FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target(); - return UNSAFE.getAndAdd$Type$Release(handle.base, + return UNSAFE.getAndAdd$Type$MO(Unsafe.MO_RELEASE, + handle.base, handle.fieldOffset, value); } @@ -663,7 +690,8 @@ final class VarHandle$Type$s { @ForceInline static $type$ getAndBitwiseOrRelease(VarHandle ob, $type$ value) { FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target(); - return UNSAFE.getAndBitwiseOr$Type$Release(handle.base, + return UNSAFE.getAndBitwiseOr$Type$MO(Unsafe.MO_RELEASE, + handle.base, handle.fieldOffset, value); } @@ -671,7 +699,8 @@ final class VarHandle$Type$s { @ForceInline static $type$ getAndBitwiseOrAcquire(VarHandle ob, $type$ value) { FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target(); - return UNSAFE.getAndBitwiseOr$Type$Acquire(handle.base, + return UNSAFE.getAndBitwiseOr$Type$MO(Unsafe.MO_ACQUIRE, + handle.base, handle.fieldOffset, value); } @@ -687,7 +716,8 @@ final class VarHandle$Type$s { @ForceInline static $type$ getAndBitwiseAndRelease(VarHandle ob, $type$ value) { FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target(); - return UNSAFE.getAndBitwiseAnd$Type$Release(handle.base, + return UNSAFE.getAndBitwiseAnd$Type$MO(Unsafe.MO_RELEASE, + handle.base, handle.fieldOffset, value); } @@ -695,7 +725,8 @@ final class VarHandle$Type$s { @ForceInline static $type$ getAndBitwiseAndAcquire(VarHandle ob, $type$ value) { FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target(); - return UNSAFE.getAndBitwiseAnd$Type$Acquire(handle.base, + return UNSAFE.getAndBitwiseAnd$Type$MO(Unsafe.MO_ACQUIRE, + handle.base, handle.fieldOffset, value); } @@ -711,7 +742,8 @@ final class VarHandle$Type$s { @ForceInline static $type$ getAndBitwiseXorRelease(VarHandle ob, $type$ value) { FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target(); - return UNSAFE.getAndBitwiseXor$Type$Release(handle.base, + return UNSAFE.getAndBitwiseXor$Type$MO(Unsafe.MO_RELEASE, + handle.base, handle.fieldOffset, value); } @@ -719,7 +751,8 @@ final class VarHandle$Type$s { @ForceInline static $type$ getAndBitwiseXorAcquire(VarHandle ob, $type$ value) { FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target(); - return UNSAFE.getAndBitwiseXor$Type$Acquire(handle.base, + return UNSAFE.getAndBitwiseXor$Type$MO(Unsafe.MO_ACQUIRE, + handle.base, handle.fieldOffset, value); } @@ -831,7 +864,7 @@ final class VarHandle$Type$s { #else[Object] $type$[] array = ($type$[]) oarray; #end[Object] - return UNSAFE.get$Type$Volatile(array, + return UNSAFE.get$Type$MO(Unsafe.MO_VOLATILE, array, (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase); } @@ -843,7 +876,7 @@ final class VarHandle$Type$s { #else[Object] $type$[] array = ($type$[]) oarray; #end[Object] - UNSAFE.put$Type$Volatile(array, + UNSAFE.put$Type$MO(Unsafe.MO_VOLATILE, array, (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase, {#if[Object]?runtimeTypeCheck(handle, array, value):value}); } @@ -856,7 +889,7 @@ final class VarHandle$Type$s { #else[Object] $type$[] array = ($type$[]) oarray; #end[Object] - return UNSAFE.get$Type$Opaque(array, + return UNSAFE.get$Type$MO(Unsafe.MO_OPAQUE, array, (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase); } @@ -868,7 +901,7 @@ final class VarHandle$Type$s { #else[Object] $type$[] array = ($type$[]) oarray; #end[Object] - UNSAFE.put$Type$Opaque(array, + UNSAFE.put$Type$MO(Unsafe.MO_OPAQUE, array, (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase, {#if[Object]?runtimeTypeCheck(handle, array, value):value}); } @@ -881,7 +914,7 @@ final class VarHandle$Type$s { #else[Object] $type$[] array = ($type$[]) oarray; #end[Object] - return UNSAFE.get$Type$Acquire(array, + return UNSAFE.get$Type$MO(Unsafe.MO_ACQUIRE, array, (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase); } @@ -893,7 +926,7 @@ final class VarHandle$Type$s { #else[Object] $type$[] array = ($type$[]) oarray; #end[Object] - UNSAFE.put$Type$Release(array, + UNSAFE.put$Type$MO(Unsafe.MO_RELEASE, array, (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase, {#if[Object]?runtimeTypeCheck(handle, array, value):value}); } @@ -935,7 +968,8 @@ final class VarHandle$Type$s { #else[Object] $type$[] array = ($type$[]) oarray; #end[Object] - return UNSAFE.compareAndExchange$Type$Acquire(array, + return UNSAFE.compareAndExchange$Type$MO(Unsafe.MO_ACQUIRE, + array, (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase, {#if[Object]?handle.componentType.cast(expected):expected}, {#if[Object]?runtimeTypeCheck(handle, array, value):value}); @@ -949,7 +983,8 @@ final class VarHandle$Type$s { #else[Object] $type$[] array = ($type$[]) oarray; #end[Object] - return UNSAFE.compareAndExchange$Type$Release(array, + return UNSAFE.compareAndExchange$Type$MO(Unsafe.MO_RELEASE, + array, (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase, {#if[Object]?handle.componentType.cast(expected):expected}, {#if[Object]?runtimeTypeCheck(handle, array, value):value}); @@ -963,7 +998,8 @@ final class VarHandle$Type$s { #else[Object] $type$[] array = ($type$[]) oarray; #end[Object] - return UNSAFE.weakCompareAndSet$Type$Plain(array, + return UNSAFE.compareAndSet$Type$MO(Unsafe.MO_WEAK_CAS_PLAIN, + array, (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase, {#if[Object]?handle.componentType.cast(expected):expected}, {#if[Object]?runtimeTypeCheck(handle, array, value):value}); @@ -977,7 +1013,8 @@ final class VarHandle$Type$s { #else[Object] $type$[] array = ($type$[]) oarray; #end[Object] - return UNSAFE.weakCompareAndSet$Type$(array, + return UNSAFE.compareAndSet$Type$MO(Unsafe.MO_WEAK_CAS_VOLATILE, + array, (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase, {#if[Object]?handle.componentType.cast(expected):expected}, {#if[Object]?runtimeTypeCheck(handle, array, value):value}); @@ -991,7 +1028,8 @@ final class VarHandle$Type$s { #else[Object] $type$[] array = ($type$[]) oarray; #end[Object] - return UNSAFE.weakCompareAndSet$Type$Acquire(array, + return UNSAFE.compareAndSet$Type$MO(Unsafe.MO_WEAK_CAS_ACQUIRE, + array, (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase, {#if[Object]?handle.componentType.cast(expected):expected}, {#if[Object]?runtimeTypeCheck(handle, array, value):value}); @@ -1005,7 +1043,8 @@ final class VarHandle$Type$s { #else[Object] $type$[] array = ($type$[]) oarray; #end[Object] - return UNSAFE.weakCompareAndSet$Type$Release(array, + return UNSAFE.compareAndSet$Type$MO(Unsafe.MO_WEAK_CAS_RELEASE, + array, (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase, {#if[Object]?handle.componentType.cast(expected):expected}, {#if[Object]?runtimeTypeCheck(handle, array, value):value}); @@ -1032,7 +1071,8 @@ final class VarHandle$Type$s { #else[Object] $type$[] array = ($type$[]) oarray; #end[Object] - return UNSAFE.getAndSet$Type$Acquire(array, + return UNSAFE.getAndSet$Type$MO(Unsafe.MO_ACQUIRE, + array, (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase, {#if[Object]?runtimeTypeCheck(handle, array, value):value}); } @@ -1045,7 +1085,8 @@ final class VarHandle$Type$s { #else[Object] $type$[] array = ($type$[]) oarray; #end[Object] - return UNSAFE.getAndSet$Type$Release(array, + return UNSAFE.getAndSet$Type$MO(Unsafe.MO_RELEASE, + array, (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase, {#if[Object]?runtimeTypeCheck(handle, array, value):value}); } @@ -1065,7 +1106,8 @@ final class VarHandle$Type$s { static $type$ getAndAddAcquire(VarHandle ob, Object oarray, int index, $type$ value) { Array handle = (Array)ob; $type$[] array = ($type$[]) oarray; - return UNSAFE.getAndAdd$Type$Acquire(array, + return UNSAFE.getAndAdd$Type$MO(Unsafe.MO_ACQUIRE, + array, (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase, value); } @@ -1074,7 +1116,8 @@ final class VarHandle$Type$s { static $type$ getAndAddRelease(VarHandle ob, Object oarray, int index, $type$ value) { Array handle = (Array)ob; $type$[] array = ($type$[]) oarray; - return UNSAFE.getAndAdd$Type$Release(array, + return UNSAFE.getAndAdd$Type$MO(Unsafe.MO_RELEASE, + array, (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase, value); } @@ -1094,7 +1137,8 @@ final class VarHandle$Type$s { static $type$ getAndBitwiseOrRelease(VarHandle ob, Object oarray, int index, $type$ value) { Array handle = (Array)ob; $type$[] array = ($type$[]) oarray; - return UNSAFE.getAndBitwiseOr$Type$Release(array, + return UNSAFE.getAndBitwiseOr$Type$MO(Unsafe.MO_RELEASE, + array, (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase, value); } @@ -1103,7 +1147,8 @@ final class VarHandle$Type$s { static $type$ getAndBitwiseOrAcquire(VarHandle ob, Object oarray, int index, $type$ value) { Array handle = (Array)ob; $type$[] array = ($type$[]) oarray; - return UNSAFE.getAndBitwiseOr$Type$Acquire(array, + return UNSAFE.getAndBitwiseOr$Type$MO(Unsafe.MO_ACQUIRE, + array, (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase, value); } @@ -1121,7 +1166,8 @@ final class VarHandle$Type$s { static $type$ getAndBitwiseAndRelease(VarHandle ob, Object oarray, int index, $type$ value) { Array handle = (Array)ob; $type$[] array = ($type$[]) oarray; - return UNSAFE.getAndBitwiseAnd$Type$Release(array, + return UNSAFE.getAndBitwiseAnd$Type$MO(Unsafe.MO_RELEASE, + array, (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase, value); } @@ -1130,7 +1176,8 @@ final class VarHandle$Type$s { static $type$ getAndBitwiseAndAcquire(VarHandle ob, Object oarray, int index, $type$ value) { Array handle = (Array)ob; $type$[] array = ($type$[]) oarray; - return UNSAFE.getAndBitwiseAnd$Type$Acquire(array, + return UNSAFE.getAndBitwiseAnd$Type$MO(Unsafe.MO_ACQUIRE, + array, (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase, value); } @@ -1148,7 +1195,8 @@ final class VarHandle$Type$s { static $type$ getAndBitwiseXorRelease(VarHandle ob, Object oarray, int index, $type$ value) { Array handle = (Array)ob; $type$[] array = ($type$[]) oarray; - return UNSAFE.getAndBitwiseXor$Type$Release(array, + return UNSAFE.getAndBitwiseXor$Type$MO(Unsafe.MO_RELEASE, + array, (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase, value); } @@ -1157,7 +1205,8 @@ final class VarHandle$Type$s { static $type$ getAndBitwiseXorAcquire(VarHandle ob, Object oarray, int index, $type$ value) { Array handle = (Array)ob; $type$[] array = ($type$[]) oarray; - return UNSAFE.getAndBitwiseXor$Type$Acquire(array, + return UNSAFE.getAndBitwiseXor$Type$MO(Unsafe.MO_ACQUIRE, + array, (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase, value); } diff --git a/src/java.base/share/classes/java/lang/invoke/X-VarHandleByteArrayView.java.template b/src/java.base/share/classes/java/lang/invoke/X-VarHandleByteArrayView.java.template index 3cd523ee1cfae..e780fbbd899d7 100644 --- a/src/java.base/share/classes/java/lang/invoke/X-VarHandleByteArrayView.java.template +++ b/src/java.base/share/classes/java/lang/invoke/X-VarHandleByteArrayView.java.template @@ -487,7 +487,8 @@ final class VarHandleByteArrayAs$Type$s extends VarHandleByteArrayBase { do { nativeExpectedValue = SCOPED_MEMORY_ACCESS.get$RawType$Volatile(session(bb), base, offset); expectedValue = $RawBoxType$.reverseBytes(nativeExpectedValue); - } while (!UNSAFE.weakCompareAndSet$RawType$(base, offset, + } while (!UNSAFE.compareAndSet$RawType$MO(Unsafe.MO_WEAK_CAS_VOLATILE, + base, offset, nativeExpectedValue, $RawBoxType$.reverseBytes(expectedValue + delta))); return expectedValue; } @@ -544,7 +545,8 @@ final class VarHandleByteArrayAs$Type$s extends VarHandleByteArrayBase { do { nativeExpectedValue = SCOPED_MEMORY_ACCESS.get$RawType$Volatile(session(bb), base, offset); expectedValue = $RawBoxType$.reverseBytes(nativeExpectedValue); - } while (!UNSAFE.weakCompareAndSet$RawType$(base, offset, + } while (!UNSAFE.compareAndSet$RawType$MO(Unsafe.MO_WEAK_CAS_VOLATILE, + base, offset, nativeExpectedValue, $RawBoxType$.reverseBytes(expectedValue | value))); return expectedValue; } @@ -599,7 +601,8 @@ final class VarHandleByteArrayAs$Type$s extends VarHandleByteArrayBase { do { nativeExpectedValue = SCOPED_MEMORY_ACCESS.get$RawType$Volatile(session(bb), base, offset); expectedValue = $RawBoxType$.reverseBytes(nativeExpectedValue); - } while (!UNSAFE.weakCompareAndSet$RawType$(base, offset, + } while (!UNSAFE.compareAndSet$RawType$MO(Unsafe.MO_WEAK_CAS_VOLATILE, + base, offset, nativeExpectedValue, $RawBoxType$.reverseBytes(expectedValue & value))); return expectedValue; } @@ -655,7 +658,8 @@ final class VarHandleByteArrayAs$Type$s extends VarHandleByteArrayBase { do { nativeExpectedValue = SCOPED_MEMORY_ACCESS.get$RawType$Volatile(session(bb), base, offset); expectedValue = $RawBoxType$.reverseBytes(nativeExpectedValue); - } while (!UNSAFE.weakCompareAndSet$RawType$(base, offset, + } while (!UNSAFE.compareAndSet$RawType$MO(Unsafe.MO_WEAK_CAS_VOLATILE, + base, offset, nativeExpectedValue, $RawBoxType$.reverseBytes(expectedValue ^ value))); return expectedValue; } diff --git a/src/java.base/share/classes/java/math/BigDecimal.java b/src/java.base/share/classes/java/math/BigDecimal.java index 14d81d30c3d40..91b1876313848 100644 --- a/src/java.base/share/classes/java/math/BigDecimal.java +++ b/src/java.base/share/classes/java/math/BigDecimal.java @@ -42,6 +42,7 @@ import jdk.internal.access.SharedSecrets; import jdk.internal.math.FormattedFPDecimal; import jdk.internal.util.DecimalDigits; +import jdk.internal.misc.Unsafe; import jdk.internal.vm.annotation.Stable; /** @@ -4423,8 +4424,8 @@ private static void matchScale(BigDecimal[] val) { } private static class UnsafeHolder { - private static final jdk.internal.misc.Unsafe unsafe - = jdk.internal.misc.Unsafe.getUnsafe(); + private static final Unsafe unsafe + = Unsafe.getUnsafe(); private static final long intCompactOffset = unsafe.objectFieldOffset(BigDecimal.class, "intCompact"); private static final long intValOffset @@ -4439,7 +4440,7 @@ static void setIntValAndScale(BigDecimal bd, BigInteger intVal, int scale) { } static void setIntValVolatile(BigDecimal bd, BigInteger val) { - unsafe.putReferenceVolatile(bd, intValOffset, val); + unsafe.putReferenceMO(Unsafe.MO_VOLATILE, bd, intValOffset, val); } } diff --git a/src/java.base/share/classes/java/util/LazyCollections.java b/src/java.base/share/classes/java/util/LazyCollections.java index 0bbdad87ac45e..f53bb0d408032 100644 --- a/src/java.base/share/classes/java/util/LazyCollections.java +++ b/src/java.base/share/classes/java/util/LazyCollections.java @@ -138,7 +138,7 @@ private T[] copyInto(Object[] a) { @SuppressWarnings("unchecked") @ForceInline private E contentsAcquire(long offset) { - return (E) UNSAFE.getReferenceAcquire(elements, offset); + return (E) UNSAFE.getReferenceMO(Unsafe.MO_ACQUIRE, elements, offset); } } @@ -289,7 +289,7 @@ public final V get(Object key) { @ForceInline final V orElseCompute(K key, int index) { final long offset = offsetFor(index); - final V v = (V) UNSAFE.getReferenceAcquire(values, offset); + final V v = (V) UNSAFE.getReferenceMO(Unsafe.MO_ACQUIRE, values, offset); if (v != null) { return v; } @@ -435,7 +435,7 @@ private Mutexes(int length) { private Object acquireMutex(long offset) { assert mutexes != null; // Check if there already is a mutex (Object or TOMB_STONE) - final Object mutex = UNSAFE.getReferenceVolatile(mutexes, offset); + final Object mutex = UNSAFE.getReferenceMO(Unsafe.MO_VOLATILE, mutexes, offset); if (mutex != null) { return mutex; } @@ -539,7 +539,7 @@ static void set(T[] array, int index, Object mutex, T newValue) { // We know we hold the monitor here so plain semantic is enough // This is an extra safety net to emulate a CAS op. if (array[index] == null) { - UNSAFE.putReferenceRelease(array, Unsafe.ARRAY_OBJECT_BASE_OFFSET + Unsafe.ARRAY_OBJECT_INDEX_SCALE * (long) index, newValue); + UNSAFE.putReferenceMO(Unsafe.MO_RELEASE, array, Unsafe.ARRAY_OBJECT_BASE_OFFSET + Unsafe.ARRAY_OBJECT_INDEX_SCALE * (long) index, newValue); } } diff --git a/src/java.base/share/classes/java/util/Random.java b/src/java.base/share/classes/java/util/Random.java index 661a29e303be3..b23705a4cf684 100644 --- a/src/java.base/share/classes/java/util/Random.java +++ b/src/java.base/share/classes/java/util/Random.java @@ -837,7 +837,7 @@ private synchronized void writeObject(ObjectOutputStream s) } catch (Exception ex) { throw new Error(ex); } } private void resetSeed(long seedVal) { - unsafe.putReferenceVolatile(this, seedOffset, new AtomicLong(seedVal)); + unsafe.putReferenceMO(Unsafe.MO_VOLATILE, this, seedOffset, new AtomicLong(seedVal)); } /** diff --git a/src/java.base/share/classes/java/util/concurrent/ConcurrentHashMap.java b/src/java.base/share/classes/java/util/concurrent/ConcurrentHashMap.java index 9295f0deb59b8..aa5793fa58821 100644 --- a/src/java.base/share/classes/java/util/concurrent/ConcurrentHashMap.java +++ b/src/java.base/share/classes/java/util/concurrent/ConcurrentHashMap.java @@ -771,7 +771,7 @@ static int compareComparables(Class kc, Object k, Object x) { @SuppressWarnings("unchecked") static final Node tabAt(Node[] tab, int i) { - return (Node)U.getReferenceAcquire(tab, ((long)i << ASHIFT) + ABASE); + return (Node)U.getReferenceMO(Unsafe.MO_ACQUIRE, tab, ((long)i << ASHIFT) + ABASE); } static final boolean casTabAt(Node[] tab, int i, @@ -780,7 +780,7 @@ static final boolean casTabAt(Node[] tab, int i, } static final void setTabAt(Node[] tab, int i, Node v) { - U.putReferenceRelease(tab, ((long)i << ASHIFT) + ABASE, v); + U.putReferenceMO(Unsafe.MO_RELEASE, tab, ((long)i << ASHIFT) + ABASE, v); } /* ---------------- Fields -------------- */ diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index f289186e0aded..c4eb5b61f6cd7 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -1200,10 +1200,10 @@ static final class WorkQueue { private static final long ARRAY; final void updateBase(int v) { - U.putIntVolatile(this, BASE, v); + U.putIntMO(Unsafe.MO_VOLATILE, this, BASE, v); } final void updateTop(int v) { - U.putIntOpaque(this, TOP, v); + U.putIntMO(Unsafe.MO_OPAQUE, this, TOP, v); } final void updateArray(ForkJoinTask[] a) { U.getAndSetReference(this, ARRAY, a); @@ -1273,7 +1273,7 @@ final void push(ForkJoinTask task, ForkJoinPool pool, boolean internal) { throw new RejectedExecutionException("Queue capacity exceeded"); if (pool != null && (room == 0 || - U.getReferenceAcquire(a, slotOffset(m & (s - 1))) == null)) + U.getReferenceMO(Unsafe.MO_ACQUIRE, a, slotOffset(m & (s - 1))) == null)) pool.signalWork(a, k); // may have appeared empty } } @@ -1334,7 +1334,7 @@ private ForkJoinTask localPoll() { if (U.getReference(a, k) == null) { if (nb == p) break; // else base is lagging - while (b == (b = U.getIntAcquire(this, BASE))) + while (b == (b = U.getIntMO(Unsafe.MO_ACQUIRE, this, BASE))) Thread.onSpinWait(); // spin to reduce memory traffic } else if ((t = (ForkJoinTask) @@ -1406,7 +1406,7 @@ final ForkJoinTask poll() { ForkJoinTask t; int cap, nb; long k; ForkJoinTask[] a; if ((a = array) == null || (cap = a.length) <= 0) break; - t = (ForkJoinTask)U.getReferenceAcquire( + t = (ForkJoinTask)U.getReferenceMO(Unsafe.MO_ACQUIRE, a, k = slotOffset((cap - 1) & (b = base))); Object u = U.getReference( // next slot a, slotOffset((cap - 1) & (nb = b + 1))); @@ -1463,7 +1463,7 @@ final void tryRemoveAndExec(ForkJoinTask task, boolean internal) { else if (i == base) // act as poll updateBase(i + 1); else { // swap with top - U.putReferenceVolatile( + U.putReferenceMO(Unsafe.MO_VOLATILE, a, k, (ForkJoinTask) U.getAndSetReference( a, slotOffset(s & m), null)); @@ -1539,7 +1539,7 @@ final void helpAsyncBlocker(ManagedBlocker blocker) { ForkJoinTask t; ForkJoinTask[] a; int b, cap; long k; if ((a = array) == null || (cap = a.length) <= 0) break; - t = (ForkJoinTask)U.getReferenceAcquire( + t = (ForkJoinTask)U.getReferenceMO(Unsafe.MO_ACQUIRE, a, k = slotOffset((cap - 1) & (b = base))); if (t == null) { if (top - b <= 0) @@ -1670,7 +1670,7 @@ private int getAndSetParallelism(int v) { return U.getAndSetInt(this, PARALLELISM, v); } private int getParallelismOpaque() { - return U.getIntOpaque(this, PARALLELISM); + return U.getIntMO(Unsafe.MO_OPAQUE, this, PARALLELISM); } private CountDownLatch cmpExTerminationSignal(CountDownLatch x) { return (CountDownLatch) @@ -1977,7 +1977,7 @@ final void runWorker(WorkQueue w) { ForkJoinTask[] a; int cap; // poll queue while ((a = q.array) != null && (cap = a.length) > 0) { int b, nb, nk; long bp; ForkJoinTask t; - t = (ForkJoinTask)U.getReferenceAcquire( + t = (ForkJoinTask)U.getReferenceMO(Unsafe.MO_ACQUIRE, a, bp = slotOffset((cap - 1) & (b = q.base))); long np = slotOffset(nk = (nb = b + 1) & (cap - 1)); if (q.base == b) { // else inconsistent @@ -1987,7 +1987,7 @@ final void runWorker(WorkQueue w) { break scan; if (U.getReference(a, np) == null && (rescans >= 0 || - (U.getReferenceAcquire(a, bp) == null && + (U.getReferenceMO(Unsafe.MO_ACQUIRE, a, bp) == null && q.top == q.base))) break; rescans = 1; // may be stalled @@ -2001,12 +2001,12 @@ else if (inactive != 0) { } else if (U.compareAndSetReference(a, bp, t, null)) { q.base = nb; - Object nt = U.getReferenceAcquire(a, np); + Object nt = U.getReferenceMO(Unsafe.MO_ACQUIRE, a, np); w.source = qid; rescans = 1; ++taken; if (nt != null && // confirm a[nk] - U.getReferenceAcquire(a, np) == nt) + U.getReferenceMO(Unsafe.MO_ACQUIRE, a, np) == nt) signalWork(a, nk); // propagate w.topLevelExec(t, fifo); } @@ -2279,7 +2279,7 @@ final int helpJoin(ForkJoinTask task, WorkQueue w, boolean internal) { int sq = q.source, b, cap; long k; if ((a = q.array) == null || (cap = a.length) <= 0) break; - t = (ForkJoinTask)U.getReferenceAcquire( + t = (ForkJoinTask)U.getReferenceMO(Unsafe.MO_ACQUIRE, a, k = slotOffset((cap - 1) & (b = q.base))); if (t == task) eligible = true; @@ -2360,7 +2360,7 @@ final int helpComplete(ForkJoinTask task, WorkQueue w, boolean internal) { boolean eligible = false; if ((a = q.array) == null || (cap = a.length) <= 0) break; - t = (ForkJoinTask)U.getReferenceAcquire( + t = (ForkJoinTask)U.getReferenceMO(Unsafe.MO_ACQUIRE, a, k = slotOffset((cap - 1) & (b = q.base))); if (t instanceof CountedCompleter) { CountedCompleter f = (CountedCompleter)t; @@ -2443,7 +2443,7 @@ private int helpQuiesce(WorkQueue w, long nanos, boolean interruptible) { int b, cap; long k; if ((a = q.array) == null || (cap = a.length) <= 0) break; - t = (ForkJoinTask)U.getReferenceAcquire( + t = (ForkJoinTask)U.getReferenceMO(Unsafe.MO_ACQUIRE, a, k = slotOffset((cap - 1) & (b = q.base))); if (t != null && phase == inactivePhase) // reactivate w.phase = phase = activePhase; @@ -2800,7 +2800,7 @@ private boolean cleanQueues() { (a = q.array) != null && (cap = a.length) > 0) { for (;;) { ForkJoinTask t; int b; long k; - t = (ForkJoinTask)U.getReferenceAcquire( + t = (ForkJoinTask)U.getReferenceMO(Unsafe.MO_ACQUIRE, a, k = slotOffset((cap - 1) & (b = q.base))); if (q.base == b && t != null && U.compareAndSetReference(a, k, t, null)) { diff --git a/src/java.base/share/classes/java/util/concurrent/atomic/AtomicInteger.java b/src/java.base/share/classes/java/util/concurrent/atomic/AtomicInteger.java index 90b4791f05a09..c1932f7ede673 100644 --- a/src/java.base/share/classes/java/util/concurrent/atomic/AtomicInteger.java +++ b/src/java.base/share/classes/java/util/concurrent/atomic/AtomicInteger.java @@ -109,7 +109,7 @@ public final void set(int newValue) { * @since 1.6 */ public final void lazySet(int newValue) { - U.putIntRelease(this, VALUE, newValue); + U.putIntMO(Unsafe.MO_RELEASE, this, VALUE, newValue); } /** @@ -155,7 +155,7 @@ public final boolean compareAndSet(int expectedValue, int newValue) { */ @Deprecated(since="9") public final boolean weakCompareAndSet(int expectedValue, int newValue) { - return U.weakCompareAndSetIntPlain(this, VALUE, expectedValue, newValue); + return U.compareAndSetIntMO(Unsafe.MO_WEAK_CAS_PLAIN, this, VALUE, expectedValue, newValue); } /** @@ -169,7 +169,7 @@ public final boolean weakCompareAndSet(int expectedValue, int newValue) { * @since 9 */ public final boolean weakCompareAndSetPlain(int expectedValue, int newValue) { - return U.weakCompareAndSetIntPlain(this, VALUE, expectedValue, newValue); + return U.compareAndSetIntMO(Unsafe.MO_WEAK_CAS_PLAIN, this, VALUE, expectedValue, newValue); } /** @@ -421,7 +421,7 @@ public final void setPlain(int newValue) { * @since 9 */ public final int getOpaque() { - return U.getIntOpaque(this, VALUE); + return U.getIntMO(Unsafe.MO_OPAQUE, this, VALUE); } /** @@ -432,7 +432,7 @@ public final int getOpaque() { * @since 9 */ public final void setOpaque(int newValue) { - U.putIntOpaque(this, VALUE, newValue); + U.putIntMO(Unsafe.MO_OPAQUE, this, VALUE, newValue); } /** @@ -443,7 +443,7 @@ public final void setOpaque(int newValue) { * @since 9 */ public final int getAcquire() { - return U.getIntAcquire(this, VALUE); + return U.getIntMO(Unsafe.MO_ACQUIRE, this, VALUE); } /** @@ -454,7 +454,7 @@ public final int getAcquire() { * @since 9 */ public final void setRelease(int newValue) { - U.putIntRelease(this, VALUE, newValue); + U.putIntMO(Unsafe.MO_RELEASE, this, VALUE, newValue); } /** @@ -486,7 +486,7 @@ public final int compareAndExchange(int expectedValue, int newValue) { * @since 9 */ public final int compareAndExchangeAcquire(int expectedValue, int newValue) { - return U.compareAndExchangeIntAcquire(this, VALUE, expectedValue, newValue); + return U.compareAndExchangeIntMO(Unsafe.MO_ACQUIRE, this, VALUE, expectedValue, newValue); } /** @@ -502,7 +502,7 @@ public final int compareAndExchangeAcquire(int expectedValue, int newValue) { * @since 9 */ public final int compareAndExchangeRelease(int expectedValue, int newValue) { - return U.compareAndExchangeIntRelease(this, VALUE, expectedValue, newValue); + return U.compareAndExchangeIntMO(Unsafe.MO_RELEASE, this, VALUE, expectedValue, newValue); } /** @@ -517,7 +517,7 @@ public final int compareAndExchangeRelease(int expectedValue, int newValue) { * @since 9 */ public final boolean weakCompareAndSetVolatile(int expectedValue, int newValue) { - return U.weakCompareAndSetInt(this, VALUE, expectedValue, newValue); + return U.compareAndSetIntMO(Unsafe.MO_WEAK_CAS_VOLATILE, this, VALUE, expectedValue, newValue); } /** @@ -532,7 +532,7 @@ public final boolean weakCompareAndSetVolatile(int expectedValue, int newValue) * @since 9 */ public final boolean weakCompareAndSetAcquire(int expectedValue, int newValue) { - return U.weakCompareAndSetIntAcquire(this, VALUE, expectedValue, newValue); + return U.compareAndSetIntMO(Unsafe.MO_WEAK_CAS_ACQUIRE, this, VALUE, expectedValue, newValue); } /** @@ -547,7 +547,7 @@ public final boolean weakCompareAndSetAcquire(int expectedValue, int newValue) { * @since 9 */ public final boolean weakCompareAndSetRelease(int expectedValue, int newValue) { - return U.weakCompareAndSetIntRelease(this, VALUE, expectedValue, newValue); + return U.compareAndSetIntMO(Unsafe.MO_WEAK_CAS_RELEASE, this, VALUE, expectedValue, newValue); } } diff --git a/src/java.base/share/classes/java/util/concurrent/atomic/AtomicIntegerFieldUpdater.java b/src/java.base/share/classes/java/util/concurrent/atomic/AtomicIntegerFieldUpdater.java index 2250009e8f599..b666efce898c5 100644 --- a/src/java.base/share/classes/java/util/concurrent/atomic/AtomicIntegerFieldUpdater.java +++ b/src/java.base/share/classes/java/util/concurrent/atomic/AtomicIntegerFieldUpdater.java @@ -485,17 +485,17 @@ public final boolean weakCompareAndSet(T obj, int expect, int update) { public final void set(T obj, int newValue) { accessCheck(obj); - U.putIntVolatile(obj, offset, newValue); + U.putIntMO(Unsafe.MO_VOLATILE, obj, offset, newValue); } public final void lazySet(T obj, int newValue) { accessCheck(obj); - U.putIntRelease(obj, offset, newValue); + U.putIntMO(Unsafe.MO_RELEASE, obj, offset, newValue); } public final int get(T obj) { accessCheck(obj); - return U.getIntVolatile(obj, offset); + return U.getIntMO(Unsafe.MO_VOLATILE, obj, offset); } public final int getAndSet(T obj, int newValue) { diff --git a/src/java.base/share/classes/java/util/concurrent/atomic/AtomicLong.java b/src/java.base/share/classes/java/util/concurrent/atomic/AtomicLong.java index 81c60305a2afb..a03151694d71f 100644 --- a/src/java.base/share/classes/java/util/concurrent/atomic/AtomicLong.java +++ b/src/java.base/share/classes/java/util/concurrent/atomic/AtomicLong.java @@ -99,7 +99,7 @@ public final long get() { */ public final void set(long newValue) { // See JDK-8180620: Clarify VarHandle mixed-access subtleties - U.putLongVolatile(this, VALUE, newValue); + U.putLongMO(Unsafe.MO_VOLATILE, this, VALUE, newValue); } /** @@ -110,7 +110,7 @@ public final void set(long newValue) { * @since 1.6 */ public final void lazySet(long newValue) { - U.putLongRelease(this, VALUE, newValue); + U.putLongMO(Unsafe.MO_RELEASE, this, VALUE, newValue); } /** @@ -156,7 +156,7 @@ public final boolean compareAndSet(long expectedValue, long newValue) { */ @Deprecated(since="9") public final boolean weakCompareAndSet(long expectedValue, long newValue) { - return U.weakCompareAndSetLongPlain(this, VALUE, expectedValue, newValue); + return U.compareAndSetLongMO(Unsafe.MO_WEAK_CAS_PLAIN, this, VALUE, expectedValue, newValue); } /** @@ -170,7 +170,7 @@ public final boolean weakCompareAndSet(long expectedValue, long newValue) { * @since 9 */ public final boolean weakCompareAndSetPlain(long expectedValue, long newValue) { - return U.weakCompareAndSetLongPlain(this, VALUE, expectedValue, newValue); + return U.compareAndSetLongMO(Unsafe.MO_WEAK_CAS_PLAIN, this, VALUE, expectedValue, newValue); } /** @@ -420,7 +420,7 @@ public final void setPlain(long newValue) { * @since 9 */ public final long getOpaque() { - return U.getLongOpaque(this, VALUE); + return U.getLongMO(Unsafe.MO_OPAQUE, this, VALUE); } /** @@ -431,7 +431,7 @@ public final long getOpaque() { * @since 9 */ public final void setOpaque(long newValue) { - U.putLongOpaque(this, VALUE, newValue); + U.putLongMO(Unsafe.MO_OPAQUE, this, VALUE, newValue); } /** @@ -442,7 +442,7 @@ public final void setOpaque(long newValue) { * @since 9 */ public final long getAcquire() { - return U.getLongAcquire(this, VALUE); + return U.getLongMO(Unsafe.MO_ACQUIRE, this, VALUE); } /** @@ -453,7 +453,7 @@ public final long getAcquire() { * @since 9 */ public final void setRelease(long newValue) { - U.putLongRelease(this, VALUE, newValue); + U.putLongMO(Unsafe.MO_RELEASE, this, VALUE, newValue); } /** @@ -485,7 +485,7 @@ public final long compareAndExchange(long expectedValue, long newValue) { * @since 9 */ public final long compareAndExchangeAcquire(long expectedValue, long newValue) { - return U.compareAndExchangeLongAcquire(this, VALUE, expectedValue, newValue); + return U.compareAndExchangeLongMO(Unsafe.MO_ACQUIRE, this, VALUE, expectedValue, newValue); } /** @@ -501,7 +501,7 @@ public final long compareAndExchangeAcquire(long expectedValue, long newValue) { * @since 9 */ public final long compareAndExchangeRelease(long expectedValue, long newValue) { - return U.compareAndExchangeLongRelease(this, VALUE, expectedValue, newValue); + return U.compareAndExchangeLongMO(Unsafe.MO_RELEASE, this, VALUE, expectedValue, newValue); } /** @@ -516,7 +516,7 @@ public final long compareAndExchangeRelease(long expectedValue, long newValue) { * @since 9 */ public final boolean weakCompareAndSetVolatile(long expectedValue, long newValue) { - return U.weakCompareAndSetLong(this, VALUE, expectedValue, newValue); + return U.compareAndSetLongMO(Unsafe.MO_WEAK_CAS_VOLATILE, this, VALUE, expectedValue, newValue); } /** @@ -531,7 +531,7 @@ public final boolean weakCompareAndSetVolatile(long expectedValue, long newValue * @since 9 */ public final boolean weakCompareAndSetAcquire(long expectedValue, long newValue) { - return U.weakCompareAndSetLongAcquire(this, VALUE, expectedValue, newValue); + return U.compareAndSetLongMO(Unsafe.MO_WEAK_CAS_ACQUIRE, this, VALUE, expectedValue, newValue); } /** @@ -546,7 +546,7 @@ public final boolean weakCompareAndSetAcquire(long expectedValue, long newValue) * @since 9 */ public final boolean weakCompareAndSetRelease(long expectedValue, long newValue) { - return U.weakCompareAndSetLongRelease(this, VALUE, expectedValue, newValue); + return U.compareAndSetLongMO(Unsafe.MO_WEAK_CAS_RELEASE, this, VALUE, expectedValue, newValue); } } diff --git a/src/java.base/share/classes/java/util/concurrent/atomic/AtomicLongFieldUpdater.java b/src/java.base/share/classes/java/util/concurrent/atomic/AtomicLongFieldUpdater.java index 5f0a666cb0438..c07f0af629ef2 100644 --- a/src/java.base/share/classes/java/util/concurrent/atomic/AtomicLongFieldUpdater.java +++ b/src/java.base/share/classes/java/util/concurrent/atomic/AtomicLongFieldUpdater.java @@ -455,17 +455,17 @@ public final boolean weakCompareAndSet(T obj, long expect, long update) { public final void set(T obj, long newValue) { accessCheck(obj); - U.putLongVolatile(obj, offset, newValue); + U.putLongMO(Unsafe.MO_VOLATILE, obj, offset, newValue); } public final void lazySet(T obj, long newValue) { accessCheck(obj); - U.putLongRelease(obj, offset, newValue); + U.putLongMO(Unsafe.MO_RELEASE, obj, offset, newValue); } public final long get(T obj) { accessCheck(obj); - return U.getLongVolatile(obj, offset); + return U.getLongMO(Unsafe.MO_VOLATILE, obj, offset); } public final long getAndSet(T obj, long newValue) { diff --git a/src/java.base/share/classes/java/util/concurrent/atomic/AtomicReferenceFieldUpdater.java b/src/java.base/share/classes/java/util/concurrent/atomic/AtomicReferenceFieldUpdater.java index 4a758f77a47e2..873afd037c257 100644 --- a/src/java.base/share/classes/java/util/concurrent/atomic/AtomicReferenceFieldUpdater.java +++ b/src/java.base/share/classes/java/util/concurrent/atomic/AtomicReferenceFieldUpdater.java @@ -459,19 +459,19 @@ public final boolean weakCompareAndSet(T obj, V expect, V update) { public final void set(T obj, V newValue) { accessCheck(obj); valueCheck(newValue); - U.putReferenceVolatile(obj, offset, newValue); + U.putReferenceMO(Unsafe.MO_VOLATILE, obj, offset, newValue); } public final void lazySet(T obj, V newValue) { accessCheck(obj); valueCheck(newValue); - U.putReferenceRelease(obj, offset, newValue); + U.putReferenceMO(Unsafe.MO_RELEASE, obj, offset, newValue); } @SuppressWarnings("unchecked") public final V get(T obj) { accessCheck(obj); - return (V)U.getReferenceVolatile(obj, offset); + return (V)U.getReferenceMO(Unsafe.MO_VOLATILE, obj, offset); } @SuppressWarnings("unchecked") diff --git a/src/java.base/share/classes/java/util/concurrent/locks/AbstractQueuedLongSynchronizer.java b/src/java.base/share/classes/java/util/concurrent/locks/AbstractQueuedLongSynchronizer.java index ba81123fc35fd..7adf81d72ea53 100644 --- a/src/java.base/share/classes/java/util/concurrent/locks/AbstractQueuedLongSynchronizer.java +++ b/src/java.base/share/classes/java/util/concurrent/locks/AbstractQueuedLongSynchronizer.java @@ -107,7 +107,7 @@ final void setStatusRelaxed(int s) { // for off-queue assignment U.putInt(this, STATUS, s); } final void clearStatus() { // for reducing unneeded signals - U.putIntOpaque(this, STATUS, 0); + U.putIntMO(Unsafe.MO_OPAQUE, this, STATUS, 0); } private static final long STATUS diff --git a/src/java.base/share/classes/java/util/concurrent/locks/AbstractQueuedSynchronizer.java b/src/java.base/share/classes/java/util/concurrent/locks/AbstractQueuedSynchronizer.java index c077954508341..2cf3ed02efe05 100644 --- a/src/java.base/share/classes/java/util/concurrent/locks/AbstractQueuedSynchronizer.java +++ b/src/java.base/share/classes/java/util/concurrent/locks/AbstractQueuedSynchronizer.java @@ -488,7 +488,7 @@ final void setStatusRelaxed(int s) { // for off-queue assignment U.putInt(this, STATUS, s); } final void clearStatus() { // for reducing unneeded signals - U.putIntOpaque(this, STATUS, 0); + U.putIntMO(Unsafe.MO_OPAQUE, this, STATUS, 0); } private static final long STATUS diff --git a/src/java.base/share/classes/java/util/concurrent/locks/LockSupport.java b/src/java.base/share/classes/java/util/concurrent/locks/LockSupport.java index 38531c80a300a..a6b967d47db79 100644 --- a/src/java.base/share/classes/java/util/concurrent/locks/LockSupport.java +++ b/src/java.base/share/classes/java/util/concurrent/locks/LockSupport.java @@ -143,7 +143,7 @@ public final class LockSupport { private LockSupport() {} // Cannot be instantiated. private static void setBlocker(Thread t, Object arg) { - U.putReferenceOpaque(t, PARKBLOCKER, arg); + U.putReferenceMO(Unsafe.MO_OPAQUE, t, PARKBLOCKER, arg); } /** @@ -161,7 +161,7 @@ private static void setBlocker(Thread t, Object arg) { * @since 14 */ public static void setCurrentBlocker(Object blocker) { - U.putReferenceOpaque(Thread.currentThread(), PARKBLOCKER, blocker); + U.putReferenceMO(Unsafe.MO_OPAQUE, Thread.currentThread(), PARKBLOCKER, blocker); } /** @@ -334,7 +334,7 @@ public static void parkUntil(Object blocker, long deadline) { public static Object getBlocker(Thread t) { if (t == null) throw new NullPointerException(); - return U.getReferenceOpaque(t, PARKBLOCKER); + return U.getReferenceMO(Unsafe.MO_OPAQUE, t, PARKBLOCKER); } /** diff --git a/src/java.base/share/classes/java/util/concurrent/locks/StampedLock.java b/src/java.base/share/classes/java/util/concurrent/locks/StampedLock.java index 3fbfad875d66d..c009794ac0103 100644 --- a/src/java.base/share/classes/java/util/concurrent/locks/StampedLock.java +++ b/src/java.base/share/classes/java/util/concurrent/locks/StampedLock.java @@ -370,7 +370,7 @@ final void setStatusRelaxed(int s) { // for off-queue assignment U.putInt(this, STATUS, s); } final void clearStatus() { // for reducing unneeded signals - U.putIntOpaque(this, STATUS, 0); + U.putIntMO(Unsafe.MO_OPAQUE, this, STATUS, 0); } private static final long STATUS @@ -473,7 +473,7 @@ private long releaseWrite(long s) { @ReservedStackAccess public long writeLock() { // try unconditional CAS confirming weak read - long s = U.getLongOpaque(this, STATE) & ~ABITS, nextState; + long s = U.getLongMO(Unsafe.MO_OPAQUE, this, STATE) & ~ABITS, nextState; if (casState(s, nextState = s | WBIT)) { U.storeStoreFence(); return nextState; @@ -548,7 +548,7 @@ public long writeLockInterruptibly() throws InterruptedException { @ReservedStackAccess public long readLock() { // unconditionally optimistically try non-overflow case once - long s = U.getLongOpaque(this, STATE) & RSAFE, nextState; + long s = U.getLongMO(Unsafe.MO_OPAQUE, this, STATE) & RSAFE, nextState; if (casState(s, nextState = s + RUNIT)) return nextState; else diff --git a/src/java.base/share/classes/jdk/internal/lang/LazyConstantImpl.java b/src/java.base/share/classes/jdk/internal/lang/LazyConstantImpl.java index 59d0174c4c956..5ff3c1b481934 100644 --- a/src/java.base/share/classes/jdk/internal/lang/LazyConstantImpl.java +++ b/src/java.base/share/classes/jdk/internal/lang/LazyConstantImpl.java @@ -151,11 +151,11 @@ private String toStringSuffix() { @SuppressWarnings("unchecked") @ForceInline private T getAcquire() { - return (T) UNSAFE.getReferenceAcquire(this, CONSTANT_OFFSET); + return (T) UNSAFE.getReferenceMO(Unsafe.MO_ACQUIRE, this, CONSTANT_OFFSET); } private void setRelease(T newValue) { - UNSAFE.putReferenceRelease(this, CONSTANT_OFFSET, newValue); + UNSAFE.putReferenceMO(Unsafe.MO_RELEASE, this, CONSTANT_OFFSET, newValue); } private void preventReentry() { diff --git a/src/java.base/share/classes/jdk/internal/misc/InnocuousThread.java b/src/java.base/share/classes/jdk/internal/misc/InnocuousThread.java index 4f7811faf343d..efd3b6ef73fe5 100644 --- a/src/java.base/share/classes/jdk/internal/misc/InnocuousThread.java +++ b/src/java.base/share/classes/jdk/internal/misc/InnocuousThread.java @@ -113,7 +113,7 @@ private static Thread createThread(String name, Runnable target, long stackSize, private InnocuousThread(ThreadGroup group, Runnable target, String name, long stackSize, ClassLoader tccl) { super(group, target, name, stackSize, false); - UNSAFE.putReferenceRelease(this, CONTEXTCLASSLOADER, tccl); + UNSAFE.putReferenceMO(Unsafe.MO_RELEASE, this, CONTEXTCLASSLOADER, tccl); } @Override diff --git a/src/java.base/share/classes/jdk/internal/misc/Unsafe.java b/src/java.base/share/classes/jdk/internal/misc/Unsafe.java index 016566ae6592f..7fc62fa195d69 100644 --- a/src/java.base/share/classes/jdk/internal/misc/Unsafe.java +++ b/src/java.base/share/classes/jdk/internal/misc/Unsafe.java @@ -104,8 +104,480 @@ public static Unsafe getUnsafe() { //--- peek and poke operations // (compilers should optimize these to memory ops) - // These work on object fields in the Java heap. - // They will not work on elements of packed arrays. + // These work on object fields (of all types) in the Java heap. + // They also work on array elements (of all types) in the Java heap. + // THey also work on off-heap values (of primitive types). + + // These basic type codes are shared with the classfile format. Do not change. + private static final byte BT_BOOLEAN = 4; + private static final byte BT_CHAR = 5; + private static final byte BT_FLOAT = 6; + private static final byte BT_DOUBLE = 7; + private static final byte BT_BYTE = 8; + private static final byte BT_SHORT = 9; + private static final byte BT_INT = 10; + private static final byte BT_LONG = 11; + + // The low three bits of each BT value encodes size of BT. + private static final byte PRIMITIVE_SIZE_MASK = 3; + + // Operators for getAndOperatePrimitiveBitsMO + private static final byte OP_ADD = '+'; + private static final byte OP_BITAND = '&'; + private static final byte OP_BITOR = '|'; + private static final byte OP_BITXOR = '^'; + private static final byte OP_SWAP = '='; + // Can add these if they become popular: + //private static final byte OP_MAX = '>'; // Cf. C++26, RISCV amomaxu + //private static final byte OP_MIN = '<'; + //private static final byte OP_MAXU = ']'; + //private static final byte OP_MINU = '['; + // Yes, ASCII symbols are hokey, but we don't have a handy enum for this. + // (Also, Unsafe is involved in the bootstrapping of the first enum.) + + // These memory order codes are specific to this API and its native methods. + // They are mapped as needed to other order codes used elsewhere in the VM. + /** + * Selects normal memory order, which is the default for Java. + * Both compiler and hardware can reorder these accesses in the + * usual ways, per the JMM and C++ {@code memory_order_relaxed}. + * This is the default MO parameter for getters and setters. + *

+ * In brief, when accesses are marked {@code MO_PLAIN}, they + * can be freely reordered, forwards and backwards relative to + * each other, as long as they do not violate the program order + * within the current thread. The reordering of these accesses can + * by limited, in well-defined ways, by fences and non-plain accesses. + *

+ * It is important to remember that both the compiler and hardware + * can perform reordering, as they work to improve performance. + * The compiler might emit object code with an improved schedule of + * loads and stores, and hardware might also appear to reorder loads + * or stores, relative to main memory, due to the dynamics of caches + * and store queues. + *

+ * The VM calls this default memory order {@code MO_UNORDERED}, + * while the the Unsafe API has historically used the name + * {@code Plain}, because access to a plain Java variable + * access works in this unordered manner. + *

+ * In theory {@code MO_PLAIN} allows 64-bit accesses to be split + * into 32-bit access pairs, but this is not a normal practice + * on current platforms. Such splitting would violate the C++ + * specification for {@code memory_order_relaxed}. In other + * respects, the behavior of {@code MO_PLAIN} corresponds to + * C++ {@code memory_order_relaxed}. + *

+ * Nonatomic memory accesses (that is, accesses that get split up) + * are completely disallowed by every other memory order mode + * defined here, except, of course, {@code MO_UNALIGNED}. + *

+ * The other access types are more limited in their reordering. + * A load marked {@code MO_ACQUIRE} must come first before other + * accesses, plain or not, that might otherwise slide before it. + * When an acquiring load acquires a data value from memory, + * the current thread is prevented from accidentally observing + * any stale memory states (from other threads) that might have + * been present before the data value was stored to memory. + * Likewise, a store marked {@code MO_RELEASE} must come last after + * other accesses, plain or not, that might otherwise slide after it. + * When a releasing store releases a data value to memory, + * the current thread is must finish up with all of its previous + * stores, before the releasing store releases its data to + * memory (and thus to other threads). + * Finally, {@code MO_VOLATILE} is sequentially consistent access, + * which is the logical combination of both {@code MO_ACQUIRE} + * and {@code MO_RELEASE}, in one operation. + */ + public static final byte MO_PLAIN = 1; + + /** + * Selects volatile memory order, as found in Java. Neither + * compiler nor hardware will reorder these accesses. + * This is the default MO parameter for any operation which + * performs getting and setting in some atomic sequence. + *

+ * In brief, volatile memory order gives the strongest possible + * guarantee that the program order of memory accesses (in the + * current thread) corresponds (as much as possible) to some + * global memory order. It does this by providing the logical + * union of assurances provided by {@code MO_RELEASE} and + * {@code MO_ACQUIRE}. In addition, the compiler is required + * to schedule the operation strictly in program order. + * It follows that {@code MO_VOLATILE} can be used as a safe + * default for memory accesses. + *

+ * Note that if threads use a mix of memory order markings, one + * thread using weaker markings (such as {@code MO_PLAIN}) can + * potentially invalidate the correctness of logic in a second + * thread using stronger markings, even if the logic of the second + * thread appears locally correct. This is why volatility in the + * Java language is a property of variables and not of individual + * memory accesses. + *

+ * Volatile access corresponds to C++ {@code memory_order_seq_cst}. + * Hardware specifically guarantees that, if a variable that is + * accessed only in {@code MO_VOLATILE} order across all threads, + * a global sequential ordering will be observed by all threads. + *

+ * The VM calls this behavior {@code MO_SEQ_CST}, while the + * the Unsafe API has historically used the name {@code Volatile}, + * since that is the name Java has always given to this behavior. + */ + public static final byte MO_VOLATILE = 2; + + /** + * Selects acquiring memory order, which means a load always + * observes its data first, before any following memory accesses. + * (That is, following in the current thread, in program order.) + * Neither compiler nor hardware will allow a following memory + * access to slide before an acquiring load. + *

+ * In brief, if a thread reads from a shared variable using an + * acquiring load, and subsequently the thread reads or writes a + * second shared variable (in any mode), the thread will observe + * a global memory state that at least as "fresh" as the state + * observed by the acquiring load. (The precise meaning of + * "fresh" derives from the sequential order provided by global + * memory on the current platform. The point is that localized + * optimizations like compiler load hoisting or hardware + * caches or store buffers, applied to the second variable access, + * will not let it touch global memory before the acquiring + * load.) + *

+ * This behavior corresponds to C++ {@code memory_order_acquire}, + * and, on certain platforms, to memory access instructions marked + * as acquiring. + *

+ * The VM also calls this behavior {@code MO_ACQUIRE}, although + * with an unrelated numerical value, while the the Unsafe API has + * historically used the name {@code Acquire}. + */ + public static final byte MO_ACQUIRE = 4; + + /** + * Selects releasing memory order, which means a store always + * commits its data last, after any preceding memory accesses. + * (That is, preceding in the current thread, in program order.) + * Neither compiler nor hardware will allow a preceding memory + * access to slide after a releasing store. + *

+ * In brief, if a thread stores to a shared variable using a + * releasing store, after the thread has previously read or + * written another shared variable (in any mode), the thread will + * update a global memory state that at least as "fresh" as the + * state operated on by the previous memory access. (The precise + * meaning of "fresh" derives from the sequential order provided + * by global memory on the current platform. The point is that + * localized optimizations like compiler store sinking or hardware + * store buffer bypasses, will not allow the previous operation + * to touch global memory after the releasing store.) + *

+ * This behavior corresponds to C++ {@code memory_order_release}, + * and, on certain platforms, to memory access instructions marked + * as releasing. + *

+ * The VM also calls this behavior {@code MO_RELEASE}, although + * with an unrelated numerical value, while the the Unsafe API has + * historically used the name {@code Release}. + */ + public static final byte MO_RELEASE = 8; + + /** + * Selects an "opaque" memory order. The compiler will preserve + * exact program order (in the current thread) for this memory + * operation. Also, atomicity is guaranteed, even on unusual + * platforms where 64-bit values are not naturally atomic. + * In other respects, this is the same as {@code MO_PLAIN}. + *

+ * As with {@code MO_PLAIN}, the hardware will not be prevented + * from reordering the operation. A opaque load might observe + * stale values from local cache. On platforms with unusual store + * semantics, an opaque store might be observed to execute in + * non-program order, relative to nearby stores to other + * variables. + *

+ * This behavior corresponds to C++ {@code memory_order_relaxed}, + * except that the load or store is additionally constrained + * to execute, in object, in program order. + *

+ * The VM calls this behavior {@code MO_RELAXED} as opposed to + * {@code MO_UNORDERED}. The C2 compiler emits so-called + * CPU-order barriers ({@code MemBarCPUOrder}) before and after + * the operation, which do not interact with the memory system, + * but merely constrain the compiler as it schedules code. + */ + public static final byte MO_OPAQUE = MO_PLAIN | MO_VOLATILE; + + // Except for unaligned loads and stores, primitive and single-reference + // accesses provided here are always atomic and always naturally aligned, + // as machine word loads and stores. + // + // The JMM gives permission to treat 64-bit memory words as pairs + // of 32-bit words, but this permission is not used on modern machines. + // Ports of HotSpot to machines lacking performant 64-bit atomic loads + // and stores would, in theory, break up MO_PLAIN accesses into + // 32-bit access pairs. Reducing such a theory to practice would be + // an adventure. On such a platform, only MO_PLAIN would allow the + // 32-bit access pairs, and other modes (including MO_OPAQUE), would + // suppress the access pairs. + // + // There is a related (but distinct) set of MO codes in accessDecorators.hpp + // Perhaps confusingly, MO_ACQUIRE and MO_RELEASE are names used both + // here and there, even though they have distinct numerical values. + // The confusion is small -- they have the same meanings in both places. + // In the VM, constants defined here have the extra prefix "UNSAFE_". + // So, UNSAFE_MO_PLAIN, UNSAFE_MO_ACQUIRE, etc. + // + // When a Java constructor writes to a final field, there is a memory + // effect beyond C++ relaxed semantics; the write is akin to a releasing + // store. This effect is not implemented in this API, so MO_PLAIN here + // might be incorrect if an unsafe store were somehow used to replace a + // putfield operation, to a final field, in a constructor. + + /** MO_WEAK_CAS is only for CAS ops, combining bitwise with lower MO values */ + private static final byte MO_WEAK_CAS = 16; + + /** + * Mode bit which can be added to {@code MO_PLAIN}, permitting + * split (non-atomic) access if the effective address is not + * naturally aligned, relative to the data type being accesses. + * It cannot be applied to managed references. + *

+ * If a variable address happens to be naturally aligned, the + * access must be atomic. Otherwise, the requested access can be + * implemented by splitting it up into several accesses. The + * splitting must not break up the largest available atomic unit. + * For example, a 64-bit variable aligned to a 16-bit boundary + * must be loaded or stored using at most four 16-bit operations. + *

+ * On platforms which support misaligned access (at reasonable + * speeds) the compiler uses single instructions to implement + * unaligned loads and stores. These may or may not be atomic, + * depending on platform-specific conditions, such as the crossing + * of cache line boundaries. + */ + private static final byte MO_UNALIGNED = 32; + private static final byte MO_UNALIGNED_PLAIN = MO_UNALIGNED|MO_PLAIN; + + /** For CAS atomics only, adds weakness to {@code MO_PLAIN} or {@code MO_OPAQUE}. */ + public static final byte MO_WEAK_CAS_PLAIN = MO_WEAK_CAS|MO_OPAQUE; + /** For CAS atomics only, adds weakness to {@code MO_VOLATILE}. */ + public static final byte MO_WEAK_CAS_VOLATILE = MO_WEAK_CAS|MO_VOLATILE; + /** For CAS atomics only, adds weakness to {@code MO_ACQUIRE}. */ + public static final byte MO_WEAK_CAS_ACQUIRE = MO_WEAK_CAS|MO_ACQUIRE; + /** For CAS atomics only, adds weakness to {@code MO_RELEASE}. */ + public static final byte MO_WEAK_CAS_RELEASE = MO_WEAK_CAS|MO_RELEASE; + + // Note that acquire and release modes are for loads and stores, respectively. + // If a load is requested with MO_RELEASE, it may error, or rise to MO_VOLATILE + // If a store is requested with MO_ACQUIRE, it may error, or rise to MO_VOLATILE + // Debug versions of the VM may throw assertion errors more vigorously. + + /** + * Loads from a variable of any primitive type, using any of a + * variety of access methods. See {@link #getInt(Object, long)} + * for the connection to Java variables and off-heap variables. + * + * @param memoryOrder an encoding of the memory access order, one of + * {@link #MO_PLAIN} or another of those constants + * @param o Java heap object in which the variable resides, if any, else + * null + * @param offset indication of where the variable resides in a Java heap + * object, if any, else a memory address locating the variable + * statically + * @param basicType an encoding of the primitive type, one of + * {@link #BT_BOOLEAN} or another of those constants + * @return the value fetched from the indicated Java variable, + * padded by garbage high-order bits (if smaller than 64 bits) + * @throws RuntimeException No defined exceptions are thrown, not even + * {@link NullPointerException} + * @see #getInt(Object, long) + */ + @ForceInline + @IntrinsicCandidate + private long getPrimitiveBitsMO(byte memoryOrder, byte basicType, + Object o, long offset) { + // This intrinsic has a body so that the JIT can refuse to + // expand the intrinsic, when it sees a request it does not + // understand. + + // The purpose of the decision tree is to present the JIT with + // optimizable statements, even if memoryOrder or basicType + // fail to be constant. This allows the JIT to compile the + // body to serve calls from the interpreter and lukewarm code. + checkBasicType(basicType); + checkMemoryOrder(memoryOrder & ~MO_UNALIGNED); + long bits; + switch (basicType & PRIMITIVE_SIZE_MASK) { // encodes size of BT + case 3 -> { + if (memoryOrder == MO_PLAIN && basicType == BT_LONG && basicType == BT_LONG) { + return getPrimitiveBitsMONative(MO_PLAIN, BT_LONG, o, offset); + } + } + case 2 -> { + if (memoryOrder == MO_PLAIN && basicType == BT_INT) { + return getPrimitiveBitsMONative(MO_PLAIN, BT_INT, o, offset); + } + } + case 1 -> { + if (memoryOrder == MO_PLAIN && basicType == BT_SHORT) { + return getPrimitiveBitsMONative(MO_PLAIN, BT_SHORT, o, offset); + } + } + default -> { + if (memoryOrder == MO_PLAIN && basicType == BT_BYTE) { + return getPrimitiveBitsMONative(MO_PLAIN, BT_BYTE, o, offset); + } + } + } + + // this might end up in the native method + return getPrimitiveBitsMONative(memoryOrder, basicType, o, offset); + } + + // Second try for the JIT, if it dislikes a memory order or alignment request. + // Both this method and the previous are handled by the same native code + // and the same compiler intrinsic logic. + // The native method can treat every non-plain memory access as MO_VOLATILE. + // The native method can assert on unaligned requests if !UNALIGNED_ACCESS. + // The native method can assert on combinations which do not make sense. + // Prefer this method when you don't need the case analysis of the previous one. + @IntrinsicCandidate + private native long getPrimitiveBitsMONative(byte memoryOrder, byte basicType, + Object o, long offset); + + + /** + * Stores to a variable of any primitive type, using any of a + * variety of access methods. See {@link #putInt(Object, long, int)} + * for the connection to Java variables and off-heap variables. + * + * @param memoryOrder an encoding of the memory access order, one of + * {@link #MO_PLAIN} or another of those constants + * @param o Java heap object in which the variable resides, if any, else + * null + * @param offset indication of where the variable resides in a Java heap + * object, if any, else a memory address locating the variable + * statically + * @return the value to be stored to the indicated Java variable, + * padded by ignored high-order bits (if smaller than 64 bits) + * @param basicType an encoding of the primitive type, one of + * {@link #BT_BOOLEAN} or another of those constants + * @throws RuntimeException No defined exceptions are thrown, not even + * {@link NullPointerException} + * @see #getInt(Object, long) + */ + @ForceInline + @IntrinsicCandidate + private void putPrimitiveBitsMO(byte memoryOrder, byte basicType, + Object o, long offset, long bits) { + // This intrinsic has a body so that the JIT can refuse to + // expand the intrinsic, when it sees a request it does not + // understand. + + // The purpose of the decision tree is to present the JIT with + // optimizable statements, even if memoryOrder or basicType + // fail to be constant. This allows the JIT to compile the + // body to serve calls from the interpreter and lukewarm code. + checkBasicType(basicType); + checkMemoryOrder(memoryOrder & ~MO_UNALIGNED); + switch (basicType & PRIMITIVE_SIZE_MASK) { // encodes size of BT + case 3 -> { + if (memoryOrder == MO_PLAIN && basicType == BT_LONG && basicType == BT_LONG) { + putPrimitiveBitsMONative(MO_PLAIN, BT_LONG, o, offset, bits); + return; + } + } + case 2 -> { + if (memoryOrder == MO_PLAIN && basicType == BT_INT) { + putPrimitiveBitsMONative(MO_PLAIN, BT_INT, o, offset, bits); + return; + } + } + case 1 -> { + if (memoryOrder == MO_PLAIN && basicType == BT_SHORT) { + putPrimitiveBitsMONative(MO_PLAIN, BT_SHORT, o, offset, bits); + return; + } + } + default -> { + if (memoryOrder == MO_PLAIN && basicType == BT_BYTE) { + putPrimitiveBitsMONative(MO_PLAIN, BT_BYTE, o, offset, bits); + return; + } else if (basicType == BT_BOOLEAN) { + bits = bool2byte(byte2bool((byte)bits)); // do not store garbage booleans + break; + } + } + } + + // this might end up in the native method + putPrimitiveBitsMONative(memoryOrder, basicType, o, offset, bits); + } + + // Second try for the JIT, if it dislikes a memory order or alignment request. + // Both this method and the previous are handled by the same native code + // and the same compiler intrinsic logic. + // The native method can treat every non-plain memory access as MO_VOLATILE. + // The native method can assert on unaligned requests if !UNALIGNED_ACCESS. + // The native method can assert on combinations which do not make sense. + // Prefer this method when you don't need the case analysis of the previous one. + @IntrinsicCandidate + private native void putPrimitiveBitsMONative(byte memoryOrder, byte basicType, + Object o, long offset, long bits); + + @ForceInline + private static void checkMemoryOrder(int memoryOrder) { + // It's worth a bit of pain to make this fast in the interpreter. + final int MASK1 = ~(MO_PLAIN | MO_VOLATILE | + MO_ACQUIRE | MO_RELEASE | MO_OPAQUE); + final int MASK2 = ~(1<= BT_BOOLEAN && basicType <= BT_LONG) return; + throw new IllegalArgumentException("bad type"); + } + + // When converting from primitive bits, normal casts work fine for most types. + // But not boolean, float, or double. They need these unsurprising operators. + // The methods are placed here because the native methods which work on the + // 64-bit primitive bit representations must make compatible conversions. + // It is ugly that byte, short, int, and float pad with up to 32 sign bits, + // but that is a harmless detail internal to this API. + // The native primitive getter method is allowed to pad with zeroes or any + // other kind of garbage. + + /** Convert primitive representation bits to a float. Ignore high half. */ + @ForceInline + public static float bitsToFloat(long bits) { + return Float.intBitsToFloat((int)bits); + } + + /** Convert a float to primitive representation bits. Pad with 32 bits. */ + @ForceInline + public static long floatToBits(float x) { + return Float.floatToRawIntBits(x); + } + + /** Convert primitive representation bits to a double. */ + @ForceInline + public static double bitsToDouble(long bits) { + return Double.longBitsToDouble(bits); + } + + /** Convert a double to primitive representation bits. */ + @ForceInline + public static long doubleToBits(double x) { + return Double.doubleToRawLongBits(x); + } /** * Fetches a value from a given Java variable. @@ -160,8 +632,16 @@ public static Unsafe getUnsafe() { * @throws RuntimeException No defined exceptions are thrown, not even * {@link NullPointerException} */ - @IntrinsicCandidate - public native int getInt(Object o, long offset); + @ForceInline + public int getInt(Object o, long offset) { + return (int) getPrimitiveBitsMONative(MO_PLAIN, BT_INT, o, offset); + } + + /** Special-access version of {@link #getInt(Object, long)} */ + @ForceInline + public int getIntMO(byte memoryOrder, Object o, long offset) { + return (int) getPrimitiveBitsMO(memoryOrder, BT_INT, o, offset); + } /** * Stores a value into a given Java variable. @@ -183,15 +663,29 @@ public static Unsafe getUnsafe() { * @throws RuntimeException No defined exceptions are thrown, not even * {@link NullPointerException} */ - @IntrinsicCandidate - public native void putInt(Object o, long offset, int x); + @ForceInline + public void putInt(Object o, long offset, int x) { + putIntMO(MO_PLAIN, o, offset, x); + } + + /** Special-access version of {@link #putInt(Object, long, int)} */ + @ForceInline + public void putIntMO(byte memoryOrder, Object o, long offset, int x) { + putPrimitiveBitsMO(memoryOrder, BT_INT, o, offset, (long)x); + } /** * Fetches a reference value from a given Java variable. * @see #getInt(Object, long) */ + @ForceInline + public Object getReference(Object o, long offset) { + return getReferenceMO(MO_PLAIN, o, offset); + } + + /** Special-access version of {@link #getReference(Object, long)} */ @IntrinsicCandidate - public native Object getReference(Object o, long offset); + public native Object getReferenceMO(byte memoryOrder, Object o, long offset); /** * Stores a reference value into a given Java variable. @@ -203,64 +697,182 @@ public static Unsafe getUnsafe() { * are updated. * @see #putInt(Object, long, int) */ + @ForceInline + public void putReference(Object o, long offset, Object x) { + putReferenceMO(MO_PLAIN, o, offset, x); + } + + /** Special-access version of {@link #putReference(Object, long, Object)} */ @IntrinsicCandidate - public native void putReference(Object o, long offset, Object x); + public native void putReferenceMO(byte memoryOrder, Object o, long offset, Object x); /** @see #getInt(Object, long) */ - @IntrinsicCandidate - public native boolean getBoolean(Object o, long offset); + @ForceInline + public boolean getBoolean(Object o, long offset) { + return byte2bool((byte)getPrimitiveBitsMO(MO_PLAIN, BT_BOOLEAN, o, offset)); + } + + /** Special-access version. */ + @ForceInline + public boolean getBooleanMO(byte memoryOrder, Object o, long offset) { + return byte2bool((byte)getPrimitiveBitsMO(memoryOrder, BT_BOOLEAN, o, offset)); + } /** @see #putInt(Object, long, int) */ - @IntrinsicCandidate - public native void putBoolean(Object o, long offset, boolean x); + @ForceInline + public void putBoolean(Object o, long offset, boolean x) { + putPrimitiveBitsMO(MO_PLAIN, BT_BOOLEAN, o, offset, bool2byte(x)); + } + + /** Special-access version. */ + @ForceInline + public void putBooleanMO(byte memoryOrder, Object o, long offset, boolean x) { + putPrimitiveBitsMO(memoryOrder, BT_BOOLEAN, o, offset, bool2byte(x)); + } /** @see #getInt(Object, long) */ - @IntrinsicCandidate - public native byte getByte(Object o, long offset); + @ForceInline + public byte getByte(Object o, long offset) { + return (byte) getPrimitiveBitsMONative(MO_PLAIN, BT_BYTE, o, offset); + } + + /** Special-access version. */ + @ForceInline + public byte getByteMO(byte memoryOrder, Object o, long offset) { + return (byte) getPrimitiveBitsMO(memoryOrder, BT_BYTE, o, offset); + } /** @see #putInt(Object, long, int) */ - @IntrinsicCandidate - public native void putByte(Object o, long offset, byte x); + @ForceInline + public void putByte(Object o, long offset, byte x) { + putPrimitiveBitsMONative(MO_PLAIN, BT_BYTE, o, offset, (long)x); + } + + /** Special-access version. */ + @ForceInline + public void putByteMO(byte memoryOrder, Object o, long offset, byte x) { + putPrimitiveBitsMO(memoryOrder, BT_BYTE, o, offset, (long)x); + } /** @see #getInt(Object, long) */ - @IntrinsicCandidate - public native short getShort(Object o, long offset); + @ForceInline + public short getShort(Object o, long offset) { + return (short) getPrimitiveBitsMONative(MO_PLAIN, BT_SHORT, o, offset); + } + + /** Special-access version. */ + @ForceInline + public short getShortMO(byte memoryOrder, Object o, long offset) { + return (short) getPrimitiveBitsMO(memoryOrder, BT_SHORT, o, offset); + } /** @see #putInt(Object, long, int) */ - @IntrinsicCandidate - public native void putShort(Object o, long offset, short x); + @ForceInline + public void putShort(Object o, long offset, short x) { + putPrimitiveBitsMONative(MO_PLAIN, BT_SHORT, o, offset, (long)x); + } + + /** Special-access version. */ + @ForceInline + public void putShortMO(byte memoryOrder, Object o, long offset, short x) { + putPrimitiveBitsMO(memoryOrder, BT_SHORT, o, offset, (long)x); + } /** @see #getInt(Object, long) */ - @IntrinsicCandidate - public native char getChar(Object o, long offset); + @ForceInline + public char getChar(Object o, long offset) { + return (char) getPrimitiveBitsMONative(MO_PLAIN, BT_CHAR, o, offset); + } + + /** Special-access version. */ + @ForceInline + public char getCharMO(byte memoryOrder, Object o, long offset) { + return (char) getPrimitiveBitsMO(memoryOrder, BT_CHAR, o, offset); + } /** @see #putInt(Object, long, int) */ - @IntrinsicCandidate - public native void putChar(Object o, long offset, char x); + @ForceInline + public void putChar(Object o, long offset, char x) { + putPrimitiveBitsMONative(MO_PLAIN, BT_CHAR, o, offset, (long)x); + } + + /** Special-access version. */ + @ForceInline + public void putCharMO(byte memoryOrder, Object o, long offset, char x) { + putPrimitiveBitsMO(memoryOrder, BT_CHAR, o, offset, (long)x); + } /** @see #getInt(Object, long) */ - @IntrinsicCandidate - public native long getLong(Object o, long offset); + @ForceInline + public long getLong(Object o, long offset) { + return getPrimitiveBitsMONative(MO_PLAIN, BT_LONG, o, offset); + } + + /** Special-access version. */ + @ForceInline + public long getLongMO(byte memoryOrder, Object o, long offset) { + return getPrimitiveBitsMO(memoryOrder, BT_LONG, o, offset); + } /** @see #putInt(Object, long, int) */ - @IntrinsicCandidate - public native void putLong(Object o, long offset, long x); + @ForceInline + public void putLong(Object o, long offset, long x) { + putPrimitiveBitsMONative(MO_PLAIN, BT_LONG, o, offset, x); + } + + /** Special-access version. */ + @ForceInline + public void putLongMO(byte memoryOrder, Object o, long offset, long x) { + putPrimitiveBitsMO(memoryOrder, BT_LONG, o, offset, x); + } /** @see #getInt(Object, long) */ - @IntrinsicCandidate - public native float getFloat(Object o, long offset); + @ForceInline + public float getFloat(Object o, long offset) { + return bitsToFloat(getPrimitiveBitsMO(MO_PLAIN, BT_FLOAT, o, offset)); + } + + /** Special-access version. */ + @ForceInline + public float getFloatMO(byte memoryOrder, Object o, long offset) { + return bitsToFloat(getPrimitiveBitsMO(memoryOrder, BT_FLOAT, o, offset)); + } /** @see #putInt(Object, long, int) */ - @IntrinsicCandidate - public native void putFloat(Object o, long offset, float x); + @ForceInline + public void putFloat(Object o, long offset, float x) { + putPrimitiveBitsMO(MO_PLAIN, BT_FLOAT, o, offset, floatToBits(x)); + } + + /** Special-access version. */ + @ForceInline + public void putFloatMO(byte memoryOrder, Object o, long offset, float x) { + putPrimitiveBitsMO(memoryOrder, BT_FLOAT, o, offset, floatToBits(x)); + } /** @see #getInt(Object, long) */ - @IntrinsicCandidate - public native double getDouble(Object o, long offset); + @ForceInline + public double getDouble(Object o, long offset) { + return bitsToDouble(getPrimitiveBitsMO(MO_PLAIN, BT_DOUBLE, o, offset)); + } + + /** Special-access version. */ + @ForceInline + public double getDoubleMO(byte memoryOrder, Object o, long offset) { + return bitsToDouble(getPrimitiveBitsMO(memoryOrder, BT_DOUBLE, o, offset)); + } /** @see #putInt(Object, long, int) */ - @IntrinsicCandidate - public native void putDouble(Object o, long offset, double x); + @ForceInline + public void putDouble(Object o, long offset, double x) { + putPrimitiveBitsMO(MO_PLAIN, BT_DOUBLE, o, offset, doubleToBits(x)); + } + + /** Special-access version. */ + @ForceInline + public void putDoubleMO(byte memoryOrder, Object o, long offset, double x) { + putPrimitiveBitsMO(memoryOrder, BT_DOUBLE, o, offset, doubleToBits(x)); + } /** * Fetches a native pointer from a given memory address. If the address is @@ -306,19 +918,27 @@ public void putAddress(Object o, long offset, long x) { } } - // These read VM internal data. + // Methods that take no Object base address work on values in the + // C heap, or VM internal data. + // + // If another method (of the same name) accepts an Object base + // address, then omitting the Object base address is exactly + // equivalent to passing null for a base address. + // + // If you need a stronger memory order, you need to supply the + // null base explicitly as well as the memory order. So + // getInt(addr) works but then getIntMO(MO_VOLATILE, null, addr). /** * Fetches an uncompressed reference value from a given native variable * ignoring the VM's compressed references mode. + * The address must be known to the VM as an oop handle (or an equivalent). * * @param address a memory address locating the variable * @return the value fetched from the indicated native variable */ public native Object getUncompressedObject(long address); - // These work on values in the C heap. - /** * Fetches a value from a given memory address. If the address is zero, or * does not point into a block obtained from {@link #allocateMemory}, the @@ -1440,339 +2060,402 @@ private Object allocateUninitializedArray0(Class componentType, int length) { /** Throws the exception without telling the verifier. */ public native void throwException(Throwable ee); + // Here is the zoo of CAS operations in use (beyond the VH later): + // compareAndSet{Reference,Long,Int} (Volatile only) + // compareAndExchange{Reference,Long,Int}{Plain,Acquire,Release,''} + // weakCompareAndSet{Reference,Long,Int}{Plain,Acquire,Release,''} + // The default is MO_VOLATILE, so MO_PLAIN must be requested specifically. + // When a specific "flavor" is not available, MO_VOLATILE is a good fallback. + + // The actual API points come in a more regular zoo: + // compareAndSet{Reference,Long,Int,{otherprims}}{'',MO} + // compareAndExchange{Reference,Long,Int,{otherprims}}{'',MO} + // weakCompareAndSet{Reference,Long,Int} (only 2 primitives) + // The explicit MO argument is optional; MO_VOLATILE is the default. + // The weak CAS can be obtained from the regular one with MO_WEAK_CAS. + /** * Atomically updates Java variable to {@code x} if it is currently * holding {@code expected}. * - *

This operation has memory semantics of a {@code volatile} read - * and write. Corresponds to C11 atomic_compare_exchange_strong. + * Volatile memory order is used, and spurious failures (weak CAS) + * are excluded, but see also {@link #compareAndSetReferenceMO}. * * @return {@code true} if successful */ - @IntrinsicCandidate - public final native boolean compareAndSetReference(Object o, long offset, - Object expected, - Object x); - - @IntrinsicCandidate - public final native Object compareAndExchangeReference(Object o, long offset, - Object expected, - Object x); - - @IntrinsicCandidate - public final Object compareAndExchangeReferenceAcquire(Object o, long offset, - Object expected, - Object x) { - return compareAndExchangeReference(o, offset, expected, x); - } - - @IntrinsicCandidate - public final Object compareAndExchangeReferenceRelease(Object o, long offset, - Object expected, - Object x) { - return compareAndExchangeReference(o, offset, expected, x); - } - - @IntrinsicCandidate - public final boolean weakCompareAndSetReferencePlain(Object o, long offset, - Object expected, - Object x) { - return compareAndSetReference(o, offset, expected, x); - } - - @IntrinsicCandidate - public final boolean weakCompareAndSetReferenceAcquire(Object o, long offset, - Object expected, - Object x) { - return compareAndSetReference(o, offset, expected, x); - } - - @IntrinsicCandidate - public final boolean weakCompareAndSetReferenceRelease(Object o, long offset, - Object expected, - Object x) { - return compareAndSetReference(o, offset, expected, x); - } - - @IntrinsicCandidate - public final boolean weakCompareAndSetReference(Object o, long offset, - Object expected, - Object x) { - return compareAndSetReference(o, offset, expected, x); + @ForceInline + public boolean compareAndSetReference(Object o, long offset, + Object expected, + Object x) { + return compareAndSetReferenceMO(MO_VOLATILE, o, offset, expected, x); } /** * Atomically updates Java variable to {@code x} if it is currently * holding {@code expected}. * - *

This operation has memory semantics of a {@code volatile} read + *

When {@code memoryOrder} is {@code MO_VOLATILE}, this + * operation has memory semantics of a {@code volatile} read * and write. Corresponds to C11 atomic_compare_exchange_strong. * + *

When the bit {@link MO_WEAK_CAS} is set within the bits of + * {@code memoryOrder}, weak compare and set is selected. + * Otherwise, it is a strong CAS, and {@code MO_VOLATILE} + * should be selected. + * * @return {@code true} if successful */ @IntrinsicCandidate - public final native boolean compareAndSetInt(Object o, long offset, - int expected, - int x); - - @IntrinsicCandidate - public final native int compareAndExchangeInt(Object o, long offset, - int expected, - int x); + public native boolean compareAndSetReferenceMO(byte memoryOrder, + Object o, long offset, + Object expected, + Object x); - @IntrinsicCandidate - public final int compareAndExchangeIntAcquire(Object o, long offset, - int expected, - int x) { - return compareAndExchangeInt(o, offset, expected, x); + /** + * Atomically updates Java variable to {@code x} if it is currently + * holding {@code expected}. + * + * Volatile memory order is used, but see also {@link #compareAndExchangeReferenceMO}. + * + * @return the previous value of the Java variable + */ + @ForceInline + public Object compareAndExchangeReference(Object o, long offset, + Object expected, + Object x) { + return compareAndExchangeReferenceMO(MO_VOLATILE, o, offset, expected, x); } - @IntrinsicCandidate - public final int compareAndExchangeIntRelease(Object o, long offset, - int expected, - int x) { - return compareAndExchangeInt(o, offset, expected, x); + /** Convenience for {@code compareAndSetReferenceMO(MO_WEAK_CAS_VOLATILE ...)}. */ + @ForceInline + public boolean weakCompareAndSetReference(Object o, long offset, + Object expected, + Object x) { + return compareAndSetReferenceMO(MO_WEAK_CAS_VOLATILE, o, offset, expected, x); } + // The native method can treat every memory access as MO_VOLATILE. @IntrinsicCandidate - public final boolean weakCompareAndSetIntPlain(Object o, long offset, - int expected, - int x) { - return compareAndSetInt(o, offset, expected, x); - } + public native Object compareAndExchangeReferenceMO(byte memoryOrder, + Object o, long offset, + Object expected, + Object x); + /** + * Intrinsic for performing compare-and-set on a primitive variable + * of any primitive type. See {@link #putPrimitiveBitsMO} for the + * relevant conventions. + * + *

As with {@link #compareAndSetReferenceMO}, the bit + * {@link MO_WEAK_CAS} may be set within {@code memoryOrder}. + */ @IntrinsicCandidate - public final boolean weakCompareAndSetIntAcquire(Object o, long offset, - int expected, - int x) { - return compareAndSetInt(o, offset, expected, x); - } + private final boolean compareAndSetPrimitiveBitsMO(byte memoryOrder, + byte basicType, + Object o, long offset, + long expected, + long x) { + // This intrinsic has a body so that the JIT can refuse to + // expand the intrinsic, when it sees a request it does not + // understand. - @IntrinsicCandidate - public final boolean weakCompareAndSetIntRelease(Object o, long offset, - int expected, - int x) { - return compareAndSetInt(o, offset, expected, x); - } + // The purpose of the decision tree is to present the JIT with + // optimizable statements, even if memoryOrder or basicType + // fail to be constant. - @IntrinsicCandidate - public final boolean weakCompareAndSetInt(Object o, long offset, - int expected, - int x) { - return compareAndSetInt(o, offset, expected, x); - } - - @IntrinsicCandidate - public final byte compareAndExchangeByte(Object o, long offset, - byte expected, - byte x) { - long wordOffset = offset & ~3; - int shift = (int) (offset & 3) << 3; - if (BIG_ENDIAN) { - shift = 24 - shift; + // This also allows the JIT to compile the body to serve calls + // from the interpreter and lukewarm code. + checkBasicType(basicType); + memoryOrder &= ~MO_WEAK_CAS; // intrinsic might use this bit but not here + checkMemoryOrder(memoryOrder); + switch (basicType & PRIMITIVE_SIZE_MASK) { // encodes size of BT + case 3 -> { + // hardware must support 8-byte CAS + return compareAndSetPrimitiveBitsMONative(MO_VOLATILE, BT_LONG, o, offset, + expected, x); + } + case 2 -> { + // hardware must support 4-byte CAS + return compareAndSetPrimitiveBitsMONative(MO_VOLATILE, BT_INT, o, offset, + expected, x); + } + case 1 -> { + // The hardware might not have 2-byte CAS; simulate with volatile 2-byte cmpxchg. + return ((short) compareAndExchangePrimitiveBitsMO(MO_VOLATILE, BT_SHORT, o, offset, + (short)expected, (short)x) + == (short)expected); + } + default -> { + // The hardware might not have 1-byte CAS; simulate with volatile 1-byte cmpxchg. + return ((byte) compareAndExchangePrimitiveBitsMO(MO_VOLATILE, BT_BYTE, o, offset, + (byte)expected, (byte)x) + == (byte)expected); + } } - int mask = 0xFF << shift; - int maskedExpected = (expected & 0xFF) << shift; - int maskedX = (x & 0xFF) << shift; - int fullWord; - do { - fullWord = getIntVolatile(o, wordOffset); - if ((fullWord & mask) != maskedExpected) - return (byte) ((fullWord & mask) >> shift); - } while (!weakCompareAndSetInt(o, wordOffset, - fullWord, (fullWord & ~mask) | maskedX)); - return expected; } - @IntrinsicCandidate - public final boolean compareAndSetByte(Object o, long offset, - byte expected, - byte x) { - return compareAndExchangeByte(o, offset, expected, x) == expected; - } - - @IntrinsicCandidate - public final boolean weakCompareAndSetByte(Object o, long offset, - byte expected, - byte x) { - return compareAndSetByte(o, offset, expected, x); - } - - @IntrinsicCandidate - public final boolean weakCompareAndSetByteAcquire(Object o, long offset, - byte expected, - byte x) { - return weakCompareAndSetByte(o, offset, expected, x); - } - - @IntrinsicCandidate - public final boolean weakCompareAndSetByteRelease(Object o, long offset, - byte expected, - byte x) { - return weakCompareAndSetByte(o, offset, expected, x); - } - - @IntrinsicCandidate - public final boolean weakCompareAndSetBytePlain(Object o, long offset, - byte expected, - byte x) { - return weakCompareAndSetByte(o, offset, expected, x); - } - - @IntrinsicCandidate - public final byte compareAndExchangeByteAcquire(Object o, long offset, - byte expected, - byte x) { - return compareAndExchangeByte(o, offset, expected, x); - } + // Second try for the JIT, if it dislikes a memory order or data type. + // Both this method and the previous are handled by the same native code + // and the same compiler intrinsic logic. + // The native method can treat every memory access as MO_VOLATILE. + // The native method can assert on data types which have no atomic support. + // The native method can assert on combinations which do not make sense. + @IntrinsicCandidate + private native boolean compareAndSetPrimitiveBitsMONative(byte memoryOrder, + byte basicType, + Object o, long offset, + long expected, + long x); + + /** + * Follows the conventions of {@link #putPrimitiveBitsMO}. + * @return the value fetched from the indicated Java variable, + * padded by garbage high-order bits (if smaller than 64 bits) + */ + @IntrinsicCandidate + private final long compareAndExchangePrimitiveBitsMO(byte memoryOrder, + byte basicType, + Object o, long offset, + long expected, + long x) { + // This intrinsic has a body so that the JIT can refuse to + // expand the intrinsic, when it sees a request it does not + // understand. + + // The purpose of the decision tree is to present the JIT with + // optimizable statements, even if memoryOrder or basicType + // fail to be constant. + + // This also allows the JIT to compile the body to serve calls + // from the interpreter and lukewarm code. + checkBasicType(basicType); + checkMemoryOrder(memoryOrder); + switch (basicType & PRIMITIVE_SIZE_MASK) { // encodes size of BT + case 3 -> { + // hardware must support 8-byte cmpxchg + return compareAndExchangePrimitiveBitsMONative(MO_VOLATILE, BT_LONG, o, offset, + expected, x); + } + case 2 -> { + // hardware must support 4-byte cmpxchg + return compareAndExchangePrimitiveBitsMONative(MO_VOLATILE, BT_INT, o, offset, + expected, x); + } + case 1 -> { + // The default implementation updates a byte or short inside an atomic + // int container, if the JIT refuses to expand this intrinsic. + return (short) compareAndExchangeUsingIntSlow(o, offset, (short)expected, (short)x, Short.SIZE / Byte.SIZE); + } - @IntrinsicCandidate - public final byte compareAndExchangeByteRelease(Object o, long offset, - byte expected, - byte x) { - return compareAndExchangeByte(o, offset, expected, x); + default -> { + return (byte) compareAndExchangeUsingIntSlow(o, offset, (byte)expected, (byte)x, 1); + } + } } - @IntrinsicCandidate - public final short compareAndExchangeShort(Object o, long offset, - short expected, - short x) { - if ((offset & 3) == 3) { + // Second try for the JIT, if it dislikes a memory order or data type. + // Both this method and the previous are handled by the same native code + // and the same compiler intrinsic logic. + // The native method can treat every memory access as MO_VOLATILE. + // The native method can assert on data types which have no atomic support. + // The native method can assert on combinations which do not make sense. + // Prefer this method when you don't need the case analysis of the previous one. + @IntrinsicCandidate + private native long compareAndExchangePrimitiveBitsMONative(byte memoryOrder, + byte basicType, + Object o, long offset, + long expected, + long x); + + // Fallback implementation of cmpxchg for subword types. + private int compareAndExchangeUsingIntSlow(Object o, long offset, + int expected, + int x, int byteSize) { + if (!(byteSize == 1 || byteSize == 2)) { + throw new IllegalArgumentException("bad subword size"); + } + int bitSize = byteSize * Byte.SIZE; + int allbits = (1 << bitSize) - 1; // 0xFFFF or 0xFF + expected &= allbits; + x &= allbits; + int byteOffset = (int)(offset & 3); + int bitOffset = byteOffset * Byte.SIZE; + if (bitOffset + bitSize > Integer.SIZE) { throw new IllegalArgumentException("Update spans the word, not supported"); } - long wordOffset = offset & ~3; - int shift = (int) (offset & 3) << 3; - if (BIG_ENDIAN) { - shift = 16 - shift; + long wordOffset = offset - byteOffset; + int shift = bitOffset; + if (BIG_ENDIAN) { // start counting from the top of the word + shift = Integer.SIZE - bitSize - shift; } - int mask = 0xFFFF << shift; - int maskedExpected = (expected & 0xFFFF) << shift; - int maskedX = (x & 0xFFFF) << shift; + int mask = allbits << shift; + int maskedExpected = expected << shift; + int maskedX = x << shift; int fullWord; do { - fullWord = getIntVolatile(o, wordOffset); + fullWord = getInt(o, wordOffset); if ((fullWord & mask) != maskedExpected) { - return (short) ((fullWord & mask) >> shift); + return (fullWord >>> shift) & allbits; } - } while (!weakCompareAndSetInt(o, wordOffset, - fullWord, (fullWord & ~mask) | maskedX)); + } while (!weakCompareAndSetInt(o, wordOffset, fullWord, (fullWord & ~mask) | maskedX)); return expected; } - @IntrinsicCandidate - public final boolean compareAndSetShort(Object o, long offset, - short expected, - short x) { - return compareAndExchangeShort(o, offset, expected, x) == expected; + /** + * Atomically updates Java variable to {@code x} if it is currently + * holding {@code expected}. + * + *

This operation has memory semantics of a {@code volatile} read + * and write. Corresponds to C11 atomic_compare_exchange_strong. + * + * @return {@code true} if successful + */ + @ForceInline + public boolean compareAndSetInt(Object o, long offset, + int expected, + int x) { + return compareAndSetPrimitiveBitsMONative(MO_VOLATILE, BT_INT, o, offset, expected, x); } - @IntrinsicCandidate - public final boolean weakCompareAndSetShort(Object o, long offset, - short expected, - short x) { - return compareAndSetShort(o, offset, expected, x); + /** Special-access version of {@code compareAndSetInt}. */ + @ForceInline + public boolean compareAndSetIntMO(byte memoryOrder, + Object o, long offset, + int expected, + int x) { + return compareAndSetPrimitiveBitsMONative(memoryOrder, BT_INT, o, offset, expected, x); } - @IntrinsicCandidate - public final boolean weakCompareAndSetShortAcquire(Object o, long offset, - short expected, - short x) { - return weakCompareAndSetShort(o, offset, expected, x); + @ForceInline + public int compareAndExchangeInt(Object o, long offset, + int expected, + int x) { + return (int) compareAndExchangePrimitiveBitsMONative(MO_VOLATILE, BT_INT, + o, offset, expected, x); } - @IntrinsicCandidate - public final boolean weakCompareAndSetShortRelease(Object o, long offset, - short expected, - short x) { - return weakCompareAndSetShort(o, offset, expected, x); + /** Special-access version of {@code compareAndExchangeInt}. */ + @ForceInline + public int compareAndExchangeIntMO(byte memoryOrder, + Object o, long offset, + int expected, + int x) { + return (int) compareAndExchangePrimitiveBitsMONative(memoryOrder, BT_INT, + o, offset, expected, x); } - @IntrinsicCandidate - public final boolean weakCompareAndSetShortPlain(Object o, long offset, - short expected, - short x) { - return weakCompareAndSetShort(o, offset, expected, x); + /** Convenience for {@code compareAndSetIntMO(MO_WEAK_CAS_VOLATILE ...)}. */ + @ForceInline + public boolean weakCompareAndSetInt(Object o, long offset, + int expected, + int x) { + return compareAndSetIntMO(MO_WEAK_CAS_VOLATILE, o, offset, expected, x); } + @ForceInline + public byte compareAndExchangeByte(Object o, long offset, + byte expected, + byte x) { + return (byte) compareAndExchangePrimitiveBitsMO(MO_VOLATILE, BT_BYTE, + o, offset, expected, x); + } - @IntrinsicCandidate - public final short compareAndExchangeShortAcquire(Object o, long offset, - short expected, - short x) { - return compareAndExchangeShort(o, offset, expected, x); + @ForceInline + public byte compareAndExchangeByteMO(byte memoryOrder, + Object o, long offset, + byte expected, + byte x) { + return (byte) compareAndExchangePrimitiveBitsMO(memoryOrder, BT_BYTE, + o, offset, expected, + x); } - @IntrinsicCandidate - public final short compareAndExchangeShortRelease(Object o, long offset, - short expected, - short x) { - return compareAndExchangeShort(o, offset, expected, x); + // The compiler may replace this intrinsic if the backend supports + // a boolean-returning CAS, for the particular requested memory order. + @ForceInline + public boolean compareAndSetByteMO(byte memoryOrder, + Object o, long offset, + byte expected, + byte x) { + return compareAndSetPrimitiveBitsMO(memoryOrder, BT_BYTE, o, offset, expected, x); } @ForceInline - private char s2c(short s) { - return (char) s; + public boolean compareAndSetByte(Object o, long offset, + byte expected, + byte x) { + return compareAndSetPrimitiveBitsMO(MO_VOLATILE, BT_BYTE, o, offset, expected, x); } @ForceInline - private short c2s(char s) { - return (short) s; + public short compareAndExchangeShort(Object o, long offset, + short expected, + short x) { + return compareAndExchangeShortMO(MO_VOLATILE, o, offset, expected, x); } @ForceInline - public final boolean compareAndSetChar(Object o, long offset, - char expected, - char x) { - return compareAndSetShort(o, offset, c2s(expected), c2s(x)); + public short compareAndExchangeShortMO(byte memoryOrder, + Object o, long offset, + short expected, + short x) { + return (short) compareAndExchangePrimitiveBitsMO(memoryOrder, BT_SHORT, + o, offset, expected, x); } @ForceInline - public final char compareAndExchangeChar(Object o, long offset, - char expected, - char x) { - return s2c(compareAndExchangeShort(o, offset, c2s(expected), c2s(x))); + public boolean compareAndSetShortMO(byte memoryOrder, + Object o, long offset, + short expected, + short x) { + return compareAndSetPrimitiveBitsMO(memoryOrder, BT_SHORT, o, offset, expected, x); } @ForceInline - public final char compareAndExchangeCharAcquire(Object o, long offset, - char expected, - char x) { - return s2c(compareAndExchangeShortAcquire(o, offset, c2s(expected), c2s(x))); + public boolean compareAndSetShort(Object o, long offset, + short expected, + short x) { + return compareAndSetPrimitiveBitsMO(MO_VOLATILE, BT_SHORT, o, offset, expected, x); } @ForceInline - public final char compareAndExchangeCharRelease(Object o, long offset, - char expected, - char x) { - return s2c(compareAndExchangeShortRelease(o, offset, c2s(expected), c2s(x))); + private char s2c(short s) { + return (char) s; } @ForceInline - public final boolean weakCompareAndSetChar(Object o, long offset, - char expected, - char x) { - return weakCompareAndSetShort(o, offset, c2s(expected), c2s(x)); + private short c2s(char s) { + return (short) s; } @ForceInline - public final boolean weakCompareAndSetCharAcquire(Object o, long offset, - char expected, - char x) { - return weakCompareAndSetShortAcquire(o, offset, c2s(expected), c2s(x)); + public boolean compareAndSetChar(Object o, long offset, + char expected, + char x) { + return compareAndSetShortMO(MO_VOLATILE, o, offset, c2s(expected), c2s(x)); } @ForceInline - public final boolean weakCompareAndSetCharRelease(Object o, long offset, - char expected, - char x) { - return weakCompareAndSetShortRelease(o, offset, c2s(expected), c2s(x)); + public boolean compareAndSetCharMO(byte memoryOrder, + Object o, long offset, + char expected, + char x) { + return compareAndSetShortMO(memoryOrder, o, offset, c2s(expected), c2s(x)); } @ForceInline - public final boolean weakCompareAndSetCharPlain(Object o, long offset, - char expected, - char x) { - return weakCompareAndSetShortPlain(o, offset, c2s(expected), c2s(x)); + public char compareAndExchangeChar(Object o, long offset, + char expected, + char x) { + return s2c(compareAndExchangeShort(o, offset, c2s(expected), c2s(x))); + } + + @ForceInline + public char compareAndExchangeCharMO(byte memoryOrder, + Object o, long offset, + char expected, + char x) { + return s2c(compareAndExchangeShortMO(memoryOrder, o, offset, c2s(expected), c2s(x))); } /** @@ -1825,59 +2508,33 @@ private byte bool2byte(boolean b) { } @ForceInline - public final boolean compareAndSetBoolean(Object o, long offset, + public boolean compareAndSetBoolean(Object o, long offset, boolean expected, boolean x) { return compareAndSetByte(o, offset, bool2byte(expected), bool2byte(x)); } @ForceInline - public final boolean compareAndExchangeBoolean(Object o, long offset, - boolean expected, - boolean x) { - return byte2bool(compareAndExchangeByte(o, offset, bool2byte(expected), bool2byte(x))); - } - - @ForceInline - public final boolean compareAndExchangeBooleanAcquire(Object o, long offset, - boolean expected, - boolean x) { - return byte2bool(compareAndExchangeByteAcquire(o, offset, bool2byte(expected), bool2byte(x))); - } - - @ForceInline - public final boolean compareAndExchangeBooleanRelease(Object o, long offset, - boolean expected, - boolean x) { - return byte2bool(compareAndExchangeByteRelease(o, offset, bool2byte(expected), bool2byte(x))); - } - - @ForceInline - public final boolean weakCompareAndSetBoolean(Object o, long offset, - boolean expected, - boolean x) { - return weakCompareAndSetByte(o, offset, bool2byte(expected), bool2byte(x)); + public boolean compareAndSetBooleanMO(byte memoryOrder, + Object o, long offset, + boolean expected, + boolean x) { + return compareAndSetByteMO(memoryOrder, o, offset, bool2byte(expected), bool2byte(x)); } @ForceInline - public final boolean weakCompareAndSetBooleanAcquire(Object o, long offset, - boolean expected, - boolean x) { - return weakCompareAndSetByteAcquire(o, offset, bool2byte(expected), bool2byte(x)); - } - - @ForceInline - public final boolean weakCompareAndSetBooleanRelease(Object o, long offset, - boolean expected, - boolean x) { - return weakCompareAndSetByteRelease(o, offset, bool2byte(expected), bool2byte(x)); + public boolean compareAndExchangeBoolean(Object o, long offset, + boolean expected, + boolean x) { + return byte2bool(compareAndExchangeByte(o, offset, bool2byte(expected), bool2byte(x))); } @ForceInline - public final boolean weakCompareAndSetBooleanPlain(Object o, long offset, - boolean expected, - boolean x) { - return weakCompareAndSetBytePlain(o, offset, bool2byte(expected), bool2byte(x)); + public boolean compareAndExchangeBooleanMO(byte memoryOrder, + Object o, long offset, + boolean expected, + boolean x) { + return byte2bool(compareAndExchangeByteMO(memoryOrder, o, offset, bool2byte(expected), bool2byte(x))); } /** @@ -1890,16 +2547,26 @@ public final boolean weakCompareAndSetBooleanPlain(Object o, long offset, * @return {@code true} if successful */ @ForceInline - public final boolean compareAndSetFloat(Object o, long offset, + public boolean compareAndSetFloat(Object o, long offset, float expected, float x) { return compareAndSetInt(o, offset, - Float.floatToRawIntBits(expected), - Float.floatToRawIntBits(x)); + Float.floatToRawIntBits(expected), + Float.floatToRawIntBits(x)); + } + + @ForceInline + public boolean compareAndSetFloatMO(byte memoryOrder, + Object o, long offset, + float expected, + float x) { + return compareAndSetIntMO(memoryOrder, o, offset, + Float.floatToRawIntBits(expected), + Float.floatToRawIntBits(x)); } @ForceInline - public final float compareAndExchangeFloat(Object o, long offset, + public float compareAndExchangeFloat(Object o, long offset, float expected, float x) { int w = compareAndExchangeInt(o, offset, @@ -1909,61 +2576,16 @@ public final float compareAndExchangeFloat(Object o, long offset, } @ForceInline - public final float compareAndExchangeFloatAcquire(Object o, long offset, - float expected, - float x) { - int w = compareAndExchangeIntAcquire(o, offset, - Float.floatToRawIntBits(expected), - Float.floatToRawIntBits(x)); - return Float.intBitsToFloat(w); - } - - @ForceInline - public final float compareAndExchangeFloatRelease(Object o, long offset, - float expected, - float x) { - int w = compareAndExchangeIntRelease(o, offset, - Float.floatToRawIntBits(expected), - Float.floatToRawIntBits(x)); + public float compareAndExchangeFloatMO(byte memoryOrder, + Object o, long offset, + float expected, + float x) { + int w = compareAndExchangeIntMO(memoryOrder, o, offset, + Float.floatToRawIntBits(expected), + Float.floatToRawIntBits(x)); return Float.intBitsToFloat(w); } - @ForceInline - public final boolean weakCompareAndSetFloatPlain(Object o, long offset, - float expected, - float x) { - return weakCompareAndSetIntPlain(o, offset, - Float.floatToRawIntBits(expected), - Float.floatToRawIntBits(x)); - } - - @ForceInline - public final boolean weakCompareAndSetFloatAcquire(Object o, long offset, - float expected, - float x) { - return weakCompareAndSetIntAcquire(o, offset, - Float.floatToRawIntBits(expected), - Float.floatToRawIntBits(x)); - } - - @ForceInline - public final boolean weakCompareAndSetFloatRelease(Object o, long offset, - float expected, - float x) { - return weakCompareAndSetIntRelease(o, offset, - Float.floatToRawIntBits(expected), - Float.floatToRawIntBits(x)); - } - - @ForceInline - public final boolean weakCompareAndSetFloat(Object o, long offset, - float expected, - float x) { - return weakCompareAndSetInt(o, offset, - Float.floatToRawIntBits(expected), - Float.floatToRawIntBits(x)); - } - /** * Atomically updates Java variable to {@code x} if it is currently * holding {@code expected}. @@ -1974,7 +2596,7 @@ public final boolean weakCompareAndSetFloat(Object o, long offset, * @return {@code true} if successful */ @ForceInline - public final boolean compareAndSetDouble(Object o, long offset, + public boolean compareAndSetDouble(Object o, long offset, double expected, double x) { return compareAndSetLong(o, offset, @@ -1983,7 +2605,17 @@ public final boolean compareAndSetDouble(Object o, long offset, } @ForceInline - public final double compareAndExchangeDouble(Object o, long offset, + public boolean compareAndSetDoubleMO(byte memoryOrder, + Object o, long offset, + double expected, + double x) { + return compareAndSetLongMO(memoryOrder, o, offset, + Double.doubleToRawLongBits(expected), + Double.doubleToRawLongBits(x)); + } + + @ForceInline + public double compareAndExchangeDouble(Object o, long offset, double expected, double x) { long w = compareAndExchangeLong(o, offset, @@ -1993,61 +2625,16 @@ public final double compareAndExchangeDouble(Object o, long offset, } @ForceInline - public final double compareAndExchangeDoubleAcquire(Object o, long offset, - double expected, - double x) { - long w = compareAndExchangeLongAcquire(o, offset, - Double.doubleToRawLongBits(expected), - Double.doubleToRawLongBits(x)); - return Double.longBitsToDouble(w); - } - - @ForceInline - public final double compareAndExchangeDoubleRelease(Object o, long offset, - double expected, - double x) { - long w = compareAndExchangeLongRelease(o, offset, - Double.doubleToRawLongBits(expected), - Double.doubleToRawLongBits(x)); + public double compareAndExchangeDoubleMO(byte memoryOrder, + Object o, long offset, + double expected, + double x) { + long w = compareAndExchangeLongMO(memoryOrder, o, offset, + Double.doubleToRawLongBits(expected), + Double.doubleToRawLongBits(x)); return Double.longBitsToDouble(w); } - @ForceInline - public final boolean weakCompareAndSetDoublePlain(Object o, long offset, - double expected, - double x) { - return weakCompareAndSetLongPlain(o, offset, - Double.doubleToRawLongBits(expected), - Double.doubleToRawLongBits(x)); - } - - @ForceInline - public final boolean weakCompareAndSetDoubleAcquire(Object o, long offset, - double expected, - double x) { - return weakCompareAndSetLongAcquire(o, offset, - Double.doubleToRawLongBits(expected), - Double.doubleToRawLongBits(x)); - } - - @ForceInline - public final boolean weakCompareAndSetDoubleRelease(Object o, long offset, - double expected, - double x) { - return weakCompareAndSetLongRelease(o, offset, - Double.doubleToRawLongBits(expected), - Double.doubleToRawLongBits(x)); - } - - @ForceInline - public final boolean weakCompareAndSetDouble(Object o, long offset, - double expected, - double x) { - return weakCompareAndSetLong(o, offset, - Double.doubleToRawLongBits(expected), - Double.doubleToRawLongBits(x)); - } - /** * Atomically updates Java variable to {@code x} if it is currently * holding {@code expected}. @@ -2057,365 +2644,189 @@ public final boolean weakCompareAndSetDouble(Object o, long offset, * * @return {@code true} if successful */ - @IntrinsicCandidate - public final native boolean compareAndSetLong(Object o, long offset, - long expected, - long x); - - @IntrinsicCandidate - public final native long compareAndExchangeLong(Object o, long offset, - long expected, - long x); - - @IntrinsicCandidate - public final long compareAndExchangeLongAcquire(Object o, long offset, - long expected, - long x) { - return compareAndExchangeLong(o, offset, expected, x); - } - - @IntrinsicCandidate - public final long compareAndExchangeLongRelease(Object o, long offset, - long expected, - long x) { - return compareAndExchangeLong(o, offset, expected, x); + @ForceInline + public boolean compareAndSetLong(Object o, long offset, + long expected, + long x) { + return compareAndSetPrimitiveBitsMONative(MO_VOLATILE, BT_LONG, o, offset, expected, x); } - @IntrinsicCandidate - public final boolean weakCompareAndSetLongPlain(Object o, long offset, - long expected, - long x) { - return compareAndSetLong(o, offset, expected, x); + /** Special-access version. */ + @ForceInline + public boolean compareAndSetLongMO(byte memoryOrder, + Object o, long offset, + long expected, + long x) { + return compareAndSetPrimitiveBitsMONative(memoryOrder, BT_LONG, o, offset, expected, x); } - @IntrinsicCandidate - public final boolean weakCompareAndSetLongAcquire(Object o, long offset, - long expected, - long x) { - return compareAndSetLong(o, offset, expected, x); + /** Regular volatile version. */ + @ForceInline + public long compareAndExchangeLong(Object o, long offset, + long expected, + long x) { + return compareAndExchangePrimitiveBitsMONative(MO_VOLATILE, BT_LONG, + o, offset, expected, x); } - @IntrinsicCandidate - public final boolean weakCompareAndSetLongRelease(Object o, long offset, - long expected, - long x) { - return compareAndSetLong(o, offset, expected, x); + /** Special-access version. */ + @ForceInline + public long compareAndExchangeLongMO(byte memoryOrder, + Object o, long offset, + long expected, + long x) { + return compareAndExchangePrimitiveBitsMONative(memoryOrder, BT_LONG, + o, offset, expected, x); } - @IntrinsicCandidate - public final boolean weakCompareAndSetLong(Object o, long offset, + /** Convenience for {@code compareAndSetLongMO(MO_WEAK_CAS_VOLATILE ...)}. */ + @ForceInline + public boolean weakCompareAndSetLong(Object o, long offset, long expected, long x) { - return compareAndSetLong(o, offset, expected, x); + return compareAndSetPrimitiveBitsMONative(MO_WEAK_CAS_VOLATILE, BT_LONG, + o, offset, expected, x); } + // Note: The best way to get fine control over memory order is to + // use an explicit memory order prefix argument like MO_RELEASE. + // There used to be a large number of API points like + // putBooleanRelease, getIntAcquire, getReferenceOpaque, etc. + // Such operations are more regularly available via the MO arguments. + + // Because "volatile" is a Java modifier (unlike acquire or release), + // old code (notably, old tests) use the MO_VOLATILE versions often. + // So we keep those API points, as a compromise. New code should use + // the prefix MO_VOLATILE. + /** * Fetches a reference value from a given Java variable, with volatile - * load semantics. Otherwise identical to {@link #getReference(Object, long)} + * load semantics. + * Identical to {@link #getReferenceMO(byte, Object, long)} + * with the prefix argument MO_VOLATILE. */ - @IntrinsicCandidate - public native Object getReferenceVolatile(Object o, long offset); + @ForceInline + public Object getReferenceVolatile(Object o, long offset) { + return getReferenceMO(MO_VOLATILE, o, offset); + } /** * Stores a reference value into a given Java variable, with - * volatile store semantics. Otherwise identical to {@link #putReference(Object, long, Object)} + * volatile store semantics. + * Identical to {@link #putReferenceMO(byte, Object, long, Object)} + * with the prefix argument MO_VOLATILE. */ - @IntrinsicCandidate - public native void putReferenceVolatile(Object o, long offset, Object x); - - /** Volatile version of {@link #getInt(Object, long)} */ - @IntrinsicCandidate - public native int getIntVolatile(Object o, long offset); - - /** Volatile version of {@link #putInt(Object, long, int)} */ - @IntrinsicCandidate - public native void putIntVolatile(Object o, long offset, int x); - - /** Volatile version of {@link #getBoolean(Object, long)} */ - @IntrinsicCandidate - public native boolean getBooleanVolatile(Object o, long offset); - - /** Volatile version of {@link #putBoolean(Object, long, boolean)} */ - @IntrinsicCandidate - public native void putBooleanVolatile(Object o, long offset, boolean x); - - /** Volatile version of {@link #getByte(Object, long)} */ - @IntrinsicCandidate - public native byte getByteVolatile(Object o, long offset); - - /** Volatile version of {@link #putByte(Object, long, byte)} */ - @IntrinsicCandidate - public native void putByteVolatile(Object o, long offset, byte x); - - /** Volatile version of {@link #getShort(Object, long)} */ - @IntrinsicCandidate - public native short getShortVolatile(Object o, long offset); - - /** Volatile version of {@link #putShort(Object, long, short)} */ - @IntrinsicCandidate - public native void putShortVolatile(Object o, long offset, short x); - - /** Volatile version of {@link #getChar(Object, long)} */ - @IntrinsicCandidate - public native char getCharVolatile(Object o, long offset); - - /** Volatile version of {@link #putChar(Object, long, char)} */ - @IntrinsicCandidate - public native void putCharVolatile(Object o, long offset, char x); - - /** Volatile version of {@link #getLong(Object, long)} */ - @IntrinsicCandidate - public native long getLongVolatile(Object o, long offset); - - /** Volatile version of {@link #putLong(Object, long, long)} */ - @IntrinsicCandidate - public native void putLongVolatile(Object o, long offset, long x); - - /** Volatile version of {@link #getFloat(Object, long)} */ - @IntrinsicCandidate - public native float getFloatVolatile(Object o, long offset); - - /** Volatile version of {@link #putFloat(Object, long, float)} */ - @IntrinsicCandidate - public native void putFloatVolatile(Object o, long offset, float x); - - /** Volatile version of {@link #getDouble(Object, long)} */ - @IntrinsicCandidate - public native double getDoubleVolatile(Object o, long offset); - - /** Volatile version of {@link #putDouble(Object, long, double)} */ - @IntrinsicCandidate - public native void putDoubleVolatile(Object o, long offset, double x); - - - - /** Acquire version of {@link #getReferenceVolatile(Object, long)} */ - @IntrinsicCandidate - public final Object getReferenceAcquire(Object o, long offset) { - return getReferenceVolatile(o, offset); - } - - /** Acquire version of {@link #getBooleanVolatile(Object, long)} */ - @IntrinsicCandidate - public final boolean getBooleanAcquire(Object o, long offset) { - return getBooleanVolatile(o, offset); - } - - /** Acquire version of {@link #getByteVolatile(Object, long)} */ - @IntrinsicCandidate - public final byte getByteAcquire(Object o, long offset) { - return getByteVolatile(o, offset); - } - - /** Acquire version of {@link #getShortVolatile(Object, long)} */ - @IntrinsicCandidate - public final short getShortAcquire(Object o, long offset) { - return getShortVolatile(o, offset); - } - - /** Acquire version of {@link #getCharVolatile(Object, long)} */ - @IntrinsicCandidate - public final char getCharAcquire(Object o, long offset) { - return getCharVolatile(o, offset); - } - - /** Acquire version of {@link #getIntVolatile(Object, long)} */ - @IntrinsicCandidate - public final int getIntAcquire(Object o, long offset) { - return getIntVolatile(o, offset); - } - - /** Acquire version of {@link #getFloatVolatile(Object, long)} */ - @IntrinsicCandidate - public final float getFloatAcquire(Object o, long offset) { - return getFloatVolatile(o, offset); - } - - /** Acquire version of {@link #getLongVolatile(Object, long)} */ - @IntrinsicCandidate - public final long getLongAcquire(Object o, long offset) { - return getLongVolatile(o, offset); - } - - /** Acquire version of {@link #getDoubleVolatile(Object, long)} */ - @IntrinsicCandidate - public final double getDoubleAcquire(Object o, long offset) { - return getDoubleVolatile(o, offset); - } - - /* - * Versions of {@link #putReferenceVolatile(Object, long, Object)} - * that do not guarantee immediate visibility of the store to - * other threads. This method is generally only useful if the - * underlying field is a Java volatile (or if an array cell, one - * that is otherwise only accessed using volatile accesses). - * - * Corresponds to C11 atomic_store_explicit(..., memory_order_release). - */ - - /** Release version of {@link #putReferenceVolatile(Object, long, Object)} */ - @IntrinsicCandidate - public final void putReferenceRelease(Object o, long offset, Object x) { - putReferenceVolatile(o, offset, x); - } - - /** Release version of {@link #putBooleanVolatile(Object, long, boolean)} */ - @IntrinsicCandidate - public final void putBooleanRelease(Object o, long offset, boolean x) { - putBooleanVolatile(o, offset, x); - } - - /** Release version of {@link #putByteVolatile(Object, long, byte)} */ - @IntrinsicCandidate - public final void putByteRelease(Object o, long offset, byte x) { - putByteVolatile(o, offset, x); - } - - /** Release version of {@link #putShortVolatile(Object, long, short)} */ - @IntrinsicCandidate - public final void putShortRelease(Object o, long offset, short x) { - putShortVolatile(o, offset, x); - } - - /** Release version of {@link #putCharVolatile(Object, long, char)} */ - @IntrinsicCandidate - public final void putCharRelease(Object o, long offset, char x) { - putCharVolatile(o, offset, x); - } - - /** Release version of {@link #putIntVolatile(Object, long, int)} */ - @IntrinsicCandidate - public final void putIntRelease(Object o, long offset, int x) { - putIntVolatile(o, offset, x); - } - - /** Release version of {@link #putFloatVolatile(Object, long, float)} */ - @IntrinsicCandidate - public final void putFloatRelease(Object o, long offset, float x) { - putFloatVolatile(o, offset, x); - } - - /** Release version of {@link #putLongVolatile(Object, long, long)} */ - @IntrinsicCandidate - public final void putLongRelease(Object o, long offset, long x) { - putLongVolatile(o, offset, x); - } - - /** Release version of {@link #putDoubleVolatile(Object, long, double)} */ - @IntrinsicCandidate - public final void putDoubleRelease(Object o, long offset, double x) { - putDoubleVolatile(o, offset, x); - } - - // ------------------------------ Opaque -------------------------------------- - - /** Opaque version of {@link #getReferenceVolatile(Object, long)} */ - @IntrinsicCandidate - public final Object getReferenceOpaque(Object o, long offset) { - return getReferenceVolatile(o, offset); + @ForceInline + public void putReferenceVolatile(Object o, long offset, Object x) { + putReferenceMO(MO_VOLATILE, o, offset, x); } - /** Opaque version of {@link #getBooleanVolatile(Object, long)} */ - @IntrinsicCandidate - public final boolean getBooleanOpaque(Object o, long offset) { - return getBooleanVolatile(o, offset); + /** Specialization to MO_VOLATILE of {@link #getIntMO(byte, Object, long)} */ + @ForceInline + public int getIntVolatile(Object o, long offset) { + return getIntMO(MO_VOLATILE, o, offset); } - /** Opaque version of {@link #getByteVolatile(Object, long)} */ - @IntrinsicCandidate - public final byte getByteOpaque(Object o, long offset) { - return getByteVolatile(o, offset); + /** Specialization to MO_VOLATILE of {@link #putIntMO(byte, Object, long, int)} */ + @ForceInline + public void putIntVolatile(Object o, long offset, int x) { + putIntMO(MO_VOLATILE, o, offset, x); } - /** Opaque version of {@link #getShortVolatile(Object, long)} */ - @IntrinsicCandidate - public final short getShortOpaque(Object o, long offset) { - return getShortVolatile(o, offset); + /** Specialization to MO_VOLATILE of {@link #getBooleanMO(byte, Object, long)} */ + @ForceInline + public boolean getBooleanVolatile(Object o, long offset) { + return getBooleanMO(MO_VOLATILE, o, offset); } - /** Opaque version of {@link #getCharVolatile(Object, long)} */ - @IntrinsicCandidate - public final char getCharOpaque(Object o, long offset) { - return getCharVolatile(o, offset); + /** Specialization to MO_VOLATILE of {@link #putBooleanMO(byte, Object, long, boolean)} */ + @ForceInline + public void putBooleanVolatile(Object o, long offset, boolean x) { + putBooleanMO(MO_VOLATILE, o, offset, x); } - /** Opaque version of {@link #getIntVolatile(Object, long)} */ - @IntrinsicCandidate - public final int getIntOpaque(Object o, long offset) { - return getIntVolatile(o, offset); + /** Specialization to MO_VOLATILE of {@link #getByteMO(byte, Object, long)} */ + @ForceInline + public byte getByteVolatile(Object o, long offset) { + return getByteMO(MO_VOLATILE, o, offset); } - /** Opaque version of {@link #getFloatVolatile(Object, long)} */ - @IntrinsicCandidate - public final float getFloatOpaque(Object o, long offset) { - return getFloatVolatile(o, offset); + /** Specialization to MO_VOLATILE of {@link #putByteMO(byte, Object, long, byte)} */ + @ForceInline + public void putByteVolatile(Object o, long offset, byte x) { + putByteMO(MO_VOLATILE, o, offset, x); } - /** Opaque version of {@link #getLongVolatile(Object, long)} */ - @IntrinsicCandidate - public final long getLongOpaque(Object o, long offset) { - return getLongVolatile(o, offset); + /** Specialization to MO_VOLATILE of {@link #getShortMO(byte, Object, long)} */ + @ForceInline + public short getShortVolatile(Object o, long offset) { + return getShortMO(MO_VOLATILE, o, offset); } - /** Opaque version of {@link #getDoubleVolatile(Object, long)} */ - @IntrinsicCandidate - public final double getDoubleOpaque(Object o, long offset) { - return getDoubleVolatile(o, offset); + /** Specialization to MO_VOLATILE of {@link #putShortMO(byte, Object, long, short)} */ + @ForceInline + public void putShortVolatile(Object o, long offset, short x) { + putShortMO(MO_VOLATILE, o, offset, x); } - /** Opaque version of {@link #putReferenceVolatile(Object, long, Object)} */ - @IntrinsicCandidate - public final void putReferenceOpaque(Object o, long offset, Object x) { - putReferenceVolatile(o, offset, x); + /** Specialization to MO_VOLATILE of {@link #getCharMO(byte, Object, long)} */ + @ForceInline + public char getCharVolatile(Object o, long offset) { + return getCharMO(MO_VOLATILE, o, offset); } - /** Opaque version of {@link #putBooleanVolatile(Object, long, boolean)} */ - @IntrinsicCandidate - public final void putBooleanOpaque(Object o, long offset, boolean x) { - putBooleanVolatile(o, offset, x); + /** Specialization to MO_VOLATILE of {@link #putCharMO(byte, Object, long, char)} */ + @ForceInline + public void putCharVolatile(Object o, long offset, char x) { + putCharMO(MO_VOLATILE, o, offset, x); } - /** Opaque version of {@link #putByteVolatile(Object, long, byte)} */ - @IntrinsicCandidate - public final void putByteOpaque(Object o, long offset, byte x) { - putByteVolatile(o, offset, x); + /** Specialization to MO_VOLATILE of {@link #getLongMO(byte, Object, long)} */ + @ForceInline + public long getLongVolatile(Object o, long offset) { + return getLongMO(MO_VOLATILE, o, offset); } - /** Opaque version of {@link #putShortVolatile(Object, long, short)} */ - @IntrinsicCandidate - public final void putShortOpaque(Object o, long offset, short x) { - putShortVolatile(o, offset, x); + /** Specialization to MO_VOLATILE of {@link #putLongMO(byte, Object, long, long)} */ + @ForceInline + public void putLongVolatile(Object o, long offset, long x) { + putLongMO(MO_VOLATILE, o, offset, x); } - /** Opaque version of {@link #putCharVolatile(Object, long, char)} */ - @IntrinsicCandidate - public final void putCharOpaque(Object o, long offset, char x) { - putCharVolatile(o, offset, x); + /** Specialization to MO_VOLATILE of {@link #getFloatMO(byte, Object, long)} */ + @ForceInline + public float getFloatVolatile(Object o, long offset) { + return getFloatMO(MO_VOLATILE, o, offset); } - /** Opaque version of {@link #putIntVolatile(Object, long, int)} */ - @IntrinsicCandidate - public final void putIntOpaque(Object o, long offset, int x) { - putIntVolatile(o, offset, x); + /** Specialization to MO_VOLATILE of {@link #putFloatMO(byte, Object, long, float)} */ + @ForceInline + public void putFloatVolatile(Object o, long offset, float x) { + putFloatMO(MO_VOLATILE, o, offset, x); } - /** Opaque version of {@link #putFloatVolatile(Object, long, float)} */ - @IntrinsicCandidate - public final void putFloatOpaque(Object o, long offset, float x) { - putFloatVolatile(o, offset, x); + /** Specialization to MO_VOLATILE of {@link #getDoubleMO(byte, Object, long)} */ + @ForceInline + public double getDoubleVolatile(Object o, long offset) { + return getDoubleMO(MO_VOLATILE, o, offset); } - /** Opaque version of {@link #putLongVolatile(Object, long, long)} */ - @IntrinsicCandidate - public final void putLongOpaque(Object o, long offset, long x) { - putLongVolatile(o, offset, x); + /** Specialization to MO_VOLATILE of {@link #putDoubleMO(byte, Object, long, double)} */ + @ForceInline + public void putDoubleVolatile(Object o, long offset, double x) { + putDoubleMO(MO_VOLATILE, o, offset, x); } - /** Opaque version of {@link #putDoubleVolatile(Object, long, double)} */ - @IntrinsicCandidate - public final void putDoubleOpaque(Object o, long offset, double x) { - putDoubleVolatile(o, offset, x); - } + /* + * Note on stores: + * Variations of {@link #putReferenceMO(MO_RELEASE, Object, long, Object)} + * do not guarantee immediate visibility of the store to + * other threads. This method is generally only useful if the + * underlying field is a Java volatile (or if an array cell, one + * that is otherwise only accessed using volatile accesses). + * + * Corresponds to C11 atomic_store_explicit(..., memory_order_release). + */ /** * Unblocks the given thread blocked on {@code park}, or, if it is @@ -2469,8 +2880,66 @@ public int getLoadAverage(double[] loadavg, int nelems) { return getLoadAverage0(loadavg, nelems); } - // The following contain CAS-based Java implementations used on - // platforms not supporting native instructions + /** + * Intrinsic for performing get-and-operate on a primitive variable + * of any primitive type. See {@link #putPrimitiveBitsMO} for the + * relevant conventions. Atomically update the variable using the + * given operation and operand, as if by {@code v = v op operand}. + * Return the previous value of the variable. + *

+ * The operator must be a character in {@code "+&|^="}, selecting + * get and one of add, bitwise and, bitwise or, bitwise xor, or + * (simply) set. + *

+ * This intrinsic has a default implementation, using a weak CAS + * loop, for platforms which do not support the relevant native + * instructions. The VM is free to use the default + * implementation, even if the instructions exist. + * @return the value previously stored in the indicated Java variable, + * padded by garbage high-order bits (if smaller than 64 bits) + */ + @ForceInline + @IntrinsicCandidate + private long getAndOperatePrimitiveBitsMO(byte memoryOrder, + byte basicType, + byte op, + Object o, long offset, + long operand) { + checkMemoryOrder(memoryOrder); + checkOperatorForCAS(op, basicType); + byte memoryOrderForLoad = memoryOrder; + memoryOrderForLoad &= ~MO_RELEASE; // do not release when loading + if (memoryOrderForLoad == 0) memoryOrderForLoad = MO_PLAIN; + byte memoryOrderForCAS = memoryOrder; + memoryOrderForCAS |= MO_WEAK_CAS; // use a weak-CAS loop, if it helps + long v; + long nextv; + // This loop used to be hand-copied 5x (once per OP) and also 4x (once per MO). + // This single-instance form is equally effective, because it folds up in the JIT. + do { + v = getPrimitiveBitsMO(memoryOrderForLoad, basicType, o, offset); + nextv = switch (op) { + // All of these are "T-functions", compatible with long evaluation. + // Float add, or signed min/max, would not be T-functions. + case OP_ADD -> v + operand; + case OP_BITOR -> v | operand; + case OP_BITAND -> v & operand; + case OP_BITXOR -> v ^ operand; + default -> operand; // reached as case OP_SWAP + }; + } while (!compareAndSetPrimitiveBitsMO(memoryOrderForCAS, basicType, o, offset, v, nextv)); + return v; + } + + @ForceInline + private static void checkOperatorForCAS(byte op, byte basicType) { + if ((op == OP_ADD | op == OP_BITOR | op == OP_BITAND | op == OP_BITXOR | op == OP_SWAP) & + (basicType >= BT_BYTE & basicType <= BT_LONG)) + return; // OK arguments + // No direct add or other bitwise ops for boolean, char, float, double. + // Those cases are handled carefully by other means. + throw new IllegalArgumentException("bad op or type"); + } /** * Atomically adds the given value to the current value of a field @@ -2483,31 +2952,14 @@ public int getLoadAverage(double[] loadavg, int nelems) { * @return the previous value * @since 1.8 */ - @IntrinsicCandidate - public final int getAndAddInt(Object o, long offset, int delta) { - int v; - do { - v = getIntVolatile(o, offset); - } while (!weakCompareAndSetInt(o, offset, v, v + delta)); - return v; - } - @ForceInline - public final int getAndAddIntRelease(Object o, long offset, int delta) { - int v; - do { - v = getInt(o, offset); - } while (!weakCompareAndSetIntRelease(o, offset, v, v + delta)); - return v; + public int getAndAddInt(Object o, long offset, int delta) { + return (int) getAndOperatePrimitiveBitsMO(MO_VOLATILE, BT_INT, OP_ADD, o, offset, delta); } @ForceInline - public final int getAndAddIntAcquire(Object o, long offset, int delta) { - int v; - do { - v = getIntAcquire(o, offset); - } while (!weakCompareAndSetIntAcquire(o, offset, v, v + delta)); - return v; + public int getAndAddIntMO(byte memoryOrder, Object o, long offset, int delta) { + return (int) getAndOperatePrimitiveBitsMO(memoryOrder, BT_INT, OP_ADD, o, offset, delta); } /** @@ -2521,189 +2973,97 @@ public final int getAndAddIntAcquire(Object o, long offset, int delta) { * @return the previous value * @since 1.8 */ - @IntrinsicCandidate - public final long getAndAddLong(Object o, long offset, long delta) { - long v; - do { - v = getLongVolatile(o, offset); - } while (!weakCompareAndSetLong(o, offset, v, v + delta)); - return v; - } - - @ForceInline - public final long getAndAddLongRelease(Object o, long offset, long delta) { - long v; - do { - v = getLong(o, offset); - } while (!weakCompareAndSetLongRelease(o, offset, v, v + delta)); - return v; - } - @ForceInline - public final long getAndAddLongAcquire(Object o, long offset, long delta) { - long v; - do { - v = getLongAcquire(o, offset); - } while (!weakCompareAndSetLongAcquire(o, offset, v, v + delta)); - return v; - } - - @IntrinsicCandidate - public final byte getAndAddByte(Object o, long offset, byte delta) { - byte v; - do { - v = getByteVolatile(o, offset); - } while (!weakCompareAndSetByte(o, offset, v, (byte) (v + delta))); - return v; - } - - @ForceInline - public final byte getAndAddByteRelease(Object o, long offset, byte delta) { - byte v; - do { - v = getByte(o, offset); - } while (!weakCompareAndSetByteRelease(o, offset, v, (byte) (v + delta))); - return v; + public long getAndAddLong(Object o, long offset, long delta) { + return getAndOperatePrimitiveBitsMO(MO_VOLATILE, BT_LONG, OP_ADD, o, offset, delta); } @ForceInline - public final byte getAndAddByteAcquire(Object o, long offset, byte delta) { - byte v; - do { - v = getByteAcquire(o, offset); - } while (!weakCompareAndSetByteAcquire(o, offset, v, (byte) (v + delta))); - return v; - } - - @IntrinsicCandidate - public final short getAndAddShort(Object o, long offset, short delta) { - short v; - do { - v = getShortVolatile(o, offset); - } while (!weakCompareAndSetShort(o, offset, v, (short) (v + delta))); - return v; + public long getAndAddLongMO(byte memoryOrder, Object o, long offset, long delta) { + return getAndOperatePrimitiveBitsMO(memoryOrder, BT_LONG, OP_ADD, o, offset, delta); } @ForceInline - public final short getAndAddShortRelease(Object o, long offset, short delta) { - short v; - do { - v = getShort(o, offset); - } while (!weakCompareAndSetShortRelease(o, offset, v, (short) (v + delta))); - return v; + public byte getAndAddByte(Object o, long offset, byte delta) { + return (byte) getAndOperatePrimitiveBitsMO(MO_VOLATILE, BT_BYTE, OP_ADD, o, offset, delta); } @ForceInline - public final short getAndAddShortAcquire(Object o, long offset, short delta) { - short v; - do { - v = getShortAcquire(o, offset); - } while (!weakCompareAndSetShortAcquire(o, offset, v, (short) (v + delta))); - return v; + public byte getAndAddByteMO(byte memoryOrder, Object o, long offset, byte delta) { + return (byte) getAndOperatePrimitiveBitsMO(memoryOrder, BT_BYTE, OP_ADD, o, offset, delta); } @ForceInline - public final char getAndAddChar(Object o, long offset, char delta) { - return (char) getAndAddShort(o, offset, (short) delta); + public short getAndAddShort(Object o, long offset, short delta) { + return (short) getAndOperatePrimitiveBitsMO(MO_VOLATILE, BT_SHORT, OP_ADD, o, offset, delta); } @ForceInline - public final char getAndAddCharRelease(Object o, long offset, char delta) { - return (char) getAndAddShortRelease(o, offset, (short) delta); + public short getAndAddShortMO(byte memoryOrder, Object o, long offset, short delta) { + return (short) getAndOperatePrimitiveBitsMO(memoryOrder, BT_SHORT, OP_ADD, o, offset, delta); } @ForceInline - public final char getAndAddCharAcquire(Object o, long offset, char delta) { - return (char) getAndAddShortAcquire(o, offset, (short) delta); + public char getAndAddChar(Object o, long offset, char delta) { + return (char) getAndAddShort(o, offset, (short) delta); } @ForceInline - public final float getAndAddFloat(Object o, long offset, float delta) { - int expectedBits; - float v; - do { - // Load and CAS with the raw bits to avoid issues with NaNs and - // possible bit conversion from signaling NaNs to quiet NaNs that - // may result in the loop not terminating. - expectedBits = getIntVolatile(o, offset); - v = Float.intBitsToFloat(expectedBits); - } while (!weakCompareAndSetInt(o, offset, - expectedBits, Float.floatToRawIntBits(v + delta))); - return v; + public char getAndAddCharMO(byte memoryOrder, Object o, long offset, char delta) { + return (char) getAndAddShortMO(memoryOrder, o, offset, (short) delta); } @ForceInline - public final float getAndAddFloatRelease(Object o, long offset, float delta) { - int expectedBits; - float v; - do { - // Load and CAS with the raw bits to avoid issues with NaNs and - // possible bit conversion from signaling NaNs to quiet NaNs that - // may result in the loop not terminating. - expectedBits = getInt(o, offset); - v = Float.intBitsToFloat(expectedBits); - } while (!weakCompareAndSetIntRelease(o, offset, - expectedBits, Float.floatToRawIntBits(v + delta))); - return v; + public float getAndAddFloat(Object o, long offset, float delta) { + return getAndAddFloatMO(MO_VOLATILE, o, offset, delta); } + /** For completeness. It is pretty ugly. */ @ForceInline - public final float getAndAddFloatAcquire(Object o, long offset, float delta) { + public float getAndAddFloatMO(byte memoryOrder, Object o, long offset, float delta) { + checkMemoryOrder(memoryOrder); + byte memoryOrderForLoad = memoryOrder; + memoryOrderForLoad &= ~MO_RELEASE; // do not release when loading + if (memoryOrderForLoad == 0) memoryOrderForLoad = MO_PLAIN; + byte memoryOrderForCAS = memoryOrder; + memoryOrderForCAS |= MO_WEAK_CAS; int expectedBits; float v; do { // Load and CAS with the raw bits to avoid issues with NaNs and // possible bit conversion from signaling NaNs to quiet NaNs that // may result in the loop not terminating. - expectedBits = getIntAcquire(o, offset); + expectedBits = getIntMO(memoryOrderForLoad, o, offset); v = Float.intBitsToFloat(expectedBits); - } while (!weakCompareAndSetIntAcquire(o, offset, - expectedBits, Float.floatToRawIntBits(v + delta))); - return v; - } - - @ForceInline - public final double getAndAddDouble(Object o, long offset, double delta) { - long expectedBits; - double v; - do { - // Load and CAS with the raw bits to avoid issues with NaNs and - // possible bit conversion from signaling NaNs to quiet NaNs that - // may result in the loop not terminating. - expectedBits = getLongVolatile(o, offset); - v = Double.longBitsToDouble(expectedBits); - } while (!weakCompareAndSetLong(o, offset, - expectedBits, Double.doubleToRawLongBits(v + delta))); + } while (!compareAndSetIntMO(memoryOrderForCAS, o, offset, + expectedBits, Float.floatToRawIntBits(v + delta))); return v; } @ForceInline - public final double getAndAddDoubleRelease(Object o, long offset, double delta) { - long expectedBits; - double v; - do { - // Load and CAS with the raw bits to avoid issues with NaNs and - // possible bit conversion from signaling NaNs to quiet NaNs that - // may result in the loop not terminating. - expectedBits = getLong(o, offset); - v = Double.longBitsToDouble(expectedBits); - } while (!weakCompareAndSetLongRelease(o, offset, - expectedBits, Double.doubleToRawLongBits(v + delta))); - return v; + public double getAndAddDouble(Object o, long offset, double delta) { + return getAndAddDoubleMO(MO_VOLATILE, o, offset, delta); } + /** For completeness. It is pretty ugly. */ @ForceInline - public final double getAndAddDoubleAcquire(Object o, long offset, double delta) { + public double getAndAddDoubleMO(byte memoryOrder, Object o, long offset, double delta) { + checkMemoryOrder(memoryOrder); + byte memoryOrderForLoad = memoryOrder; + memoryOrderForLoad &= ~MO_RELEASE; // do not release when loading + if (memoryOrderForLoad == 0) memoryOrderForLoad = MO_PLAIN; + byte memoryOrderForCAS = memoryOrder; + memoryOrderForCAS |= MO_WEAK_CAS; long expectedBits; double v; do { // Load and CAS with the raw bits to avoid issues with NaNs and // possible bit conversion from signaling NaNs to quiet NaNs that // may result in the loop not terminating. - expectedBits = getLongAcquire(o, offset); + expectedBits = getLongMO(memoryOrderForLoad, o, offset); v = Double.longBitsToDouble(expectedBits); - } while (!weakCompareAndSetLongAcquire(o, offset, - expectedBits, Double.doubleToRawLongBits(v + delta))); + } while (!compareAndSetLongMO(memoryOrderForCAS, o, offset, + expectedBits, Double.doubleToRawLongBits(v + delta))); return v; } @@ -2718,31 +3078,14 @@ public final double getAndAddDoubleAcquire(Object o, long offset, double delta) * @return the previous value * @since 1.8 */ - @IntrinsicCandidate - public final int getAndSetInt(Object o, long offset, int newValue) { - int v; - do { - v = getIntVolatile(o, offset); - } while (!weakCompareAndSetInt(o, offset, v, newValue)); - return v; - } - @ForceInline - public final int getAndSetIntRelease(Object o, long offset, int newValue) { - int v; - do { - v = getInt(o, offset); - } while (!weakCompareAndSetIntRelease(o, offset, v, newValue)); - return v; + public int getAndSetInt(Object o, long offset, int newValue) { + return (int) getAndOperatePrimitiveBitsMO(MO_VOLATILE, BT_INT, OP_SWAP, o, offset, newValue); } @ForceInline - public final int getAndSetIntAcquire(Object o, long offset, int newValue) { - int v; - do { - v = getIntAcquire(o, offset); - } while (!weakCompareAndSetIntAcquire(o, offset, v, newValue)); - return v; + public int getAndSetIntMO(byte memoryOrder, Object o, long offset, int newValue) { + return (int) getAndOperatePrimitiveBitsMO(memoryOrder, BT_INT, OP_SWAP, o, offset, newValue); } /** @@ -2756,31 +3099,14 @@ public final int getAndSetIntAcquire(Object o, long offset, int newValue) { * @return the previous value * @since 1.8 */ - @IntrinsicCandidate - public final long getAndSetLong(Object o, long offset, long newValue) { - long v; - do { - v = getLongVolatile(o, offset); - } while (!weakCompareAndSetLong(o, offset, v, newValue)); - return v; - } - @ForceInline - public final long getAndSetLongRelease(Object o, long offset, long newValue) { - long v; - do { - v = getLong(o, offset); - } while (!weakCompareAndSetLongRelease(o, offset, v, newValue)); - return v; + public long getAndSetLong(Object o, long offset, long newValue) { + return getAndOperatePrimitiveBitsMO(MO_VOLATILE, BT_LONG, OP_SWAP, o, offset, newValue); } @ForceInline - public final long getAndSetLongAcquire(Object o, long offset, long newValue) { - long v; - do { - v = getLongAcquire(o, offset); - } while (!weakCompareAndSetLongAcquire(o, offset, v, newValue)); - return v; + public long getAndSetLongMO(byte memoryOrder, Object o, long offset, long newValue) { + return getAndOperatePrimitiveBitsMO(memoryOrder, BT_LONG, OP_SWAP, o, offset, newValue); } /** @@ -2794,466 +3120,222 @@ public final long getAndSetLongAcquire(Object o, long offset, long newValue) { * @return the previous value * @since 1.8 */ - @IntrinsicCandidate - public final Object getAndSetReference(Object o, long offset, Object newValue) { - Object v; - do { - v = getReferenceVolatile(o, offset); - } while (!weakCompareAndSetReference(o, offset, v, newValue)); - return v; - } - @ForceInline - public final Object getAndSetReferenceRelease(Object o, long offset, Object newValue) { - Object v; - do { - v = getReference(o, offset); - } while (!weakCompareAndSetReferenceRelease(o, offset, v, newValue)); - return v; + public Object getAndSetReference(Object o, long offset, Object newValue) { + return getAndSetReferenceMO(MO_VOLATILE, o, offset, newValue); } @ForceInline - public final Object getAndSetReferenceAcquire(Object o, long offset, Object newValue) { - Object v; - do { - v = getReferenceAcquire(o, offset); - } while (!weakCompareAndSetReferenceAcquire(o, offset, v, newValue)); - return v; - } - @IntrinsicCandidate - public final byte getAndSetByte(Object o, long offset, byte newValue) { - byte v; + public Object getAndSetReferenceMO(byte memoryOrder, Object o, long offset, Object newValue) { + checkMemoryOrder(memoryOrder); + byte memoryOrderForLoad = memoryOrder; + memoryOrderForLoad &= ~MO_RELEASE; // do not release when loading + if (memoryOrderForLoad == 0) memoryOrderForLoad = MO_PLAIN; + byte memoryOrderForCAS = memoryOrder; + memoryOrderForCAS |= MO_WEAK_CAS; + Object v; do { - v = getByteVolatile(o, offset); - } while (!weakCompareAndSetByte(o, offset, v, newValue)); + v = getReferenceMO(memoryOrderForLoad, o, offset); + } while (!compareAndSetReferenceMO(memoryOrderForCAS, o, offset, v, newValue)); return v; } @ForceInline - public final byte getAndSetByteRelease(Object o, long offset, byte newValue) { - byte v; - do { - v = getByte(o, offset); - } while (!weakCompareAndSetByteRelease(o, offset, v, newValue)); - return v; + public byte getAndSetByte(Object o, long offset, byte newValue) { + return (byte) getAndOperatePrimitiveBitsMO(MO_VOLATILE, BT_BYTE, OP_SWAP, o, offset, newValue); } @ForceInline - public final byte getAndSetByteAcquire(Object o, long offset, byte newValue) { - byte v; - do { - v = getByteAcquire(o, offset); - } while (!weakCompareAndSetByteAcquire(o, offset, v, newValue)); - return v; + public byte getAndSetByteMO(byte memoryOrder, Object o, long offset, byte newValue) { + return (byte) getAndOperatePrimitiveBitsMO(memoryOrder, BT_BYTE, OP_SWAP, o, offset, newValue); } @ForceInline - public final boolean getAndSetBoolean(Object o, long offset, boolean newValue) { + public boolean getAndSetBoolean(Object o, long offset, boolean newValue) { return byte2bool(getAndSetByte(o, offset, bool2byte(newValue))); } @ForceInline - public final boolean getAndSetBooleanRelease(Object o, long offset, boolean newValue) { - return byte2bool(getAndSetByteRelease(o, offset, bool2byte(newValue))); - } - - @ForceInline - public final boolean getAndSetBooleanAcquire(Object o, long offset, boolean newValue) { - return byte2bool(getAndSetByteAcquire(o, offset, bool2byte(newValue))); - } - - @IntrinsicCandidate - public final short getAndSetShort(Object o, long offset, short newValue) { - short v; - do { - v = getShortVolatile(o, offset); - } while (!weakCompareAndSetShort(o, offset, v, newValue)); - return v; + public boolean getAndSetBooleanMO(byte memoryOrder, Object o, long offset, boolean newValue) { + return byte2bool(getAndSetByteMO(memoryOrder, o, offset, bool2byte(newValue))); } @ForceInline - public final short getAndSetShortRelease(Object o, long offset, short newValue) { - short v; - do { - v = getShort(o, offset); - } while (!weakCompareAndSetShortRelease(o, offset, v, newValue)); - return v; + public short getAndSetShort(Object o, long offset, short newValue) { + return (short) getAndOperatePrimitiveBitsMO(MO_VOLATILE, BT_SHORT, OP_SWAP, o, offset, newValue); } @ForceInline - public final short getAndSetShortAcquire(Object o, long offset, short newValue) { - short v; - do { - v = getShortAcquire(o, offset); - } while (!weakCompareAndSetShortAcquire(o, offset, v, newValue)); - return v; + public short getAndSetShortMO(byte memoryOrder, Object o, long offset, short newValue) { + return (short) getAndOperatePrimitiveBitsMO(memoryOrder, BT_SHORT, OP_SWAP, o, offset, newValue); } @ForceInline - public final char getAndSetChar(Object o, long offset, char newValue) { + public char getAndSetChar(Object o, long offset, char newValue) { return s2c(getAndSetShort(o, offset, c2s(newValue))); } @ForceInline - public final char getAndSetCharRelease(Object o, long offset, char newValue) { - return s2c(getAndSetShortRelease(o, offset, c2s(newValue))); - } - - @ForceInline - public final char getAndSetCharAcquire(Object o, long offset, char newValue) { - return s2c(getAndSetShortAcquire(o, offset, c2s(newValue))); + public char getAndSetCharMO(byte memoryOrder, Object o, long offset, char newValue) { + return s2c(getAndSetShortMO(memoryOrder, o, offset, c2s(newValue))); } @ForceInline - public final float getAndSetFloat(Object o, long offset, float newValue) { + public float getAndSetFloat(Object o, long offset, float newValue) { int v = getAndSetInt(o, offset, Float.floatToRawIntBits(newValue)); return Float.intBitsToFloat(v); } @ForceInline - public final float getAndSetFloatRelease(Object o, long offset, float newValue) { - int v = getAndSetIntRelease(o, offset, Float.floatToRawIntBits(newValue)); - return Float.intBitsToFloat(v); - } - - @ForceInline - public final float getAndSetFloatAcquire(Object o, long offset, float newValue) { - int v = getAndSetIntAcquire(o, offset, Float.floatToRawIntBits(newValue)); + public float getAndSetFloatMO(byte memoryOrder, Object o, long offset, float newValue) { + int v = getAndSetIntMO(memoryOrder, o, offset, Float.floatToRawIntBits(newValue)); return Float.intBitsToFloat(v); } @ForceInline - public final double getAndSetDouble(Object o, long offset, double newValue) { + public double getAndSetDouble(Object o, long offset, double newValue) { long v = getAndSetLong(o, offset, Double.doubleToRawLongBits(newValue)); return Double.longBitsToDouble(v); - } - - @ForceInline - public final double getAndSetDoubleRelease(Object o, long offset, double newValue) { - long v = getAndSetLongRelease(o, offset, Double.doubleToRawLongBits(newValue)); - return Double.longBitsToDouble(v); - } + } @ForceInline - public final double getAndSetDoubleAcquire(Object o, long offset, double newValue) { - long v = getAndSetLongAcquire(o, offset, Double.doubleToRawLongBits(newValue)); + public double getAndSetDoubleMO(byte memoryOrder, Object o, long offset, double newValue) { + long v = getAndSetLongMO(memoryOrder, o, offset, Double.doubleToRawLongBits(newValue)); return Double.longBitsToDouble(v); } - // The following contain CAS-based Java implementations used on // platforms not supporting native instructions @ForceInline - public final boolean getAndBitwiseOrBoolean(Object o, long offset, boolean mask) { + public boolean getAndBitwiseOrBoolean(Object o, long offset, boolean mask) { return byte2bool(getAndBitwiseOrByte(o, offset, bool2byte(mask))); } @ForceInline - public final boolean getAndBitwiseOrBooleanRelease(Object o, long offset, boolean mask) { - return byte2bool(getAndBitwiseOrByteRelease(o, offset, bool2byte(mask))); - } - - @ForceInline - public final boolean getAndBitwiseOrBooleanAcquire(Object o, long offset, boolean mask) { - return byte2bool(getAndBitwiseOrByteAcquire(o, offset, bool2byte(mask))); + public boolean getAndBitwiseOrBooleanMO(byte memoryOrder, Object o, long offset, boolean mask) { + return byte2bool(getAndBitwiseOrByteMO(memoryOrder, o, offset, bool2byte(mask))); } @ForceInline - public final boolean getAndBitwiseAndBoolean(Object o, long offset, boolean mask) { + public boolean getAndBitwiseAndBoolean(Object o, long offset, boolean mask) { return byte2bool(getAndBitwiseAndByte(o, offset, bool2byte(mask))); } @ForceInline - public final boolean getAndBitwiseAndBooleanRelease(Object o, long offset, boolean mask) { - return byte2bool(getAndBitwiseAndByteRelease(o, offset, bool2byte(mask))); - } - - @ForceInline - public final boolean getAndBitwiseAndBooleanAcquire(Object o, long offset, boolean mask) { - return byte2bool(getAndBitwiseAndByteAcquire(o, offset, bool2byte(mask))); + public boolean getAndBitwiseAndBooleanMO(byte memoryOrder, Object o, long offset, boolean mask) { + return byte2bool(getAndBitwiseAndByteMO(memoryOrder, o, offset, bool2byte(mask))); } @ForceInline - public final boolean getAndBitwiseXorBoolean(Object o, long offset, boolean mask) { + public boolean getAndBitwiseXorBoolean(Object o, long offset, boolean mask) { return byte2bool(getAndBitwiseXorByte(o, offset, bool2byte(mask))); } @ForceInline - public final boolean getAndBitwiseXorBooleanRelease(Object o, long offset, boolean mask) { - return byte2bool(getAndBitwiseXorByteRelease(o, offset, bool2byte(mask))); - } - - @ForceInline - public final boolean getAndBitwiseXorBooleanAcquire(Object o, long offset, boolean mask) { - return byte2bool(getAndBitwiseXorByteAcquire(o, offset, bool2byte(mask))); - } - - - @ForceInline - public final byte getAndBitwiseOrByte(Object o, long offset, byte mask) { - byte current; - do { - current = getByteVolatile(o, offset); - } while (!weakCompareAndSetByte(o, offset, - current, (byte) (current | mask))); - return current; - } - - @ForceInline - public final byte getAndBitwiseOrByteRelease(Object o, long offset, byte mask) { - byte current; - do { - current = getByte(o, offset); - } while (!weakCompareAndSetByteRelease(o, offset, - current, (byte) (current | mask))); - return current; - } - - @ForceInline - public final byte getAndBitwiseOrByteAcquire(Object o, long offset, byte mask) { - byte current; - do { - // Plain read, the value is a hint, the acquire CAS does the work - current = getByte(o, offset); - } while (!weakCompareAndSetByteAcquire(o, offset, - current, (byte) (current | mask))); - return current; + public boolean getAndBitwiseXorBooleanMO(byte memoryOrder, Object o, long offset, boolean mask) { + return byte2bool(getAndBitwiseXorByteMO(memoryOrder, o, offset, bool2byte(mask))); } @ForceInline - public final byte getAndBitwiseAndByte(Object o, long offset, byte mask) { - byte current; - do { - current = getByteVolatile(o, offset); - } while (!weakCompareAndSetByte(o, offset, - current, (byte) (current & mask))); - return current; + public byte getAndBitwiseOrByte(Object o, long offset, byte mask) { + return getAndBitwiseOrByteMO(MO_VOLATILE, o, offset, mask); } @ForceInline - public final byte getAndBitwiseAndByteRelease(Object o, long offset, byte mask) { - byte current; - do { - current = getByte(o, offset); - } while (!weakCompareAndSetByteRelease(o, offset, - current, (byte) (current & mask))); - return current; + public byte getAndBitwiseOrByteMO(byte memoryOrder, Object o, long offset, byte mask) { + return (byte) getAndOperatePrimitiveBitsMO(memoryOrder, BT_BYTE, OP_BITOR, o, offset, mask); } @ForceInline - public final byte getAndBitwiseAndByteAcquire(Object o, long offset, byte mask) { - byte current; - do { - // Plain read, the value is a hint, the acquire CAS does the work - current = getByte(o, offset); - } while (!weakCompareAndSetByteAcquire(o, offset, - current, (byte) (current & mask))); - return current; + public byte getAndBitwiseAndByte(Object o, long offset, byte mask) { + return getAndBitwiseAndByteMO(MO_VOLATILE, o, offset, mask); } @ForceInline - public final byte getAndBitwiseXorByte(Object o, long offset, byte mask) { - byte current; - do { - current = getByteVolatile(o, offset); - } while (!weakCompareAndSetByte(o, offset, - current, (byte) (current ^ mask))); - return current; + public byte getAndBitwiseAndByteMO(byte memoryOrder, Object o, long offset, byte mask) { + return (byte) getAndOperatePrimitiveBitsMO(memoryOrder, BT_BYTE, OP_BITAND, o, offset, mask); } @ForceInline - public final byte getAndBitwiseXorByteRelease(Object o, long offset, byte mask) { - byte current; - do { - current = getByte(o, offset); - } while (!weakCompareAndSetByteRelease(o, offset, - current, (byte) (current ^ mask))); - return current; + public byte getAndBitwiseXorByte(Object o, long offset, byte mask) { + return getAndBitwiseXorByteMO(MO_VOLATILE, o, offset, mask); } @ForceInline - public final byte getAndBitwiseXorByteAcquire(Object o, long offset, byte mask) { - byte current; - do { - // Plain read, the value is a hint, the acquire CAS does the work - current = getByte(o, offset); - } while (!weakCompareAndSetByteAcquire(o, offset, - current, (byte) (current ^ mask))); - return current; + public byte getAndBitwiseXorByteMO(byte memoryOrder, Object o, long offset, byte mask) { + return (byte) getAndOperatePrimitiveBitsMO(memoryOrder, BT_BYTE, OP_BITXOR, o, offset, mask); } - @ForceInline - public final char getAndBitwiseOrChar(Object o, long offset, char mask) { + public char getAndBitwiseOrChar(Object o, long offset, char mask) { return s2c(getAndBitwiseOrShort(o, offset, c2s(mask))); } @ForceInline - public final char getAndBitwiseOrCharRelease(Object o, long offset, char mask) { - return s2c(getAndBitwiseOrShortRelease(o, offset, c2s(mask))); - } - - @ForceInline - public final char getAndBitwiseOrCharAcquire(Object o, long offset, char mask) { - return s2c(getAndBitwiseOrShortAcquire(o, offset, c2s(mask))); + public char getAndBitwiseOrCharMO(byte memoryOrder, Object o, long offset, char mask) { + return s2c(getAndBitwiseOrShortMO(memoryOrder, o, offset, c2s(mask))); } @ForceInline - public final char getAndBitwiseAndChar(Object o, long offset, char mask) { + public char getAndBitwiseAndChar(Object o, long offset, char mask) { return s2c(getAndBitwiseAndShort(o, offset, c2s(mask))); } @ForceInline - public final char getAndBitwiseAndCharRelease(Object o, long offset, char mask) { - return s2c(getAndBitwiseAndShortRelease(o, offset, c2s(mask))); - } - - @ForceInline - public final char getAndBitwiseAndCharAcquire(Object o, long offset, char mask) { - return s2c(getAndBitwiseAndShortAcquire(o, offset, c2s(mask))); + public char getAndBitwiseAndCharMO(byte memoryOrder, Object o, long offset, char mask) { + return s2c(getAndBitwiseAndShortMO(memoryOrder, o, offset, c2s(mask))); } @ForceInline - public final char getAndBitwiseXorChar(Object o, long offset, char mask) { + public char getAndBitwiseXorChar(Object o, long offset, char mask) { return s2c(getAndBitwiseXorShort(o, offset, c2s(mask))); } @ForceInline - public final char getAndBitwiseXorCharRelease(Object o, long offset, char mask) { - return s2c(getAndBitwiseXorShortRelease(o, offset, c2s(mask))); - } - - @ForceInline - public final char getAndBitwiseXorCharAcquire(Object o, long offset, char mask) { - return s2c(getAndBitwiseXorShortAcquire(o, offset, c2s(mask))); - } - - - @ForceInline - public final short getAndBitwiseOrShort(Object o, long offset, short mask) { - short current; - do { - current = getShortVolatile(o, offset); - } while (!weakCompareAndSetShort(o, offset, - current, (short) (current | mask))); - return current; - } - - @ForceInline - public final short getAndBitwiseOrShortRelease(Object o, long offset, short mask) { - short current; - do { - current = getShort(o, offset); - } while (!weakCompareAndSetShortRelease(o, offset, - current, (short) (current | mask))); - return current; - } - - @ForceInline - public final short getAndBitwiseOrShortAcquire(Object o, long offset, short mask) { - short current; - do { - // Plain read, the value is a hint, the acquire CAS does the work - current = getShort(o, offset); - } while (!weakCompareAndSetShortAcquire(o, offset, - current, (short) (current | mask))); - return current; - } - - @ForceInline - public final short getAndBitwiseAndShort(Object o, long offset, short mask) { - short current; - do { - current = getShortVolatile(o, offset); - } while (!weakCompareAndSetShort(o, offset, - current, (short) (current & mask))); - return current; + public char getAndBitwiseXorCharMO(byte memoryOrder, Object o, long offset, char mask) { + return s2c(getAndBitwiseXorShortMO(memoryOrder, o, offset, c2s(mask))); } @ForceInline - public final short getAndBitwiseAndShortRelease(Object o, long offset, short mask) { - short current; - do { - current = getShort(o, offset); - } while (!weakCompareAndSetShortRelease(o, offset, - current, (short) (current & mask))); - return current; + public short getAndBitwiseOrShort(Object o, long offset, short mask) { + return getAndBitwiseOrShortMO(MO_VOLATILE, o, offset, mask); } @ForceInline - public final short getAndBitwiseAndShortAcquire(Object o, long offset, short mask) { - short current; - do { - // Plain read, the value is a hint, the acquire CAS does the work - current = getShort(o, offset); - } while (!weakCompareAndSetShortAcquire(o, offset, - current, (short) (current & mask))); - return current; + public short getAndBitwiseOrShortMO(byte memoryOrder, Object o, long offset, short mask) { + return (short) getAndOperatePrimitiveBitsMO(memoryOrder, BT_SHORT, OP_BITOR, o, offset, mask); } @ForceInline - public final short getAndBitwiseXorShort(Object o, long offset, short mask) { - short current; - do { - current = getShortVolatile(o, offset); - } while (!weakCompareAndSetShort(o, offset, - current, (short) (current ^ mask))); - return current; + public short getAndBitwiseAndShort(Object o, long offset, short mask) { + return getAndBitwiseAndShortMO(MO_VOLATILE, o, offset, mask); } @ForceInline - public final short getAndBitwiseXorShortRelease(Object o, long offset, short mask) { - short current; - do { - current = getShort(o, offset); - } while (!weakCompareAndSetShortRelease(o, offset, - current, (short) (current ^ mask))); - return current; + public short getAndBitwiseAndShortMO(byte memoryOrder, Object o, long offset, short mask) { + return (short) getAndOperatePrimitiveBitsMO(memoryOrder, BT_SHORT, OP_BITAND, o, offset, mask); } @ForceInline - public final short getAndBitwiseXorShortAcquire(Object o, long offset, short mask) { - short current; - do { - // Plain read, the value is a hint, the acquire CAS does the work - current = getShort(o, offset); - } while (!weakCompareAndSetShortAcquire(o, offset, - current, (short) (current ^ mask))); - return current; + public short getAndBitwiseXorShort(Object o, long offset, short mask) { + return getAndBitwiseXorShortMO(MO_VOLATILE, o, offset, mask); } - @ForceInline - public final int getAndBitwiseOrInt(Object o, long offset, int mask) { - int current; - do { - current = getIntVolatile(o, offset); - } while (!weakCompareAndSetInt(o, offset, - current, current | mask)); - return current; + public short getAndBitwiseXorShortMO(byte memoryOrder, Object o, long offset, short mask) { + return (short) getAndOperatePrimitiveBitsMO(memoryOrder, BT_SHORT, OP_BITXOR, o, offset, mask); } @ForceInline - public final int getAndBitwiseOrIntRelease(Object o, long offset, int mask) { - int current; - do { - current = getInt(o, offset); - } while (!weakCompareAndSetIntRelease(o, offset, - current, current | mask)); - return current; + public int getAndBitwiseOrInt(Object o, long offset, int mask) { + return getAndBitwiseOrIntMO(MO_VOLATILE, o, offset, mask); } @ForceInline - public final int getAndBitwiseOrIntAcquire(Object o, long offset, int mask) { - int current; - do { - // Plain read, the value is a hint, the acquire CAS does the work - current = getInt(o, offset); - } while (!weakCompareAndSetIntAcquire(o, offset, - current, current | mask)); - return current; + public int getAndBitwiseOrIntMO(byte memoryOrder, Object o, long offset, int mask) { + return (int) getAndOperatePrimitiveBitsMO(memoryOrder, BT_INT, OP_BITOR, o, offset, mask); } /** @@ -3268,163 +3350,55 @@ public final int getAndBitwiseOrIntAcquire(Object o, long offset, int mask) { * @since 9 */ @ForceInline - public final int getAndBitwiseAndInt(Object o, long offset, int mask) { - int current; - do { - current = getIntVolatile(o, offset); - } while (!weakCompareAndSetInt(o, offset, - current, current & mask)); - return current; - } - - @ForceInline - public final int getAndBitwiseAndIntRelease(Object o, long offset, int mask) { - int current; - do { - current = getInt(o, offset); - } while (!weakCompareAndSetIntRelease(o, offset, - current, current & mask)); - return current; - } - - @ForceInline - public final int getAndBitwiseAndIntAcquire(Object o, long offset, int mask) { - int current; - do { - // Plain read, the value is a hint, the acquire CAS does the work - current = getInt(o, offset); - } while (!weakCompareAndSetIntAcquire(o, offset, - current, current & mask)); - return current; - } - - @ForceInline - public final int getAndBitwiseXorInt(Object o, long offset, int mask) { - int current; - do { - current = getIntVolatile(o, offset); - } while (!weakCompareAndSetInt(o, offset, - current, current ^ mask)); - return current; - } - - @ForceInline - public final int getAndBitwiseXorIntRelease(Object o, long offset, int mask) { - int current; - do { - current = getInt(o, offset); - } while (!weakCompareAndSetIntRelease(o, offset, - current, current ^ mask)); - return current; - } - - @ForceInline - public final int getAndBitwiseXorIntAcquire(Object o, long offset, int mask) { - int current; - do { - // Plain read, the value is a hint, the acquire CAS does the work - current = getInt(o, offset); - } while (!weakCompareAndSetIntAcquire(o, offset, - current, current ^ mask)); - return current; + public int getAndBitwiseAndInt(Object o, long offset, int mask) { + return getAndBitwiseAndIntMO(MO_VOLATILE, o, offset, mask); } - @ForceInline - public final long getAndBitwiseOrLong(Object o, long offset, long mask) { - long current; - do { - current = getLongVolatile(o, offset); - } while (!weakCompareAndSetLong(o, offset, - current, current | mask)); - return current; + public int getAndBitwiseAndIntMO(byte memoryOrder, Object o, long offset, int mask) { + return (int) getAndOperatePrimitiveBitsMO(memoryOrder, BT_INT, OP_BITAND, o, offset, mask); } @ForceInline - public final long getAndBitwiseOrLongRelease(Object o, long offset, long mask) { - long current; - do { - current = getLong(o, offset); - } while (!weakCompareAndSetLongRelease(o, offset, - current, current | mask)); - return current; + public int getAndBitwiseXorInt(Object o, long offset, int mask) { + return getAndBitwiseXorIntMO(MO_VOLATILE, o, offset, mask); } @ForceInline - public final long getAndBitwiseOrLongAcquire(Object o, long offset, long mask) { - long current; - do { - // Plain read, the value is a hint, the acquire CAS does the work - current = getLong(o, offset); - } while (!weakCompareAndSetLongAcquire(o, offset, - current, current | mask)); - return current; + public int getAndBitwiseXorIntMO(byte memoryOrder, Object o, long offset, int mask) { + return (int) getAndOperatePrimitiveBitsMO(memoryOrder, BT_INT, OP_BITXOR, o, offset, mask); } @ForceInline - public final long getAndBitwiseAndLong(Object o, long offset, long mask) { - long current; - do { - current = getLongVolatile(o, offset); - } while (!weakCompareAndSetLong(o, offset, - current, current & mask)); - return current; + public long getAndBitwiseOrLong(Object o, long offset, long mask) { + return getAndBitwiseOrLongMO(MO_VOLATILE, o, offset, mask); } @ForceInline - public final long getAndBitwiseAndLongRelease(Object o, long offset, long mask) { - long current; - do { - current = getLong(o, offset); - } while (!weakCompareAndSetLongRelease(o, offset, - current, current & mask)); - return current; + public long getAndBitwiseOrLongMO(byte memoryOrder, Object o, long offset, long mask) { + return getAndOperatePrimitiveBitsMO(memoryOrder, BT_LONG, OP_BITOR, o, offset, mask); } @ForceInline - public final long getAndBitwiseAndLongAcquire(Object o, long offset, long mask) { - long current; - do { - // Plain read, the value is a hint, the acquire CAS does the work - current = getLong(o, offset); - } while (!weakCompareAndSetLongAcquire(o, offset, - current, current & mask)); - return current; + public long getAndBitwiseAndLong(Object o, long offset, long mask) { + return getAndBitwiseAndLongMO(MO_VOLATILE, o, offset, mask); } @ForceInline - public final long getAndBitwiseXorLong(Object o, long offset, long mask) { - long current; - do { - current = getLongVolatile(o, offset); - } while (!weakCompareAndSetLong(o, offset, - current, current ^ mask)); - return current; + public long getAndBitwiseAndLongMO(byte memoryOrder, Object o, long offset, long mask) { + return getAndOperatePrimitiveBitsMO(memoryOrder, BT_LONG, OP_BITAND, o, offset, mask); } @ForceInline - public final long getAndBitwiseXorLongRelease(Object o, long offset, long mask) { - long current; - do { - current = getLong(o, offset); - } while (!weakCompareAndSetLongRelease(o, offset, - current, current ^ mask)); - return current; + public long getAndBitwiseXorLong(Object o, long offset, long mask) { + return getAndBitwiseXorLongMO(MO_VOLATILE, o, offset, mask); } @ForceInline - public final long getAndBitwiseXorLongAcquire(Object o, long offset, long mask) { - long current; - do { - // Plain read, the value is a hint, the acquire CAS does the work - current = getLong(o, offset); - } while (!weakCompareAndSetLongAcquire(o, offset, - current, current ^ mask)); - return current; + public long getAndBitwiseXorLongMO(byte memoryOrder, Object o, long offset, long mask) { + return getAndOperatePrimitiveBitsMO(memoryOrder, BT_LONG, OP_BITXOR, o, offset, mask); } - - /** * Ensures that loads before the fence will not be reordered with loads and * stores after the fence; a "LoadLoad plus LoadStore barrier". @@ -3437,7 +3411,7 @@ public final long getAndBitwiseXorLongAcquire(Object o, long offset, long mask) * @since 1.8 */ @IntrinsicCandidate - public final void loadFence() { + public void loadFence() { // If loadFence intrinsic is not available, fall back to full fence. fullFence(); } @@ -3454,7 +3428,7 @@ public final void loadFence() { * @since 1.8 */ @IntrinsicCandidate - public final void storeFence() { + public void storeFence() { // If storeFence intrinsic is not available, fall back to full fence. fullFence(); } @@ -3480,7 +3454,7 @@ public final void storeFence() { * * @since 9 */ - public final void loadLoadFence() { + public void loadLoadFence() { loadFence(); } @@ -3491,7 +3465,7 @@ public final void loadLoadFence() { * @since 9 */ @IntrinsicCandidate - public final void storeStoreFence() { + public void storeStoreFence() { // If storeStoreFence intrinsic is not available, fall back to storeFence. storeFence(); } @@ -3517,14 +3491,14 @@ private static void throwNoSuchMethodError() { * @return Returns true if the native byte ordering of this * platform is big-endian, false if it is little-endian. */ - public final boolean isBigEndian() { return BIG_ENDIAN; } + public boolean isBigEndian() { return BIG_ENDIAN; } /** * @return Returns true if this platform is capable of performing * accesses at addresses which are not aligned for the type of the * primitive type being accessed, false otherwise. */ - public final boolean unalignedAccess() { return UNALIGNED_ACCESS; } + public boolean unalignedAccess() { return UNALIGNED_ACCESS; } /** * Fetches a value at some byte offset into a given Java object. @@ -3559,8 +3533,15 @@ private static void throwNoSuchMethodError() { * {@link NullPointerException} * @since 9 */ - @IntrinsicCandidate - public final long getLongUnaligned(Object o, long offset) { + @ForceInline + public long getLongUnaligned(Object o, long offset) { + if (UNALIGNED_ACCESS || (offset & 7) == 0) { + return getPrimitiveBitsMONative(MO_UNALIGNED_PLAIN, BT_LONG, o, offset); + } else { + return getLongUnalignedSlow(o, offset); + } + } + private long getLongUnalignedSlow(Object o, long offset) { if ((offset & 7) == 0) { return getLong(o, offset); } else if ((offset & 3) == 0) { @@ -3593,13 +3574,21 @@ public final long getLongUnaligned(Object o, long offset) { * @return the value fetched from the indicated object * @since 9 */ - public final long getLongUnaligned(Object o, long offset, boolean bigEndian) { + @ForceInline + public long getLongUnaligned(Object o, long offset, boolean bigEndian) { return convEndian(bigEndian, getLongUnaligned(o, offset)); } /** @see #getLongUnaligned(Object, long) */ - @IntrinsicCandidate - public final int getIntUnaligned(Object o, long offset) { + @ForceInline + public int getIntUnaligned(Object o, long offset) { + if (UNALIGNED_ACCESS || (offset & 3) == 0) { + return (int) getPrimitiveBitsMONative(MO_UNALIGNED_PLAIN, BT_INT, o, offset); + } else { + return getIntUnalignedSlow(o, offset); + } + } + private int getIntUnalignedSlow(Object o, long offset) { if ((offset & 3) == 0) { return getInt(o, offset); } else if ((offset & 1) == 0) { @@ -3613,13 +3602,21 @@ public final int getIntUnaligned(Object o, long offset) { } } /** @see #getLongUnaligned(Object, long, boolean) */ - public final int getIntUnaligned(Object o, long offset, boolean bigEndian) { + @ForceInline + public int getIntUnaligned(Object o, long offset, boolean bigEndian) { return convEndian(bigEndian, getIntUnaligned(o, offset)); } /** @see #getLongUnaligned(Object, long) */ - @IntrinsicCandidate - public final short getShortUnaligned(Object o, long offset) { + @ForceInline + public short getShortUnaligned(Object o, long offset) { + if (UNALIGNED_ACCESS) { + return (short) getPrimitiveBitsMONative(MO_UNALIGNED_PLAIN, BT_SHORT, o, offset); + } else { + return getShortUnalignedSlow(o, offset); + } + } + private short getShortUnalignedSlow(Object o, long offset) { if ((offset & 1) == 0) { return getShort(o, offset); } else { @@ -3628,23 +3625,24 @@ public final short getShortUnaligned(Object o, long offset) { } } /** @see #getLongUnaligned(Object, long, boolean) */ - public final short getShortUnaligned(Object o, long offset, boolean bigEndian) { + @ForceInline + public short getShortUnaligned(Object o, long offset, boolean bigEndian) { return convEndian(bigEndian, getShortUnaligned(o, offset)); } /** @see #getLongUnaligned(Object, long) */ - @IntrinsicCandidate - public final char getCharUnaligned(Object o, long offset) { - if ((offset & 1) == 0) { - return getChar(o, offset); + @ForceInline + public char getCharUnaligned(Object o, long offset) { + if (UNALIGNED_ACCESS || (offset & 1) == 0) { + return (char) getPrimitiveBitsMONative(MO_UNALIGNED_PLAIN, BT_CHAR, o, offset); } else { - return (char)makeShort(getByte(o, offset), - getByte(o, offset + 1)); + return (char) getShortUnalignedSlow(o, offset); } } /** @see #getLongUnaligned(Object, long, boolean) */ - public final char getCharUnaligned(Object o, long offset, boolean bigEndian) { + @ForceInline + public char getCharUnaligned(Object o, long offset, boolean bigEndian) { return convEndian(bigEndian, getCharUnaligned(o, offset)); } @@ -3676,8 +3674,15 @@ public final char getCharUnaligned(Object o, long offset, boolean bigEndian) { * {@link NullPointerException} * @since 9 */ - @IntrinsicCandidate - public final void putLongUnaligned(Object o, long offset, long x) { + @ForceInline + public void putLongUnaligned(Object o, long offset, long x) { + if (UNALIGNED_ACCESS || (offset & 7) == 0) { + putPrimitiveBitsMONative(MO_UNALIGNED_PLAIN, BT_LONG, o, offset, x); + } else { + putLongUnalignedSlow(o, offset, x); + } + } + private void putLongUnalignedSlow(Object o, long offset, long x) { if ((offset & 7) == 0) { putLong(o, offset, x); } else if ((offset & 3) == 0) { @@ -3714,13 +3719,21 @@ public final void putLongUnaligned(Object o, long offset, long x) { * {@link NullPointerException} * @since 9 */ - public final void putLongUnaligned(Object o, long offset, long x, boolean bigEndian) { + @ForceInline + public void putLongUnaligned(Object o, long offset, long x, boolean bigEndian) { putLongUnaligned(o, offset, convEndian(bigEndian, x)); } /** @see #putLongUnaligned(Object, long, long) */ - @IntrinsicCandidate - public final void putIntUnaligned(Object o, long offset, int x) { + @ForceInline + public void putIntUnaligned(Object o, long offset, int x) { + if (UNALIGNED_ACCESS || (offset & 3) == 0) { + putPrimitiveBitsMONative(MO_UNALIGNED_PLAIN, BT_INT, o, offset, x); + } else { + putIntUnalignedSlow(o, offset, x); + } + } + private void putIntUnalignedSlow(Object o, long offset, int x) { if ((offset & 3) == 0) { putInt(o, offset, x); } else if ((offset & 1) == 0) { @@ -3736,13 +3749,21 @@ public final void putIntUnaligned(Object o, long offset, int x) { } } /** @see #putLongUnaligned(Object, long, long, boolean) */ - public final void putIntUnaligned(Object o, long offset, int x, boolean bigEndian) { + @ForceInline + public void putIntUnaligned(Object o, long offset, int x, boolean bigEndian) { putIntUnaligned(o, offset, convEndian(bigEndian, x)); } /** @see #putLongUnaligned(Object, long, long) */ - @IntrinsicCandidate - public final void putShortUnaligned(Object o, long offset, short x) { + @ForceInline + public void putShortUnaligned(Object o, long offset, short x) { + if (UNALIGNED_ACCESS || (offset & 1) == 0) { + putPrimitiveBitsMONative(MO_UNALIGNED_PLAIN, BT_SHORT, o, offset, x); + } else { + putShortUnalignedSlow(o, offset, x); + } + } + private void putShortUnalignedSlow(Object o, long offset, short x) { if ((offset & 1) == 0) { putShort(o, offset, x); } else { @@ -3752,17 +3773,23 @@ public final void putShortUnaligned(Object o, long offset, short x) { } } /** @see #putLongUnaligned(Object, long, long, boolean) */ - public final void putShortUnaligned(Object o, long offset, short x, boolean bigEndian) { + @ForceInline + public void putShortUnaligned(Object o, long offset, short x, boolean bigEndian) { putShortUnaligned(o, offset, convEndian(bigEndian, x)); } /** @see #putLongUnaligned(Object, long, long) */ - @IntrinsicCandidate - public final void putCharUnaligned(Object o, long offset, char x) { - putShortUnaligned(o, offset, (short)x); + @ForceInline + public void putCharUnaligned(Object o, long offset, char x) { + if (UNALIGNED_ACCESS || (offset & 1) == 0) { + putPrimitiveBitsMONative(MO_UNALIGNED_PLAIN, BT_CHAR, o, offset, x); + } else { + putShortUnalignedSlow(o, offset, (short)x); + } } /** @see #putLongUnaligned(Object, long, long, boolean) */ - public final void putCharUnaligned(Object o, long offset, char x, boolean bigEndian) { + @ForceInline + public void putCharUnaligned(Object o, long offset, char x, boolean bigEndian) { putCharUnaligned(o, offset, convEndian(bigEndian, x)); } @@ -3770,6 +3797,7 @@ public final void putCharUnaligned(Object o, long offset, char x, boolean bigEnd // These methods construct integers from bytes. The byte ordering // is the native endianness of this platform. + @ForceInline private static long makeLong(byte i0, byte i1, byte i2, byte i3, byte i4, byte i5, byte i6, byte i7) { return ((toUnsignedLong(i0) << pickPos(56, 0)) | (toUnsignedLong(i1) << pickPos(56, 8)) @@ -3780,26 +3808,31 @@ private static long makeLong(byte i0, byte i1, byte i2, byte i3, byte i4, byte i | (toUnsignedLong(i6) << pickPos(56, 48)) | (toUnsignedLong(i7) << pickPos(56, 56))); } + @ForceInline private static long makeLong(short i0, short i1, short i2, short i3) { return ((toUnsignedLong(i0) << pickPos(48, 0)) | (toUnsignedLong(i1) << pickPos(48, 16)) | (toUnsignedLong(i2) << pickPos(48, 32)) | (toUnsignedLong(i3) << pickPos(48, 48))); } + @ForceInline private static long makeLong(int i0, int i1) { return (toUnsignedLong(i0) << pickPos(32, 0)) | (toUnsignedLong(i1) << pickPos(32, 32)); } + @ForceInline private static int makeInt(short i0, short i1) { return (toUnsignedInt(i0) << pickPos(16, 0)) | (toUnsignedInt(i1) << pickPos(16, 16)); } + @ForceInline private static int makeInt(byte i0, byte i1, byte i2, byte i3) { return ((toUnsignedInt(i0) << pickPos(24, 0)) | (toUnsignedInt(i1) << pickPos(24, 8)) | (toUnsignedInt(i2) << pickPos(24, 16)) | (toUnsignedInt(i3) << pickPos(24, 24))); } + @ForceInline private static short makeShort(byte i0, byte i1) { return (short)((toUnsignedInt(i0) << pickPos(8, 0)) | (toUnsignedInt(i1) << pickPos(8, 8))); @@ -3812,6 +3845,7 @@ private static short makeShort(byte i0, byte i1) { // These methods write integers to memory from smaller parts // provided by their caller. The ordering in which these parts // are written is the native endianness of this platform. + @ForceInline private void putLongParts(Object o, long offset, byte i0, byte i1, byte i2, byte i3, byte i4, byte i5, byte i6, byte i7) { putByte(o, offset + 0, pick(i0, i7)); putByte(o, offset + 1, pick(i1, i6)); @@ -3822,26 +3856,31 @@ private void putLongParts(Object o, long offset, byte i0, byte i1, byte i2, byte putByte(o, offset + 6, pick(i6, i1)); putByte(o, offset + 7, pick(i7, i0)); } + @ForceInline private void putLongParts(Object o, long offset, short i0, short i1, short i2, short i3) { putShort(o, offset + 0, pick(i0, i3)); putShort(o, offset + 2, pick(i1, i2)); putShort(o, offset + 4, pick(i2, i1)); putShort(o, offset + 6, pick(i3, i0)); } + @ForceInline private void putLongParts(Object o, long offset, int i0, int i1) { putInt(o, offset + 0, pick(i0, i1)); putInt(o, offset + 4, pick(i1, i0)); } + @ForceInline private void putIntParts(Object o, long offset, short i0, short i1) { putShort(o, offset + 0, pick(i0, i1)); putShort(o, offset + 2, pick(i1, i0)); } + @ForceInline private void putIntParts(Object o, long offset, byte i0, byte i1, byte i2, byte i3) { putByte(o, offset + 0, pick(i0, i3)); putByte(o, offset + 1, pick(i1, i2)); putByte(o, offset + 2, pick(i2, i1)); putByte(o, offset + 3, pick(i3, i0)); } + @ForceInline private void putShortParts(Object o, long offset, byte i0, byte i1) { putByte(o, offset + 0, pick(i0, i1)); putByte(o, offset + 1, pick(i1, i0)); diff --git a/src/java.base/share/classes/jdk/internal/misc/X-ScopedMemoryAccess-bin.java.template b/src/java.base/share/classes/jdk/internal/misc/X-ScopedMemoryAccess-bin.java.template index dd8fabfceaaaf..dfd26bff45bdd 100644 --- a/src/java.base/share/classes/jdk/internal/misc/X-ScopedMemoryAccess-bin.java.template +++ b/src/java.base/share/classes/jdk/internal/misc/X-ScopedMemoryAccess-bin.java.template @@ -99,7 +99,7 @@ if (session != null) { session.checkValidStateRaw(); } - return UNSAFE.get$Type$Volatile(base, offset); + return UNSAFE.get$Type$MO(Unsafe.MO_VOLATILE, base, offset); } finally { Reference.reachabilityFence(session); } @@ -120,7 +120,7 @@ if (session != null) { session.checkValidStateRaw(); } - UNSAFE.put$Type$Volatile(base, offset, value); + UNSAFE.put$Type$MO(Unsafe.MO_VOLATILE, base, offset, value); } finally { Reference.reachabilityFence(session); } @@ -141,7 +141,7 @@ if (session != null) { session.checkValidStateRaw(); } - return UNSAFE.get$Type$Acquire(base, offset); + return UNSAFE.get$Type$MO(Unsafe.MO_ACQUIRE, base, offset); } finally { Reference.reachabilityFence(session); } @@ -162,7 +162,7 @@ if (session != null) { session.checkValidStateRaw(); } - UNSAFE.put$Type$Release(base, offset, value); + UNSAFE.put$Type$MO(Unsafe.MO_RELEASE, base, offset, value); } finally { Reference.reachabilityFence(session); } @@ -183,7 +183,7 @@ if (session != null) { session.checkValidStateRaw(); } - return UNSAFE.get$Type$Opaque(base, offset); + return UNSAFE.get$Type$MO(Unsafe.MO_OPAQUE, base, offset); } finally { Reference.reachabilityFence(session); } @@ -203,7 +203,7 @@ if (session != null) { session.checkValidStateRaw(); } - UNSAFE.put$Type$Opaque(base, offset, value); + UNSAFE.put$Type$MO(Unsafe.MO_OPAQUE, base, offset, value); } finally { Reference.reachabilityFence(session); } @@ -266,7 +266,7 @@ if (session != null) { session.checkValidStateRaw(); } - return UNSAFE.compareAndExchange$Type$Acquire(base, offset, expected, value); + return UNSAFE.compareAndExchange$Type$MO(Unsafe.MO_ACQUIRE, base, offset, expected, value); } finally { Reference.reachabilityFence(session); } @@ -287,7 +287,7 @@ if (session != null) { session.checkValidStateRaw(); } - return UNSAFE.compareAndExchange$Type$Release(base, offset, expected, value); + return UNSAFE.compareAndExchange$Type$MO(Unsafe.MO_RELEASE, base, offset, expected, value); } finally { Reference.reachabilityFence(session); } @@ -308,7 +308,7 @@ if (session != null) { session.checkValidStateRaw(); } - return UNSAFE.weakCompareAndSet$Type$Plain(base, offset, expected, value); + return UNSAFE.compareAndSet$Type$MO(Unsafe.MO_WEAK_CAS_PLAIN, base, offset, expected, value); } finally { Reference.reachabilityFence(session); } @@ -329,7 +329,7 @@ if (session != null) { session.checkValidStateRaw(); } - return UNSAFE.weakCompareAndSet$Type$(base, offset, expected, value); + return UNSAFE.compareAndSet$Type$MO(Unsafe.MO_WEAK_CAS_VOLATILE, base, offset, expected, value); } finally { Reference.reachabilityFence(session); } @@ -350,7 +350,7 @@ if (session != null) { session.checkValidStateRaw(); } - return UNSAFE.weakCompareAndSet$Type$Acquire(base, offset, expected, value); + return UNSAFE.compareAndSet$Type$MO(Unsafe.MO_WEAK_CAS_ACQUIRE, base, offset, expected, value); } finally { Reference.reachabilityFence(session); } @@ -371,7 +371,7 @@ if (session != null) { session.checkValidStateRaw(); } - return UNSAFE.weakCompareAndSet$Type$Release(base, offset, expected, value); + return UNSAFE.compareAndSet$Type$MO(Unsafe.MO_WEAK_CAS_RELEASE, base, offset, expected, value); } finally { Reference.reachabilityFence(session); } @@ -413,7 +413,7 @@ if (session != null) { session.checkValidStateRaw(); } - return UNSAFE.getAndSet$Type$Acquire(base, offset, value); + return UNSAFE.getAndSet$Type$MO(Unsafe.MO_ACQUIRE, base, offset, value); } finally { Reference.reachabilityFence(session); } @@ -434,7 +434,7 @@ if (session != null) { session.checkValidStateRaw(); } - return UNSAFE.getAndSet$Type$Release(base, offset, value); + return UNSAFE.getAndSet$Type$MO(Unsafe.MO_RELEASE, base, offset, value); } finally { Reference.reachabilityFence(session); } @@ -478,7 +478,7 @@ if (session != null) { session.checkValidStateRaw(); } - return UNSAFE.getAndAdd$Type$Acquire(base, offset, delta); + return UNSAFE.getAndAdd$Type$MO(Unsafe.MO_ACQUIRE, base, offset, delta); } finally { Reference.reachabilityFence(session); } @@ -499,7 +499,7 @@ if (session != null) { session.checkValidStateRaw(); } - return UNSAFE.getAndAdd$Type$Release(base, offset, delta); + return UNSAFE.getAndAdd$Type$MO(Unsafe.MO_RELEASE, base, offset, delta); } finally { Reference.reachabilityFence(session); } @@ -543,7 +543,7 @@ if (session != null) { session.checkValidStateRaw(); } - return UNSAFE.getAndBitwiseOr$Type$Acquire(base, offset, value); + return UNSAFE.getAndBitwiseOr$Type$MO(Unsafe.MO_ACQUIRE, base, offset, value); } finally { Reference.reachabilityFence(session); } @@ -564,7 +564,7 @@ if (session != null) { session.checkValidStateRaw(); } - return UNSAFE.getAndBitwiseOr$Type$Release(base, offset, value); + return UNSAFE.getAndBitwiseOr$Type$MO(Unsafe.MO_RELEASE, base, offset, value); } finally { Reference.reachabilityFence(session); } @@ -606,7 +606,7 @@ if (session != null) { session.checkValidStateRaw(); } - return UNSAFE.getAndBitwiseAnd$Type$Acquire(base, offset, value); + return UNSAFE.getAndBitwiseAnd$Type$MO(Unsafe.MO_ACQUIRE, base, offset, value); } finally { Reference.reachabilityFence(session); } @@ -627,7 +627,7 @@ if (session != null) { session.checkValidStateRaw(); } - return UNSAFE.getAndBitwiseAnd$Type$Release(base, offset, value); + return UNSAFE.getAndBitwiseAnd$Type$MO(Unsafe.MO_RELEASE, base, offset, value); } finally { Reference.reachabilityFence(session); } @@ -669,7 +669,7 @@ if (session != null) { session.checkValidStateRaw(); } - return UNSAFE.getAndBitwiseXor$Type$Acquire(base, offset, value); + return UNSAFE.getAndBitwiseXor$Type$MO(Unsafe.MO_ACQUIRE, base, offset, value); } finally { Reference.reachabilityFence(session); } @@ -690,7 +690,7 @@ if (session != null) { session.checkValidStateRaw(); } - return UNSAFE.getAndBitwiseXor$Type$Release(base, offset, value); + return UNSAFE.getAndBitwiseXor$Type$MO(Unsafe.MO_RELEASE, base, offset, value); } finally { Reference.reachabilityFence(session); } diff --git a/src/java.base/share/classes/sun/nio/fs/Cancellable.java b/src/java.base/share/classes/sun/nio/fs/Cancellable.java index df3c95abcc936..6b45df3e7671e 100644 --- a/src/java.base/share/classes/sun/nio/fs/Cancellable.java +++ b/src/java.base/share/classes/sun/nio/fs/Cancellable.java @@ -48,7 +48,7 @@ abstract class Cancellable implements Runnable { protected Cancellable() { pollingAddress = unsafe.allocateMemory(4); - unsafe.putIntVolatile(null, pollingAddress, 0); + unsafe.putIntMO(Unsafe.MO_VOLATILE, null, pollingAddress, 0); } /** @@ -75,7 +75,7 @@ protected int cancelValue() { final void cancel() { synchronized (lock) { if (!completed) { - unsafe.putIntVolatile(null, pollingAddress, cancelValue()); + unsafe.putIntMO(Unsafe.MO_VOLATILE, null, pollingAddress, cancelValue()); } } } diff --git a/src/jdk.internal.vm.ci/share/classes/jdk/vm/ci/hotspot/HotSpotConstantPool.java b/src/jdk.internal.vm.ci/share/classes/jdk/vm/ci/hotspot/HotSpotConstantPool.java index 20e3915527d11..7175964cc9715 100644 --- a/src/jdk.internal.vm.ci/share/classes/jdk/vm/ci/hotspot/HotSpotConstantPool.java +++ b/src/jdk.internal.vm.ci/share/classes/jdk/vm/ci/hotspot/HotSpotConstantPool.java @@ -277,8 +277,8 @@ public long getMetadataHandle() { private JvmConstant getTagAt(int index) { checkBounds(index); HotSpotVMConfig config = config(); - final long metaspaceConstantPoolTags = UNSAFE.getAddress(getConstantPoolPointer() + config.constantPoolTagsOffset); - final int tag = UNSAFE.getByteVolatile(null, metaspaceConstantPoolTags + config.arrayU1DataOffset + index); + final long metaspaceConstantPoolTags = UNSAFE.getAddress(null, getConstantPoolPointer() + config.constantPoolTagsOffset); + final int tag = UNSAFE.getByteMO(UnsafeAccess.MO_VOLATILE, null, metaspaceConstantPoolTags + config.arrayU1DataOffset + index); if (tag == 0) { return null; } diff --git a/src/jdk.internal.vm.ci/share/classes/jdk/vm/ci/hotspot/UnsafeAccess.java b/src/jdk.internal.vm.ci/share/classes/jdk/vm/ci/hotspot/UnsafeAccess.java index afdff7a88901b..73960623ecc3b 100644 --- a/src/jdk.internal.vm.ci/share/classes/jdk/vm/ci/hotspot/UnsafeAccess.java +++ b/src/jdk.internal.vm.ci/share/classes/jdk/vm/ci/hotspot/UnsafeAccess.java @@ -30,4 +30,5 @@ class UnsafeAccess { static final Unsafe UNSAFE = Unsafe.getUnsafe(); + public static final byte MO_VOLATILE = Unsafe.MO_VOLATILE; } diff --git a/src/jdk.unsupported/share/classes/sun/misc/Unsafe.java b/src/jdk.unsupported/share/classes/sun/misc/Unsafe.java index b0a27d368ff7c..80c2712fadb4a 100644 --- a/src/jdk.unsupported/share/classes/sun/misc/Unsafe.java +++ b/src/jdk.unsupported/share/classes/sun/misc/Unsafe.java @@ -41,6 +41,8 @@ import jdk.internal.reflect.CallerSensitive; import jdk.internal.reflect.Reflection; +import static jdk.internal.misc.Unsafe.*; //MO_VOLATILE, etc. + /** * A collection of methods for performing low-level, unsafe operations. * Although the class and all methods are public, use of this class is @@ -437,7 +439,7 @@ public void putDouble(Object o, long offset, double x) { @ForceInline public byte getByte(long address) { beforeMemoryAccess(); - return theInternalUnsafe.getByte(address); + return theInternalUnsafe.getByte(null, address); } /** @@ -453,7 +455,7 @@ public byte getByte(long address) { @ForceInline public void putByte(long address, byte x) { beforeMemoryAccess(); - theInternalUnsafe.putByte(address, x); + theInternalUnsafe.putByte(null, address, x); } /** @@ -465,7 +467,7 @@ public void putByte(long address, byte x) { @ForceInline public short getShort(long address) { beforeMemoryAccess(); - return theInternalUnsafe.getShort(address); + return theInternalUnsafe.getShort(null, address); } /** @@ -477,7 +479,7 @@ public short getShort(long address) { @ForceInline public void putShort(long address, short x) { beforeMemoryAccess(); - theInternalUnsafe.putShort(address, x); + theInternalUnsafe.putShort(null, address, x); } /** @@ -489,7 +491,7 @@ public void putShort(long address, short x) { @ForceInline public char getChar(long address) { beforeMemoryAccess(); - return theInternalUnsafe.getChar(address); + return theInternalUnsafe.getChar(null, address); } /** @@ -501,7 +503,7 @@ public char getChar(long address) { @ForceInline public void putChar(long address, char x) { beforeMemoryAccess(); - theInternalUnsafe.putChar(address, x); + theInternalUnsafe.putChar(null, address, x); } /** @@ -513,7 +515,7 @@ public void putChar(long address, char x) { @ForceInline public int getInt(long address) { beforeMemoryAccess(); - return theInternalUnsafe.getInt(address); + return theInternalUnsafe.getInt(null, address); } /** @@ -525,7 +527,7 @@ public int getInt(long address) { @ForceInline public void putInt(long address, int x) { beforeMemoryAccess(); - theInternalUnsafe.putInt(address, x); + theInternalUnsafe.putInt(null, address, x); } /** @@ -537,7 +539,7 @@ public void putInt(long address, int x) { @ForceInline public long getLong(long address) { beforeMemoryAccess(); - return theInternalUnsafe.getLong(address); + return theInternalUnsafe.getLong(null, address); } /** @@ -549,7 +551,7 @@ public long getLong(long address) { @ForceInline public void putLong(long address, long x) { beforeMemoryAccess(); - theInternalUnsafe.putLong(address, x); + theInternalUnsafe.putLong(null, address, x); } /** @@ -561,7 +563,7 @@ public void putLong(long address, long x) { @ForceInline public float getFloat(long address) { beforeMemoryAccess(); - return theInternalUnsafe.getFloat(address); + return theInternalUnsafe.getFloat(null, address); } /** @@ -573,7 +575,7 @@ public float getFloat(long address) { @ForceInline public void putFloat(long address, float x) { beforeMemoryAccess(); - theInternalUnsafe.putFloat(address, x); + theInternalUnsafe.putFloat(null, address, x); } /** @@ -585,7 +587,7 @@ public void putFloat(long address, float x) { @ForceInline public double getDouble(long address) { beforeMemoryAccess(); - return theInternalUnsafe.getDouble(address); + return theInternalUnsafe.getDouble(null, address); } /** @@ -597,7 +599,7 @@ public double getDouble(long address) { @ForceInline public void putDouble(long address, double x) { beforeMemoryAccess(); - theInternalUnsafe.putDouble(address, x); + theInternalUnsafe.putDouble(null, address, x); } @@ -621,7 +623,7 @@ public void putDouble(long address, double x) { @ForceInline public long getAddress(long address) { beforeMemoryAccess(); - return theInternalUnsafe.getAddress(address); + return theInternalUnsafe.getAddress(null, address); } /** @@ -640,7 +642,7 @@ public long getAddress(long address) { @ForceInline public void putAddress(long address, long x) { beforeMemoryAccess(); - theInternalUnsafe.putAddress(address, x); + theInternalUnsafe.putAddress(null, address, x); } @@ -1261,7 +1263,7 @@ public final boolean compareAndSwapLong(Object o, long offset, @ForceInline public Object getObjectVolatile(Object o, long offset) { beforeMemoryAccess(); - return theInternalUnsafe.getReferenceVolatile(o, offset); + return theInternalUnsafe.getReferenceMO(MO_VOLATILE, o, offset); } /** @@ -1274,7 +1276,7 @@ public Object getObjectVolatile(Object o, long offset) { @ForceInline public void putObjectVolatile(Object o, long offset, Object x) { beforeMemoryAccess(); - theInternalUnsafe.putReferenceVolatile(o, offset, x); + theInternalUnsafe.putReferenceMO(MO_VOLATILE, o, offset, x); } /** Volatile version of {@link #getInt(Object, long)}. @@ -1285,7 +1287,7 @@ public void putObjectVolatile(Object o, long offset, Object x) { @ForceInline public int getIntVolatile(Object o, long offset) { beforeMemoryAccess(); - return theInternalUnsafe.getIntVolatile(o, offset); + return theInternalUnsafe.getIntMO(MO_VOLATILE, o, offset); } /** Volatile version of {@link #putInt(Object, long, int)}. @@ -1296,7 +1298,7 @@ public int getIntVolatile(Object o, long offset) { @ForceInline public void putIntVolatile(Object o, long offset, int x) { beforeMemoryAccess(); - theInternalUnsafe.putIntVolatile(o, offset, x); + theInternalUnsafe.putIntMO(MO_VOLATILE, o, offset, x); } /** Volatile version of {@link #getBoolean(Object, long)}. @@ -1307,7 +1309,7 @@ public void putIntVolatile(Object o, long offset, int x) { @ForceInline public boolean getBooleanVolatile(Object o, long offset) { beforeMemoryAccess(); - return theInternalUnsafe.getBooleanVolatile(o, offset); + return theInternalUnsafe.getBooleanMO(MO_VOLATILE, o, offset); } /** Volatile version of {@link #putBoolean(Object, long, boolean)}. @@ -1318,7 +1320,7 @@ public boolean getBooleanVolatile(Object o, long offset) { @ForceInline public void putBooleanVolatile(Object o, long offset, boolean x) { beforeMemoryAccess(); - theInternalUnsafe.putBooleanVolatile(o, offset, x); + theInternalUnsafe.putBooleanMO(MO_VOLATILE, o, offset, x); } /** Volatile version of {@link #getByte(Object, long)}. @@ -1330,7 +1332,7 @@ public void putBooleanVolatile(Object o, long offset, boolean x) { @ForceInline public byte getByteVolatile(Object o, long offset) { beforeMemoryAccess(); - return theInternalUnsafe.getByteVolatile(o, offset); + return theInternalUnsafe.getByteMO(MO_VOLATILE, o, offset); } /** Volatile version of {@link #putByte(Object, long, byte)}. @@ -1341,7 +1343,7 @@ public byte getByteVolatile(Object o, long offset) { @ForceInline public void putByteVolatile(Object o, long offset, byte x) { beforeMemoryAccess(); - theInternalUnsafe.putByteVolatile(o, offset, x); + theInternalUnsafe.putByteMO(MO_VOLATILE, o, offset, x); } /** Volatile version of {@link #getShort(Object, long)}. @@ -1352,7 +1354,7 @@ public void putByteVolatile(Object o, long offset, byte x) { @ForceInline public short getShortVolatile(Object o, long offset) { beforeMemoryAccess(); - return theInternalUnsafe.getShortVolatile(o, offset); + return theInternalUnsafe.getShortMO(MO_VOLATILE, o, offset); } /** Volatile version of {@link #putShort(Object, long, short)}. @@ -1363,7 +1365,7 @@ public short getShortVolatile(Object o, long offset) { @ForceInline public void putShortVolatile(Object o, long offset, short x) { beforeMemoryAccess(); - theInternalUnsafe.putShortVolatile(o, offset, x); + theInternalUnsafe.putShortMO(MO_VOLATILE, o, offset, x); } /** Volatile version of {@link #getChar(Object, long)}. @@ -1374,7 +1376,7 @@ public void putShortVolatile(Object o, long offset, short x) { @ForceInline public char getCharVolatile(Object o, long offset) { beforeMemoryAccess(); - return theInternalUnsafe.getCharVolatile(o, offset); + return theInternalUnsafe.getCharMO(MO_VOLATILE, o, offset); } /** Volatile version of {@link #putChar(Object, long, char)}. @@ -1385,7 +1387,7 @@ public char getCharVolatile(Object o, long offset) { @ForceInline public void putCharVolatile(Object o, long offset, char x) { beforeMemoryAccess(); - theInternalUnsafe.putCharVolatile(o, offset, x); + theInternalUnsafe.putCharMO(MO_VOLATILE, o, offset, x); } /** Volatile version of {@link #getLong(Object, long)}. @@ -1396,7 +1398,7 @@ public void putCharVolatile(Object o, long offset, char x) { @ForceInline public long getLongVolatile(Object o, long offset) { beforeMemoryAccess(); - return theInternalUnsafe.getLongVolatile(o, offset); + return theInternalUnsafe.getLongMO(MO_VOLATILE, o, offset); } /** Volatile version of {@link #putLong(Object, long, long)}. @@ -1407,7 +1409,7 @@ public long getLongVolatile(Object o, long offset) { @ForceInline public void putLongVolatile(Object o, long offset, long x) { beforeMemoryAccess(); - theInternalUnsafe.putLongVolatile(o, offset, x); + theInternalUnsafe.putLongMO(MO_VOLATILE, o, offset, x); } /** Volatile version of {@link #getFloat(Object, long)}. @@ -1418,7 +1420,7 @@ public void putLongVolatile(Object o, long offset, long x) { @ForceInline public float getFloatVolatile(Object o, long offset) { beforeMemoryAccess(); - return theInternalUnsafe.getFloatVolatile(o, offset); + return theInternalUnsafe.getFloatMO(MO_VOLATILE, o, offset); } /** Volatile version of {@link #putFloat(Object, long, float)}. @@ -1429,7 +1431,7 @@ public float getFloatVolatile(Object o, long offset) { @ForceInline public void putFloatVolatile(Object o, long offset, float x) { beforeMemoryAccess(); - theInternalUnsafe.putFloatVolatile(o, offset, x); + theInternalUnsafe.putFloatMO(MO_VOLATILE, o, offset, x); } /** Volatile version of {@link #getDouble(Object, long)}. @@ -1440,7 +1442,7 @@ public void putFloatVolatile(Object o, long offset, float x) { @ForceInline public double getDoubleVolatile(Object o, long offset) { beforeMemoryAccess(); - return theInternalUnsafe.getDoubleVolatile(o, offset); + return theInternalUnsafe.getDoubleMO(MO_VOLATILE, o, offset); } /** Volatile version of {@link #putDouble(Object, long, double)}. @@ -1451,7 +1453,7 @@ public double getDoubleVolatile(Object o, long offset) { @ForceInline public void putDoubleVolatile(Object o, long offset, double x) { beforeMemoryAccess(); - theInternalUnsafe.putDoubleVolatile(o, offset, x); + theInternalUnsafe.putDoubleMO(MO_VOLATILE, o, offset, x); } /** @@ -1469,7 +1471,7 @@ public void putDoubleVolatile(Object o, long offset, double x) { @ForceInline public void putOrderedObject(Object o, long offset, Object x) { beforeMemoryAccess(); - theInternalUnsafe.putReferenceRelease(o, offset, x); + theInternalUnsafe.putReferenceMO(MO_RELEASE, o, offset, x); } /** Ordered/Lazy version of {@link #putIntVolatile(Object, long, int)}. @@ -1480,7 +1482,7 @@ public void putOrderedObject(Object o, long offset, Object x) { @ForceInline public void putOrderedInt(Object o, long offset, int x) { beforeMemoryAccess(); - theInternalUnsafe.putIntRelease(o, offset, x); + theInternalUnsafe.putIntMO(MO_RELEASE, o, offset, x); } /** Ordered/Lazy version of {@link #putLongVolatile(Object, long, long)}. @@ -1491,7 +1493,7 @@ public void putOrderedInt(Object o, long offset, int x) { @ForceInline public void putOrderedLong(Object o, long offset, long x) { beforeMemoryAccess(); - theInternalUnsafe.putLongRelease(o, offset, x); + theInternalUnsafe.putLongMO(MO_RELEASE, o, offset, x); } /** @@ -1760,7 +1762,7 @@ public void invokeCleaner(java.nio.ByteBuffer directBuffer) { private static @Stable boolean memoryAccessWarned; private static boolean isMemoryAccessWarned() { - return theInternalUnsafe.getBooleanVolatile(MEMORY_ACCESS_WARNED_BASE, MEMORY_ACCESS_WARNED_OFFSET); + return theInternalUnsafe.getBooleanMO(MO_VOLATILE, MEMORY_ACCESS_WARNED_BASE, MEMORY_ACCESS_WARNED_OFFSET); } private static boolean trySetMemoryAccessWarned() { diff --git a/test/hotspot/jtreg/compiler/c2/TestMergeStores.java b/test/hotspot/jtreg/compiler/c2/TestMergeStores.java index 5e6a757dd5f8f..7419ec8a6f1e0 100644 --- a/test/hotspot/jtreg/compiler/c2/TestMergeStores.java +++ b/test/hotspot/jtreg/compiler/c2/TestMergeStores.java @@ -687,14 +687,14 @@ static Object[] test1f(byte[] a) { IRNode.STORE_B_OF_CLASS, "byte\\[int:>=0] \\(java/lang/Cloneable,java/io/Serializable\\)", "8"}, applyIf = {"UseUnalignedAccesses", "true"}) static Object[] test1g(byte[] a) { - UNSAFE.putByteRelease(a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 0, (byte)0xbe); - UNSAFE.putByteRelease(a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 1, (byte)0xba); - UNSAFE.putByteRelease(a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 2, (byte)0xad); - UNSAFE.putByteRelease(a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 3, (byte)0xba); - UNSAFE.putByteRelease(a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 4, (byte)0xef); - UNSAFE.putByteRelease(a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 5, (byte)0xbe); - UNSAFE.putByteRelease(a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 6, (byte)0xad); - UNSAFE.putByteRelease(a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 7, (byte)0xde); + UNSAFE.putByteMO(Unsafe.MO_RELEASE, a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 0, (byte)0xbe); + UNSAFE.putByteMO(Unsafe.MO_RELEASE, a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 1, (byte)0xba); + UNSAFE.putByteMO(Unsafe.MO_RELEASE, a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 2, (byte)0xad); + UNSAFE.putByteMO(Unsafe.MO_RELEASE, a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 3, (byte)0xba); + UNSAFE.putByteMO(Unsafe.MO_RELEASE, a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 4, (byte)0xef); + UNSAFE.putByteMO(Unsafe.MO_RELEASE, a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 5, (byte)0xbe); + UNSAFE.putByteMO(Unsafe.MO_RELEASE, a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 6, (byte)0xad); + UNSAFE.putByteMO(Unsafe.MO_RELEASE, a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 7, (byte)0xde); return new Object[]{ a }; } @@ -721,14 +721,14 @@ static Object[] test1h(byte[] a) { IRNode.STORE_B_OF_CLASS, "byte\\[int:>=0] \\(java/lang/Cloneable,java/io/Serializable\\)", "8"}, applyIf = {"UseUnalignedAccesses", "true"}) static Object[] test1i(byte[] a) { - UNSAFE.putByteOpaque(a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 0, (byte)0xbe); - UNSAFE.putByteOpaque(a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 1, (byte)0xba); - UNSAFE.putByteOpaque(a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 2, (byte)0xad); - UNSAFE.putByteOpaque(a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 3, (byte)0xba); - UNSAFE.putByteOpaque(a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 4, (byte)0xef); - UNSAFE.putByteOpaque(a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 5, (byte)0xbe); - UNSAFE.putByteOpaque(a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 6, (byte)0xad); - UNSAFE.putByteOpaque(a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 7, (byte)0xde); + UNSAFE.putByteMO(Unsafe.MO_OPAQUE, a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 0, (byte)0xbe); + UNSAFE.putByteMO(Unsafe.MO_OPAQUE, a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 1, (byte)0xba); + UNSAFE.putByteMO(Unsafe.MO_OPAQUE, a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 2, (byte)0xad); + UNSAFE.putByteMO(Unsafe.MO_OPAQUE, a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 3, (byte)0xba); + UNSAFE.putByteMO(Unsafe.MO_OPAQUE, a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 4, (byte)0xef); + UNSAFE.putByteMO(Unsafe.MO_OPAQUE, a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 5, (byte)0xbe); + UNSAFE.putByteMO(Unsafe.MO_OPAQUE, a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 6, (byte)0xad); + UNSAFE.putByteMO(Unsafe.MO_OPAQUE, a, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 7, (byte)0xde); return new Object[]{ a }; } diff --git a/test/hotspot/jtreg/compiler/c2/aarch64/TestUnsafeVolatileWeakCAS.java b/test/hotspot/jtreg/compiler/c2/aarch64/TestUnsafeVolatileWeakCAS.java index 54d3a8ad34cb5..b385068703f03 100644 --- a/test/hotspot/jtreg/compiler/c2/aarch64/TestUnsafeVolatileWeakCAS.java +++ b/test/hotspot/jtreg/compiler/c2/aarch64/TestUnsafeVolatileWeakCAS.java @@ -114,26 +114,26 @@ public static void main(String[] args) public boolean testInt(int x, int i) { - return unsafe.weakCompareAndSetInt(this, f_int_off, x, i); + return unsafe.compareAndSetIntMO(Unsafe.MO_WEAK_CAS_VOLATILE, this, f_int_off, x, i); } public boolean testObj(Object x, Object o) { - return unsafe.weakCompareAndSetReference(this, f_obj_off, x, o); + return unsafe.compareAndSetReferenceMO(Unsafe.MO_WEAK_CAS_VOLATILE, this, f_obj_off, x, o); } public boolean testLong(long x, long i) { - return unsafe.weakCompareAndSetLong(this, f_long_off, x, i); + return unsafe.compareAndSetLongMO(Unsafe.MO_WEAK_CAS_VOLATILE, this, f_long_off, x, i); } public boolean testByte(byte x, byte i) { - return unsafe.weakCompareAndSetByte(this, f_byte_off, x, i); + return unsafe.compareAndSetByteMO(Unsafe.MO_WEAK_CAS_VOLATILE, this, f_byte_off, x, i); } public boolean testShort(short x, short i) { - return unsafe.weakCompareAndSetShort(this, f_short_off, x, i); + return unsafe.compareAndSetShortMO(Unsafe.MO_WEAK_CAS_VOLATILE, this, f_short_off, x, i); } } diff --git a/test/hotspot/jtreg/compiler/c2/cr8004867/TestIntUnsafeOrdered.java b/test/hotspot/jtreg/compiler/c2/cr8004867/TestIntUnsafeOrdered.java index 342b146a0c893..74b4da7cf92a6 100644 --- a/test/hotspot/jtreg/compiler/c2/cr8004867/TestIntUnsafeOrdered.java +++ b/test/hotspot/jtreg/compiler/c2/cr8004867/TestIntUnsafeOrdered.java @@ -773,213 +773,213 @@ private final static long byte_offset(int i) { static void test_ci(int[] a) { for (int i = 0; i < ARRLEN; i+=1) { - unsafe.putIntRelease(a, byte_offset(i), -123); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i), -123); } } static void test_vi(int[] a, int b, int old) { for (int i = 0; i < ARRLEN; i+=1) { - unsafe.putIntRelease(a, byte_offset(i), b); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i), b); } } static void test_cp(int[] a, int[] b) { for (int i = 0; i < ARRLEN; i+=1) { - unsafe.putIntRelease(a, byte_offset(i), b[i]); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i), b[i]); } } static void test_2ci(int[] a, int[] b) { for (int i = 0; i < ARRLEN; i+=1) { - unsafe.putIntRelease(a, byte_offset(i), -123); - unsafe.putIntRelease(b, byte_offset(i), -103); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i), -123); + unsafe.putIntMO(Unsafe.MO_RELEASE, b, byte_offset(i), -103); } } static void test_2vi(int[] a, int[] b, int c, int d) { for (int i = 0; i < ARRLEN; i+=1) { - unsafe.putIntRelease(a, byte_offset(i), c); - unsafe.putIntRelease(b, byte_offset(i), d); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i), c); + unsafe.putIntMO(Unsafe.MO_RELEASE, b, byte_offset(i), d); } } static void test_ci_neg(int[] a, int old) { for (int i = ARRLEN-1; i >= 0; i-=1) { - unsafe.putIntRelease(a, byte_offset(i), -123); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i), -123); } } static void test_vi_neg(int[] a, int b, int old) { for (int i = ARRLEN-1; i >= 0; i-=1) { - unsafe.putIntRelease(a, byte_offset(i), b); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i), b); } } static void test_cp_neg(int[] a, int[] b) { for (int i = ARRLEN-1; i >= 0; i-=1) { - unsafe.putIntRelease(a, byte_offset(i), b[i]); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i), b[i]); } } static void test_2ci_neg(int[] a, int[] b) { for (int i = ARRLEN-1; i >= 0; i-=1) { - unsafe.putIntRelease(a, byte_offset(i), -123); - unsafe.putIntRelease(b, byte_offset(i), -103); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i), -123); + unsafe.putIntMO(Unsafe.MO_RELEASE, b, byte_offset(i), -103); } } static void test_2vi_neg(int[] a, int[] b, int c, int d) { for (int i = ARRLEN-1; i >= 0; i-=1) { - unsafe.putIntRelease(a, byte_offset(i), c); - unsafe.putIntRelease(b, byte_offset(i), d); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i), c); + unsafe.putIntMO(Unsafe.MO_RELEASE, b, byte_offset(i), d); } } static void test_ci_oppos(int[] a, int old) { int limit = ARRLEN-1; for (int i = 0; i < ARRLEN; i+=1) { - unsafe.putIntRelease(a, byte_offset(limit-i), -123); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(limit-i), -123); } } static void test_vi_oppos(int[] a, int b, int old) { int limit = ARRLEN-1; for (int i = limit; i >= 0; i-=1) { - unsafe.putIntRelease(a, byte_offset(limit-i), b); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(limit-i), b); } } static void test_cp_oppos(int[] a, int[] b) { int limit = ARRLEN-1; for (int i = 0; i < ARRLEN; i+=1) { - unsafe.putIntRelease(a, byte_offset(i), b[limit-i]); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i), b[limit-i]); } } static void test_2ci_oppos(int[] a, int[] b) { int limit = ARRLEN-1; for (int i = 0; i < ARRLEN; i+=1) { - unsafe.putIntRelease(a, byte_offset(limit-i), -123); - unsafe.putIntRelease(b, byte_offset(i), -103); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(limit-i), -123); + unsafe.putIntMO(Unsafe.MO_RELEASE, b, byte_offset(i), -103); } } static void test_2vi_oppos(int[] a, int[] b, int c, int d) { int limit = ARRLEN-1; for (int i = limit; i >= 0; i-=1) { - unsafe.putIntRelease(a, byte_offset(i), c); - unsafe.putIntRelease(b, byte_offset(limit-i), d); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i), c); + unsafe.putIntMO(Unsafe.MO_RELEASE, b, byte_offset(limit-i), d); } } static void test_ci_off(int[] a, int old) { for (int i = 0; i < ARRLEN-OFFSET; i+=1) { - unsafe.putIntRelease(a, byte_offset(i+OFFSET), -123); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i+OFFSET), -123); } } static void test_vi_off(int[] a, int b, int old) { for (int i = 0; i < ARRLEN-OFFSET; i+=1) { - unsafe.putIntRelease(a, byte_offset(i+OFFSET), b); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i+OFFSET), b); } } static void test_cp_off(int[] a, int[] b) { for (int i = 0; i < ARRLEN-OFFSET; i+=1) { - unsafe.putIntRelease(a, byte_offset(i+OFFSET), b[i+OFFSET]); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i+OFFSET), b[i+OFFSET]); } } static void test_2ci_off(int[] a, int[] b) { for (int i = 0; i < ARRLEN-OFFSET; i+=1) { - unsafe.putIntRelease(a, byte_offset(i+OFFSET), -123); - unsafe.putIntRelease(b, byte_offset(i+OFFSET), -103); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i+OFFSET), -123); + unsafe.putIntMO(Unsafe.MO_RELEASE, b, byte_offset(i+OFFSET), -103); } } static void test_2vi_off(int[] a, int[] b, int c, int d) { for (int i = 0; i < ARRLEN-OFFSET; i+=1) { - unsafe.putIntRelease(a, byte_offset(i+OFFSET), c); - unsafe.putIntRelease(b, byte_offset(i+OFFSET), d); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i+OFFSET), c); + unsafe.putIntMO(Unsafe.MO_RELEASE, b, byte_offset(i+OFFSET), d); } } static void test_ci_inv(int[] a, int k, int old) { for (int i = 0; i < ARRLEN-k; i+=1) { - unsafe.putIntRelease(a, byte_offset(i+k), -123); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i+k), -123); } } static void test_vi_inv(int[] a, int b, int k, int old) { for (int i = 0; i < ARRLEN-k; i+=1) { - unsafe.putIntRelease(a, byte_offset(i+k), b); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i+k), b); } } static void test_cp_inv(int[] a, int[] b, int k) { for (int i = 0; i < ARRLEN-k; i+=1) { - unsafe.putIntRelease(a, byte_offset(i+k), b[i+k]); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i+k), b[i+k]); } } static void test_2ci_inv(int[] a, int[] b, int k) { for (int i = 0; i < ARRLEN-k; i+=1) { - unsafe.putIntRelease(a, byte_offset(i+k), -123); - unsafe.putIntRelease(b, byte_offset(i+k), -103); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i+k), -123); + unsafe.putIntMO(Unsafe.MO_RELEASE, b, byte_offset(i+k), -103); } } static void test_2vi_inv(int[] a, int[] b, int c, int d, int k) { for (int i = 0; i < ARRLEN-k; i+=1) { - unsafe.putIntRelease(a, byte_offset(i+k), c); - unsafe.putIntRelease(b, byte_offset(i+k), d); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i+k), c); + unsafe.putIntMO(Unsafe.MO_RELEASE, b, byte_offset(i+k), d); } } static void test_ci_scl(int[] a, int old) { for (int i = 0; i*SCALE < ARRLEN; i+=1) { - unsafe.putIntRelease(a, byte_offset(i*SCALE), -123); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i*SCALE), -123); } } static void test_vi_scl(int[] a, int b, int old) { for (int i = 0; i*SCALE < ARRLEN; i+=1) { - unsafe.putIntRelease(a, byte_offset(i*SCALE), b); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i*SCALE), b); } } static void test_cp_scl(int[] a, int[] b) { for (int i = 0; i*SCALE < ARRLEN; i+=1) { - unsafe.putIntRelease(a, byte_offset(i*SCALE), b[i*SCALE]); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i*SCALE), b[i*SCALE]); } } static void test_2ci_scl(int[] a, int[] b) { for (int i = 0; i*SCALE < ARRLEN; i+=1) { - unsafe.putIntRelease(a, byte_offset(i*SCALE), -123); - unsafe.putIntRelease(b, byte_offset(i*SCALE), -103); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i*SCALE), -123); + unsafe.putIntMO(Unsafe.MO_RELEASE, b, byte_offset(i*SCALE), -103); } } static void test_2vi_scl(int[] a, int[] b, int c, int d) { for (int i = 0; i*SCALE < ARRLEN; i+=1) { - unsafe.putIntRelease(a, byte_offset(i*SCALE), c); - unsafe.putIntRelease(b, byte_offset(i*SCALE), d); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i*SCALE), c); + unsafe.putIntMO(Unsafe.MO_RELEASE, b, byte_offset(i*SCALE), d); } } static void test_cp_alndst(int[] a, int[] b) { for (int i = 0; i < ARRLEN-ALIGN_OFF; i+=1) { - unsafe.putIntRelease(a, byte_offset(i+ALIGN_OFF), b[i]); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i+ALIGN_OFF), b[i]); } } static void test_cp_alnsrc(int[] a, int[] b) { for (int i = 0; i < ARRLEN-ALIGN_OFF; i+=1) { - unsafe.putIntRelease(a, byte_offset(i), b[i+ALIGN_OFF]); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i), b[i+ALIGN_OFF]); } } static void test_2ci_aln(int[] a, int[] b) { for (int i = 0; i < ARRLEN-ALIGN_OFF; i+=1) { - unsafe.putIntRelease(a, byte_offset(i+ALIGN_OFF), -123); - unsafe.putIntRelease(b, byte_offset(i), -103); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i+ALIGN_OFF), -123); + unsafe.putIntMO(Unsafe.MO_RELEASE, b, byte_offset(i), -103); } } static void test_2vi_aln(int[] a, int[] b, int c, int d) { for (int i = 0; i < ARRLEN-ALIGN_OFF; i+=1) { - unsafe.putIntRelease(a, byte_offset(i), c); - unsafe.putIntRelease(b, byte_offset(i+ALIGN_OFF), d); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i), c); + unsafe.putIntMO(Unsafe.MO_RELEASE, b, byte_offset(i+ALIGN_OFF), d); } } static void test_cp_unalndst(int[] a, int[] b) { for (int i = 0; i < ARRLEN-UNALIGN_OFF; i+=1) { - unsafe.putIntRelease(a, byte_offset(i+UNALIGN_OFF), b[i]); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i+UNALIGN_OFF), b[i]); } } static void test_cp_unalnsrc(int[] a, int[] b) { for (int i = 0; i < ARRLEN-UNALIGN_OFF; i+=1) { - unsafe.putIntRelease(a, byte_offset(i), b[i+UNALIGN_OFF]); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i), b[i+UNALIGN_OFF]); } } static void test_2ci_unaln(int[] a, int[] b) { for (int i = 0; i < ARRLEN-UNALIGN_OFF; i+=1) { - unsafe.putIntRelease(a, byte_offset(i+UNALIGN_OFF), -123); - unsafe.putIntRelease(b, byte_offset(i), -103); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i+UNALIGN_OFF), -123); + unsafe.putIntMO(Unsafe.MO_RELEASE, b, byte_offset(i), -103); } } static void test_2vi_unaln(int[] a, int[] b, int c, int d) { for (int i = 0; i < ARRLEN-UNALIGN_OFF; i+=1) { - unsafe.putIntRelease(a, byte_offset(i), c); - unsafe.putIntRelease(b, byte_offset(i+UNALIGN_OFF), d); + unsafe.putIntMO(Unsafe.MO_RELEASE, a, byte_offset(i), c); + unsafe.putIntMO(Unsafe.MO_RELEASE, b, byte_offset(i+UNALIGN_OFF), d); } } diff --git a/test/hotspot/jtreg/compiler/gcbarriers/UnsafeIntrinsicsTest.java b/test/hotspot/jtreg/compiler/gcbarriers/UnsafeIntrinsicsTest.java index 7542f0cd2374d..33311d08092f0 100644 --- a/test/hotspot/jtreg/compiler/gcbarriers/UnsafeIntrinsicsTest.java +++ b/test/hotspot/jtreg/compiler/gcbarriers/UnsafeIntrinsicsTest.java @@ -338,7 +338,7 @@ private Node mergeImplWeakCAS(Node startNode, Node expectedNext, Node head) { // Spurious failures are allowed. So, we retry a couple of times on failure. boolean ok = false; for (int i = 0; i < 3; ++i) { - ok = UNSAFE.weakCompareAndSetReference(startNode, offset, expectedNext, head); + ok = UNSAFE.compareAndSetReferenceMO(Unsafe.MO_WEAK_CAS_VOLATILE, startNode, offset, expectedNext, head); if (ok) break; } if (!ok) { @@ -349,7 +349,7 @@ private Node mergeImplWeakCAS(Node startNode, Node expectedNext, Node head) { private Node mergeImplWeakCASFail(Node startNode, Node expectedNext, Node head) { // Force a fail - if (UNSAFE.weakCompareAndSetReference(startNode, offset, "fail", head)) { + if (UNSAFE.compareAndSetReferenceMO(Unsafe.MO_WEAK_CAS_VOLATILE, startNode, offset, "fail", head)) { throw new Error("This weak CAS should always fail, check your barrier implementation"); } if (startNode.next() != expectedNext) { diff --git a/test/hotspot/jtreg/compiler/unsafe/CASandCAEwithNegExpected.java b/test/hotspot/jtreg/compiler/unsafe/CASandCAEwithNegExpected.java index d1ad8a5f756ef..87f8b4a451a22 100644 --- a/test/hotspot/jtreg/compiler/unsafe/CASandCAEwithNegExpected.java +++ b/test/hotspot/jtreg/compiler/unsafe/CASandCAEwithNegExpected.java @@ -123,19 +123,19 @@ void test() { f_byte = -1; f_short = -1; - if (unsafe.weakCompareAndSetInt(this, f_int_off, -1, 42) && f_int != 42) { + if (unsafe.compareAndSetIntMO(Unsafe.MO_WEAK_CAS_VOLATILE, this, f_int_off, -1, 42) && f_int != 42) { throw new RuntimeException("CAS failed"); } - if (unsafe.weakCompareAndSetLong(this, f_long_off, -1, 42) && f_long != 42) { + if (unsafe.compareAndSetLongMO(Unsafe.MO_WEAK_CAS_VOLATILE, this, f_long_off, -1, 42) && f_long != 42) { throw new RuntimeException("CAS failed"); } - if (unsafe.weakCompareAndSetByte(this, f_byte_off, (byte)-1, (byte)42) && f_byte != 42) { + if (unsafe.compareAndSetByteMO(Unsafe.MO_WEAK_CAS_VOLATILE, this, f_byte_off, (byte)-1, (byte)42) && f_byte != 42) { throw new RuntimeException("CAS failed"); } - if (unsafe.weakCompareAndSetShort(this, f_short_off, (short)-1, (short)42) && f_short != 42) { + if (unsafe.compareAndSetShortMO(Unsafe.MO_WEAK_CAS_VOLATILE, this, f_short_off, (short)-1, (short)42) && f_short != 42) { throw new RuntimeException("CAS failed"); } } diff --git a/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestBoolean.java b/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestBoolean.java index 2a4015daca209..d45ce53225598 100644 --- a/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestBoolean.java +++ b/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestBoolean.java @@ -64,6 +64,10 @@ public class JdkInternalMiscUnsafeAccessTestBoolean { static int ARRAY_SHIFT; + // copies of Unsafe.MO_RELEASE and other MO constants + static byte UNSAFE_MO_RELEASE, UNSAFE_MO_ACQUIRE, UNSAFE_MO_OPAQUE, UNSAFE_MO_VOLATILE; + static byte UNSAFE_MO_WEAK_CAS_ACQUIRE, UNSAFE_MO_WEAK_CAS_PLAIN, UNSAFE_MO_WEAK_CAS_RELEASE, UNSAFE_MO_WEAK_CAS_VOLATILE; + static { try { Field f = jdk.internal.misc.Unsafe.class.getDeclaredField("theUnsafe"); @@ -91,6 +95,18 @@ public class JdkInternalMiscUnsafeAccessTestBoolean { ARRAY_OFFSET = UNSAFE.arrayBaseOffset(boolean[].class); int ascale = UNSAFE.arrayIndexScale(boolean[].class); ARRAY_SHIFT = 31 - Integer.numberOfLeadingZeros(ascale); + + try { + for (Field umoField : JdkInternalMiscUnsafeAccessTestBoolean.class.getDeclaredFields()) { + String name = umoField.getName(); + if (!name.startsWith("UNSAFE_MO_")) continue; + Field moField = UNSAFE.getClass().getDeclaredField(name.substring("UNSAFE_".length())); + byte value = (Byte) moField.get(null); + umoField.set(null, value); + } + } catch (Exception e) { + throw new RuntimeException(e); + } } static void weakDelay() { @@ -151,15 +167,15 @@ static void testAccess(Object base, long offset) { // Lazy { - UNSAFE.putBooleanRelease(base, offset, true); - boolean x = UNSAFE.getBooleanAcquire(base, offset); + UNSAFE.putBooleanMO(UNSAFE_MO_RELEASE, base, offset, true); + boolean x = UNSAFE.getBooleanMO(UNSAFE_MO_ACQUIRE, base, offset); assertEquals(x, true, "putRelease boolean value"); } // Opaque { - UNSAFE.putBooleanOpaque(base, offset, false); - boolean x = UNSAFE.getBooleanOpaque(base, offset); + UNSAFE.putBooleanMO(UNSAFE_MO_OPAQUE, base, offset, false); + boolean x = UNSAFE.getBooleanMO(UNSAFE_MO_OPAQUE, base, offset); assertEquals(x, false, "putOpaque boolean value"); } @@ -197,28 +213,28 @@ static void testAccess(Object base, long offset) { } { - boolean r = UNSAFE.compareAndExchangeBooleanAcquire(base, offset, true, false); + boolean r = UNSAFE.compareAndExchangeBooleanMO(UNSAFE_MO_ACQUIRE, base, offset, true, false); assertEquals(r, true, "success compareAndExchangeAcquire boolean"); boolean x = UNSAFE.getBoolean(base, offset); assertEquals(x, false, "success compareAndExchangeAcquire boolean value"); } { - boolean r = UNSAFE.compareAndExchangeBooleanAcquire(base, offset, true, false); + boolean r = UNSAFE.compareAndExchangeBooleanMO(UNSAFE_MO_ACQUIRE, base, offset, true, false); assertEquals(r, false, "failing compareAndExchangeAcquire boolean"); boolean x = UNSAFE.getBoolean(base, offset); assertEquals(x, false, "failing compareAndExchangeAcquire boolean value"); } { - boolean r = UNSAFE.compareAndExchangeBooleanRelease(base, offset, false, true); + boolean r = UNSAFE.compareAndExchangeBooleanMO(UNSAFE_MO_RELEASE, base, offset, false, true); assertEquals(r, false, "success compareAndExchangeRelease boolean"); boolean x = UNSAFE.getBoolean(base, offset); assertEquals(x, true, "success compareAndExchangeRelease boolean value"); } { - boolean r = UNSAFE.compareAndExchangeBooleanRelease(base, offset, false, false); + boolean r = UNSAFE.compareAndExchangeBooleanMO(UNSAFE_MO_RELEASE, base, offset, false, false); assertEquals(r, true, "failing compareAndExchangeRelease boolean"); boolean x = UNSAFE.getBoolean(base, offset); assertEquals(x, true, "failing compareAndExchangeRelease boolean value"); @@ -227,7 +243,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetBooleanPlain(base, offset, true, false); + success = UNSAFE.compareAndSetBooleanMO(UNSAFE_MO_WEAK_CAS_PLAIN, base, offset, true, false); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSetPlain boolean"); @@ -236,7 +252,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetBooleanPlain(base, offset, true, false); + boolean success = UNSAFE.compareAndSetBooleanMO(UNSAFE_MO_WEAK_CAS_PLAIN, base, offset, true, false); assertEquals(success, false, "failing weakCompareAndSetPlain boolean"); boolean x = UNSAFE.getBoolean(base, offset); assertEquals(x, false, "failing weakCompareAndSetPlain boolean value"); @@ -245,7 +261,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetBooleanAcquire(base, offset, false, true); + success = UNSAFE.compareAndSetBooleanMO(UNSAFE_MO_WEAK_CAS_ACQUIRE, base, offset, false, true); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSetAcquire boolean"); @@ -254,7 +270,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetBooleanAcquire(base, offset, false, false); + boolean success = UNSAFE.compareAndSetBooleanMO(UNSAFE_MO_WEAK_CAS_ACQUIRE, base, offset, false, false); assertEquals(success, false, "failing weakCompareAndSetAcquire boolean"); boolean x = UNSAFE.getBoolean(base, offset); assertEquals(x, true, "failing weakCompareAndSetAcquire boolean value"); @@ -263,7 +279,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetBooleanRelease(base, offset, true, false); + success = UNSAFE.compareAndSetBooleanMO(UNSAFE_MO_WEAK_CAS_RELEASE, base, offset, true, false); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSetRelease boolean"); @@ -272,7 +288,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetBooleanRelease(base, offset, true, false); + boolean success = UNSAFE.compareAndSetBooleanMO(UNSAFE_MO_WEAK_CAS_RELEASE, base, offset, true, false); assertEquals(success, false, "failing weakCompareAndSetRelease boolean"); boolean x = UNSAFE.getBoolean(base, offset); assertEquals(x, false, "failing weakCompareAndSetRelease boolean value"); @@ -281,7 +297,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetBoolean(base, offset, false, true); + success = UNSAFE.compareAndSetBooleanMO(UNSAFE_MO_WEAK_CAS_VOLATILE, base, offset, false, true); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSet boolean"); @@ -290,7 +306,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetBoolean(base, offset, false, false); + boolean success = UNSAFE.compareAndSetBooleanMO(UNSAFE_MO_WEAK_CAS_VOLATILE, base, offset, false, false); assertEquals(success, false, "failing weakCompareAndSet boolean"); boolean x = UNSAFE.getBoolean(base, offset); assertEquals(x, true, "failing weakCompareAndSet boolean value"); diff --git a/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestByte.java b/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestByte.java index 8b188cb7cb52e..23d29960438f6 100644 --- a/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestByte.java +++ b/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestByte.java @@ -64,6 +64,10 @@ public class JdkInternalMiscUnsafeAccessTestByte { static int ARRAY_SHIFT; + // copies of Unsafe.MO_RELEASE and other MO constants + static byte UNSAFE_MO_RELEASE, UNSAFE_MO_ACQUIRE, UNSAFE_MO_OPAQUE, UNSAFE_MO_VOLATILE; + static byte UNSAFE_MO_WEAK_CAS_ACQUIRE, UNSAFE_MO_WEAK_CAS_PLAIN, UNSAFE_MO_WEAK_CAS_RELEASE, UNSAFE_MO_WEAK_CAS_VOLATILE; + static { try { Field f = jdk.internal.misc.Unsafe.class.getDeclaredField("theUnsafe"); @@ -91,6 +95,18 @@ public class JdkInternalMiscUnsafeAccessTestByte { ARRAY_OFFSET = UNSAFE.arrayBaseOffset(byte[].class); int ascale = UNSAFE.arrayIndexScale(byte[].class); ARRAY_SHIFT = 31 - Integer.numberOfLeadingZeros(ascale); + + try { + for (Field umoField : JdkInternalMiscUnsafeAccessTestByte.class.getDeclaredFields()) { + String name = umoField.getName(); + if (!name.startsWith("UNSAFE_MO_")) continue; + Field moField = UNSAFE.getClass().getDeclaredField(name.substring("UNSAFE_".length())); + byte value = (Byte) moField.get(null); + umoField.set(null, value); + } + } catch (Exception e) { + throw new RuntimeException(e); + } } static void weakDelay() { @@ -180,15 +196,15 @@ static void testAccess(Object base, long offset) { // Lazy { - UNSAFE.putByteRelease(base, offset, (byte)0x01); - byte x = UNSAFE.getByteAcquire(base, offset); + UNSAFE.putByteMO(UNSAFE_MO_RELEASE, base, offset, (byte)0x01); + byte x = UNSAFE.getByteMO(UNSAFE_MO_ACQUIRE, base, offset); assertEquals(x, (byte)0x01, "putRelease byte value"); } // Opaque { - UNSAFE.putByteOpaque(base, offset, (byte)0x23); - byte x = UNSAFE.getByteOpaque(base, offset); + UNSAFE.putByteMO(UNSAFE_MO_OPAQUE, base, offset, (byte)0x23); + byte x = UNSAFE.getByteMO(UNSAFE_MO_OPAQUE, base, offset); assertEquals(x, (byte)0x23, "putOpaque byte value"); } @@ -226,28 +242,28 @@ static void testAccess(Object base, long offset) { } { - byte r = UNSAFE.compareAndExchangeByteAcquire(base, offset, (byte)0x01, (byte)0x23); + byte r = UNSAFE.compareAndExchangeByteMO(UNSAFE_MO_ACQUIRE, base, offset, (byte)0x01, (byte)0x23); assertEquals(r, (byte)0x01, "success compareAndExchangeAcquire byte"); byte x = UNSAFE.getByte(base, offset); assertEquals(x, (byte)0x23, "success compareAndExchangeAcquire byte value"); } { - byte r = UNSAFE.compareAndExchangeByteAcquire(base, offset, (byte)0x01, (byte)0x45); + byte r = UNSAFE.compareAndExchangeByteMO(UNSAFE_MO_ACQUIRE, base, offset, (byte)0x01, (byte)0x45); assertEquals(r, (byte)0x23, "failing compareAndExchangeAcquire byte"); byte x = UNSAFE.getByte(base, offset); assertEquals(x, (byte)0x23, "failing compareAndExchangeAcquire byte value"); } { - byte r = UNSAFE.compareAndExchangeByteRelease(base, offset, (byte)0x23, (byte)0x01); + byte r = UNSAFE.compareAndExchangeByteMO(UNSAFE_MO_RELEASE, base, offset, (byte)0x23, (byte)0x01); assertEquals(r, (byte)0x23, "success compareAndExchangeRelease byte"); byte x = UNSAFE.getByte(base, offset); assertEquals(x, (byte)0x01, "success compareAndExchangeRelease byte value"); } { - byte r = UNSAFE.compareAndExchangeByteRelease(base, offset, (byte)0x23, (byte)0x45); + byte r = UNSAFE.compareAndExchangeByteMO(UNSAFE_MO_RELEASE, base, offset, (byte)0x23, (byte)0x45); assertEquals(r, (byte)0x01, "failing compareAndExchangeRelease byte"); byte x = UNSAFE.getByte(base, offset); assertEquals(x, (byte)0x01, "failing compareAndExchangeRelease byte value"); @@ -256,7 +272,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetBytePlain(base, offset, (byte)0x01, (byte)0x23); + success = UNSAFE.compareAndSetByteMO(UNSAFE_MO_WEAK_CAS_PLAIN, base, offset, (byte)0x01, (byte)0x23); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSetPlain byte"); @@ -265,7 +281,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetBytePlain(base, offset, (byte)0x01, (byte)0x45); + boolean success = UNSAFE.compareAndSetByteMO(UNSAFE_MO_WEAK_CAS_PLAIN, base, offset, (byte)0x01, (byte)0x45); assertEquals(success, false, "failing weakCompareAndSetPlain byte"); byte x = UNSAFE.getByte(base, offset); assertEquals(x, (byte)0x23, "failing weakCompareAndSetPlain byte value"); @@ -274,7 +290,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetByteAcquire(base, offset, (byte)0x23, (byte)0x01); + success = UNSAFE.compareAndSetByteMO(UNSAFE_MO_WEAK_CAS_ACQUIRE, base, offset, (byte)0x23, (byte)0x01); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSetAcquire byte"); @@ -283,7 +299,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetByteAcquire(base, offset, (byte)0x23, (byte)0x45); + boolean success = UNSAFE.compareAndSetByteMO(UNSAFE_MO_WEAK_CAS_ACQUIRE, base, offset, (byte)0x23, (byte)0x45); assertEquals(success, false, "failing weakCompareAndSetAcquire byte"); byte x = UNSAFE.getByte(base, offset); assertEquals(x, (byte)0x01, "failing weakCompareAndSetAcquire byte value"); @@ -292,7 +308,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetByteRelease(base, offset, (byte)0x01, (byte)0x23); + success = UNSAFE.compareAndSetByteMO(UNSAFE_MO_WEAK_CAS_RELEASE, base, offset, (byte)0x01, (byte)0x23); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSetRelease byte"); @@ -301,7 +317,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetByteRelease(base, offset, (byte)0x01, (byte)0x45); + boolean success = UNSAFE.compareAndSetByteMO(UNSAFE_MO_WEAK_CAS_RELEASE, base, offset, (byte)0x01, (byte)0x45); assertEquals(success, false, "failing weakCompareAndSetRelease byte"); byte x = UNSAFE.getByte(base, offset); assertEquals(x, (byte)0x23, "failing weakCompareAndSetRelease byte value"); @@ -310,7 +326,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetByte(base, offset, (byte)0x23, (byte)0x01); + success = UNSAFE.compareAndSetByteMO(UNSAFE_MO_WEAK_CAS_VOLATILE, base, offset, (byte)0x23, (byte)0x01); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSet byte"); @@ -319,7 +335,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetByte(base, offset, (byte)0x23, (byte)0x45); + boolean success = UNSAFE.compareAndSetByteMO(UNSAFE_MO_WEAK_CAS_VOLATILE, base, offset, (byte)0x23, (byte)0x45); assertEquals(success, false, "failing weakCompareAndSet byte"); byte x = UNSAFE.getByte(base, offset); assertEquals(x, (byte)0x01, "failing weakCompareAndSet byte value"); diff --git a/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestChar.java b/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestChar.java index 4b5b876d50720..99982e1221d96 100644 --- a/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestChar.java +++ b/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestChar.java @@ -64,6 +64,10 @@ public class JdkInternalMiscUnsafeAccessTestChar { static int ARRAY_SHIFT; + // copies of Unsafe.MO_RELEASE and other MO constants + static byte UNSAFE_MO_RELEASE, UNSAFE_MO_ACQUIRE, UNSAFE_MO_OPAQUE, UNSAFE_MO_VOLATILE; + static byte UNSAFE_MO_WEAK_CAS_ACQUIRE, UNSAFE_MO_WEAK_CAS_PLAIN, UNSAFE_MO_WEAK_CAS_RELEASE, UNSAFE_MO_WEAK_CAS_VOLATILE; + static { try { Field f = jdk.internal.misc.Unsafe.class.getDeclaredField("theUnsafe"); @@ -91,6 +95,18 @@ public class JdkInternalMiscUnsafeAccessTestChar { ARRAY_OFFSET = UNSAFE.arrayBaseOffset(char[].class); int ascale = UNSAFE.arrayIndexScale(char[].class); ARRAY_SHIFT = 31 - Integer.numberOfLeadingZeros(ascale); + + try { + for (Field umoField : JdkInternalMiscUnsafeAccessTestChar.class.getDeclaredFields()) { + String name = umoField.getName(); + if (!name.startsWith("UNSAFE_MO_")) continue; + Field moField = UNSAFE.getClass().getDeclaredField(name.substring("UNSAFE_".length())); + byte value = (Byte) moField.get(null); + umoField.set(null, value); + } + } catch (Exception e) { + throw new RuntimeException(e); + } } static void weakDelay() { @@ -180,15 +196,15 @@ static void testAccess(Object base, long offset) { // Lazy { - UNSAFE.putCharRelease(base, offset, '\u0123'); - char x = UNSAFE.getCharAcquire(base, offset); + UNSAFE.putCharMO(UNSAFE_MO_RELEASE, base, offset, '\u0123'); + char x = UNSAFE.getCharMO(UNSAFE_MO_ACQUIRE, base, offset); assertEquals(x, '\u0123', "putRelease char value"); } // Opaque { - UNSAFE.putCharOpaque(base, offset, '\u4567'); - char x = UNSAFE.getCharOpaque(base, offset); + UNSAFE.putCharMO(UNSAFE_MO_OPAQUE, base, offset, '\u4567'); + char x = UNSAFE.getCharMO(UNSAFE_MO_OPAQUE, base, offset); assertEquals(x, '\u4567', "putOpaque char value"); } @@ -244,28 +260,28 @@ static void testAccess(Object base, long offset) { } { - char r = UNSAFE.compareAndExchangeCharAcquire(base, offset, '\u0123', '\u4567'); + char r = UNSAFE.compareAndExchangeCharMO(UNSAFE_MO_ACQUIRE, base, offset, '\u0123', '\u4567'); assertEquals(r, '\u0123', "success compareAndExchangeAcquire char"); char x = UNSAFE.getChar(base, offset); assertEquals(x, '\u4567', "success compareAndExchangeAcquire char value"); } { - char r = UNSAFE.compareAndExchangeCharAcquire(base, offset, '\u0123', '\u89AB'); + char r = UNSAFE.compareAndExchangeCharMO(UNSAFE_MO_ACQUIRE, base, offset, '\u0123', '\u89AB'); assertEquals(r, '\u4567', "failing compareAndExchangeAcquire char"); char x = UNSAFE.getChar(base, offset); assertEquals(x, '\u4567', "failing compareAndExchangeAcquire char value"); } { - char r = UNSAFE.compareAndExchangeCharRelease(base, offset, '\u4567', '\u0123'); + char r = UNSAFE.compareAndExchangeCharMO(UNSAFE_MO_RELEASE, base, offset, '\u4567', '\u0123'); assertEquals(r, '\u4567', "success compareAndExchangeRelease char"); char x = UNSAFE.getChar(base, offset); assertEquals(x, '\u0123', "success compareAndExchangeRelease char value"); } { - char r = UNSAFE.compareAndExchangeCharRelease(base, offset, '\u4567', '\u89AB'); + char r = UNSAFE.compareAndExchangeCharMO(UNSAFE_MO_RELEASE, base, offset, '\u4567', '\u89AB'); assertEquals(r, '\u0123', "failing compareAndExchangeRelease char"); char x = UNSAFE.getChar(base, offset); assertEquals(x, '\u0123', "failing compareAndExchangeRelease char value"); @@ -274,7 +290,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetCharPlain(base, offset, '\u0123', '\u4567'); + success = UNSAFE.compareAndSetCharMO(UNSAFE_MO_WEAK_CAS_PLAIN, base, offset, '\u0123', '\u4567'); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSetPlain char"); @@ -283,7 +299,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetCharPlain(base, offset, '\u0123', '\u89AB'); + boolean success = UNSAFE.compareAndSetCharMO(UNSAFE_MO_WEAK_CAS_PLAIN, base, offset, '\u0123', '\u89AB'); assertEquals(success, false, "failing weakCompareAndSetPlain char"); char x = UNSAFE.getChar(base, offset); assertEquals(x, '\u4567', "failing weakCompareAndSetPlain char value"); @@ -292,7 +308,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetCharAcquire(base, offset, '\u4567', '\u0123'); + success = UNSAFE.compareAndSetCharMO(UNSAFE_MO_WEAK_CAS_ACQUIRE, base, offset, '\u4567', '\u0123'); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSetAcquire char"); @@ -301,7 +317,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetCharAcquire(base, offset, '\u4567', '\u89AB'); + boolean success = UNSAFE.compareAndSetCharMO(UNSAFE_MO_WEAK_CAS_ACQUIRE, base, offset, '\u4567', '\u89AB'); assertEquals(success, false, "failing weakCompareAndSetAcquire char"); char x = UNSAFE.getChar(base, offset); assertEquals(x, '\u0123', "failing weakCompareAndSetAcquire char value"); @@ -310,7 +326,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetCharRelease(base, offset, '\u0123', '\u4567'); + success = UNSAFE.compareAndSetCharMO(UNSAFE_MO_WEAK_CAS_RELEASE, base, offset, '\u0123', '\u4567'); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSetRelease char"); @@ -319,7 +335,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetCharRelease(base, offset, '\u0123', '\u89AB'); + boolean success = UNSAFE.compareAndSetCharMO(UNSAFE_MO_WEAK_CAS_RELEASE, base, offset, '\u0123', '\u89AB'); assertEquals(success, false, "failing weakCompareAndSetRelease char"); char x = UNSAFE.getChar(base, offset); assertEquals(x, '\u4567', "failing weakCompareAndSetRelease char value"); @@ -328,7 +344,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetChar(base, offset, '\u4567', '\u0123'); + success = UNSAFE.compareAndSetCharMO(UNSAFE_MO_WEAK_CAS_VOLATILE, base, offset, '\u4567', '\u0123'); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSet char"); @@ -337,7 +353,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetChar(base, offset, '\u4567', '\u89AB'); + boolean success = UNSAFE.compareAndSetCharMO(UNSAFE_MO_WEAK_CAS_VOLATILE, base, offset, '\u4567', '\u89AB'); assertEquals(success, false, "failing weakCompareAndSet char"); char x = UNSAFE.getChar(base, offset); assertEquals(x, '\u0123', "failing weakCompareAndSet char value"); diff --git a/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestDouble.java b/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestDouble.java index 5334da98714d0..48fcdb2c3d484 100644 --- a/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestDouble.java +++ b/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestDouble.java @@ -64,6 +64,10 @@ public class JdkInternalMiscUnsafeAccessTestDouble { static int ARRAY_SHIFT; + // copies of Unsafe.MO_RELEASE and other MO constants + static byte UNSAFE_MO_RELEASE, UNSAFE_MO_ACQUIRE, UNSAFE_MO_OPAQUE, UNSAFE_MO_VOLATILE; + static byte UNSAFE_MO_WEAK_CAS_ACQUIRE, UNSAFE_MO_WEAK_CAS_PLAIN, UNSAFE_MO_WEAK_CAS_RELEASE, UNSAFE_MO_WEAK_CAS_VOLATILE; + static { try { Field f = jdk.internal.misc.Unsafe.class.getDeclaredField("theUnsafe"); @@ -91,6 +95,18 @@ public class JdkInternalMiscUnsafeAccessTestDouble { ARRAY_OFFSET = UNSAFE.arrayBaseOffset(double[].class); int ascale = UNSAFE.arrayIndexScale(double[].class); ARRAY_SHIFT = 31 - Integer.numberOfLeadingZeros(ascale); + + try { + for (Field umoField : JdkInternalMiscUnsafeAccessTestDouble.class.getDeclaredFields()) { + String name = umoField.getName(); + if (!name.startsWith("UNSAFE_MO_")) continue; + Field moField = UNSAFE.getClass().getDeclaredField(name.substring("UNSAFE_".length())); + byte value = (Byte) moField.get(null); + umoField.set(null, value); + } + } catch (Exception e) { + throw new RuntimeException(e); + } } static void weakDelay() { @@ -180,15 +196,15 @@ static void testAccess(Object base, long offset) { // Lazy { - UNSAFE.putDoubleRelease(base, offset, 1.0d); - double x = UNSAFE.getDoubleAcquire(base, offset); + UNSAFE.putDoubleMO(UNSAFE_MO_RELEASE, base, offset, 1.0d); + double x = UNSAFE.getDoubleMO(UNSAFE_MO_ACQUIRE, base, offset); assertEquals(x, 1.0d, "putRelease double value"); } // Opaque { - UNSAFE.putDoubleOpaque(base, offset, 2.0d); - double x = UNSAFE.getDoubleOpaque(base, offset); + UNSAFE.putDoubleMO(UNSAFE_MO_OPAQUE, base, offset, 2.0d); + double x = UNSAFE.getDoubleMO(UNSAFE_MO_OPAQUE, base, offset); assertEquals(x, 2.0d, "putOpaque double value"); } @@ -226,28 +242,28 @@ static void testAccess(Object base, long offset) { } { - double r = UNSAFE.compareAndExchangeDoubleAcquire(base, offset, 1.0d, 2.0d); + double r = UNSAFE.compareAndExchangeDoubleMO(UNSAFE_MO_ACQUIRE, base, offset, 1.0d, 2.0d); assertEquals(r, 1.0d, "success compareAndExchangeAcquire double"); double x = UNSAFE.getDouble(base, offset); assertEquals(x, 2.0d, "success compareAndExchangeAcquire double value"); } { - double r = UNSAFE.compareAndExchangeDoubleAcquire(base, offset, 1.0d, 3.0d); + double r = UNSAFE.compareAndExchangeDoubleMO(UNSAFE_MO_ACQUIRE, base, offset, 1.0d, 3.0d); assertEquals(r, 2.0d, "failing compareAndExchangeAcquire double"); double x = UNSAFE.getDouble(base, offset); assertEquals(x, 2.0d, "failing compareAndExchangeAcquire double value"); } { - double r = UNSAFE.compareAndExchangeDoubleRelease(base, offset, 2.0d, 1.0d); + double r = UNSAFE.compareAndExchangeDoubleMO(UNSAFE_MO_RELEASE, base, offset, 2.0d, 1.0d); assertEquals(r, 2.0d, "success compareAndExchangeRelease double"); double x = UNSAFE.getDouble(base, offset); assertEquals(x, 1.0d, "success compareAndExchangeRelease double value"); } { - double r = UNSAFE.compareAndExchangeDoubleRelease(base, offset, 2.0d, 3.0d); + double r = UNSAFE.compareAndExchangeDoubleMO(UNSAFE_MO_RELEASE, base, offset, 2.0d, 3.0d); assertEquals(r, 1.0d, "failing compareAndExchangeRelease double"); double x = UNSAFE.getDouble(base, offset); assertEquals(x, 1.0d, "failing compareAndExchangeRelease double value"); @@ -256,7 +272,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetDoublePlain(base, offset, 1.0d, 2.0d); + success = UNSAFE.compareAndSetDoubleMO(UNSAFE_MO_WEAK_CAS_PLAIN, base, offset, 1.0d, 2.0d); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSetPlain double"); @@ -265,7 +281,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetDoublePlain(base, offset, 1.0d, 3.0d); + boolean success = UNSAFE.compareAndSetDoubleMO(UNSAFE_MO_WEAK_CAS_PLAIN, base, offset, 1.0d, 3.0d); assertEquals(success, false, "failing weakCompareAndSetPlain double"); double x = UNSAFE.getDouble(base, offset); assertEquals(x, 2.0d, "failing weakCompareAndSetPlain double value"); @@ -274,7 +290,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetDoubleAcquire(base, offset, 2.0d, 1.0d); + success = UNSAFE.compareAndSetDoubleMO(UNSAFE_MO_WEAK_CAS_ACQUIRE, base, offset, 2.0d, 1.0d); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSetAcquire double"); @@ -283,7 +299,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetDoubleAcquire(base, offset, 2.0d, 3.0d); + boolean success = UNSAFE.compareAndSetDoubleMO(UNSAFE_MO_WEAK_CAS_ACQUIRE, base, offset, 2.0d, 3.0d); assertEquals(success, false, "failing weakCompareAndSetAcquire double"); double x = UNSAFE.getDouble(base, offset); assertEquals(x, 1.0d, "failing weakCompareAndSetAcquire double value"); @@ -292,7 +308,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetDoubleRelease(base, offset, 1.0d, 2.0d); + success = UNSAFE.compareAndSetDoubleMO(UNSAFE_MO_WEAK_CAS_RELEASE, base, offset, 1.0d, 2.0d); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSetRelease double"); @@ -301,7 +317,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetDoubleRelease(base, offset, 1.0d, 3.0d); + boolean success = UNSAFE.compareAndSetDoubleMO(UNSAFE_MO_WEAK_CAS_RELEASE, base, offset, 1.0d, 3.0d); assertEquals(success, false, "failing weakCompareAndSetRelease double"); double x = UNSAFE.getDouble(base, offset); assertEquals(x, 2.0d, "failing weakCompareAndSetRelease double value"); @@ -310,7 +326,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetDouble(base, offset, 2.0d, 1.0d); + success = UNSAFE.compareAndSetDoubleMO(UNSAFE_MO_WEAK_CAS_VOLATILE, base, offset, 2.0d, 1.0d); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSet double"); @@ -319,7 +335,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetDouble(base, offset, 2.0d, 3.0d); + boolean success = UNSAFE.compareAndSetDoubleMO(UNSAFE_MO_WEAK_CAS_VOLATILE, base, offset, 2.0d, 3.0d); assertEquals(success, false, "failing weakCompareAndSet double"); double x = UNSAFE.getDouble(base, offset); assertEquals(x, 1.0d, "failing weakCompareAndSet double value"); diff --git a/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestFloat.java b/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestFloat.java index 946f6d7d5dede..12e7007095ac8 100644 --- a/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestFloat.java +++ b/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestFloat.java @@ -64,6 +64,10 @@ public class JdkInternalMiscUnsafeAccessTestFloat { static int ARRAY_SHIFT; + // copies of Unsafe.MO_RELEASE and other MO constants + static byte UNSAFE_MO_RELEASE, UNSAFE_MO_ACQUIRE, UNSAFE_MO_OPAQUE, UNSAFE_MO_VOLATILE; + static byte UNSAFE_MO_WEAK_CAS_ACQUIRE, UNSAFE_MO_WEAK_CAS_PLAIN, UNSAFE_MO_WEAK_CAS_RELEASE, UNSAFE_MO_WEAK_CAS_VOLATILE; + static { try { Field f = jdk.internal.misc.Unsafe.class.getDeclaredField("theUnsafe"); @@ -91,6 +95,18 @@ public class JdkInternalMiscUnsafeAccessTestFloat { ARRAY_OFFSET = UNSAFE.arrayBaseOffset(float[].class); int ascale = UNSAFE.arrayIndexScale(float[].class); ARRAY_SHIFT = 31 - Integer.numberOfLeadingZeros(ascale); + + try { + for (Field umoField : JdkInternalMiscUnsafeAccessTestFloat.class.getDeclaredFields()) { + String name = umoField.getName(); + if (!name.startsWith("UNSAFE_MO_")) continue; + Field moField = UNSAFE.getClass().getDeclaredField(name.substring("UNSAFE_".length())); + byte value = (Byte) moField.get(null); + umoField.set(null, value); + } + } catch (Exception e) { + throw new RuntimeException(e); + } } static void weakDelay() { @@ -180,15 +196,15 @@ static void testAccess(Object base, long offset) { // Lazy { - UNSAFE.putFloatRelease(base, offset, 1.0f); - float x = UNSAFE.getFloatAcquire(base, offset); + UNSAFE.putFloatMO(UNSAFE_MO_RELEASE, base, offset, 1.0f); + float x = UNSAFE.getFloatMO(UNSAFE_MO_ACQUIRE, base, offset); assertEquals(x, 1.0f, "putRelease float value"); } // Opaque { - UNSAFE.putFloatOpaque(base, offset, 2.0f); - float x = UNSAFE.getFloatOpaque(base, offset); + UNSAFE.putFloatMO(UNSAFE_MO_OPAQUE, base, offset, 2.0f); + float x = UNSAFE.getFloatMO(UNSAFE_MO_OPAQUE, base, offset); assertEquals(x, 2.0f, "putOpaque float value"); } @@ -226,28 +242,28 @@ static void testAccess(Object base, long offset) { } { - float r = UNSAFE.compareAndExchangeFloatAcquire(base, offset, 1.0f, 2.0f); + float r = UNSAFE.compareAndExchangeFloatMO(UNSAFE_MO_ACQUIRE, base, offset, 1.0f, 2.0f); assertEquals(r, 1.0f, "success compareAndExchangeAcquire float"); float x = UNSAFE.getFloat(base, offset); assertEquals(x, 2.0f, "success compareAndExchangeAcquire float value"); } { - float r = UNSAFE.compareAndExchangeFloatAcquire(base, offset, 1.0f, 3.0f); + float r = UNSAFE.compareAndExchangeFloatMO(UNSAFE_MO_ACQUIRE, base, offset, 1.0f, 3.0f); assertEquals(r, 2.0f, "failing compareAndExchangeAcquire float"); float x = UNSAFE.getFloat(base, offset); assertEquals(x, 2.0f, "failing compareAndExchangeAcquire float value"); } { - float r = UNSAFE.compareAndExchangeFloatRelease(base, offset, 2.0f, 1.0f); + float r = UNSAFE.compareAndExchangeFloatMO(UNSAFE_MO_RELEASE, base, offset, 2.0f, 1.0f); assertEquals(r, 2.0f, "success compareAndExchangeRelease float"); float x = UNSAFE.getFloat(base, offset); assertEquals(x, 1.0f, "success compareAndExchangeRelease float value"); } { - float r = UNSAFE.compareAndExchangeFloatRelease(base, offset, 2.0f, 3.0f); + float r = UNSAFE.compareAndExchangeFloatMO(UNSAFE_MO_RELEASE, base, offset, 2.0f, 3.0f); assertEquals(r, 1.0f, "failing compareAndExchangeRelease float"); float x = UNSAFE.getFloat(base, offset); assertEquals(x, 1.0f, "failing compareAndExchangeRelease float value"); @@ -256,7 +272,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetFloatPlain(base, offset, 1.0f, 2.0f); + success = UNSAFE.compareAndSetFloatMO(UNSAFE_MO_WEAK_CAS_PLAIN, base, offset, 1.0f, 2.0f); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSetPlain float"); @@ -265,7 +281,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetFloatPlain(base, offset, 1.0f, 3.0f); + boolean success = UNSAFE.compareAndSetFloatMO(UNSAFE_MO_WEAK_CAS_PLAIN, base, offset, 1.0f, 3.0f); assertEquals(success, false, "failing weakCompareAndSetPlain float"); float x = UNSAFE.getFloat(base, offset); assertEquals(x, 2.0f, "failing weakCompareAndSetPlain float value"); @@ -274,7 +290,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetFloatAcquire(base, offset, 2.0f, 1.0f); + success = UNSAFE.compareAndSetFloatMO(UNSAFE_MO_WEAK_CAS_ACQUIRE, base, offset, 2.0f, 1.0f); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSetAcquire float"); @@ -283,7 +299,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetFloatAcquire(base, offset, 2.0f, 3.0f); + boolean success = UNSAFE.compareAndSetFloatMO(UNSAFE_MO_WEAK_CAS_ACQUIRE, base, offset, 2.0f, 3.0f); assertEquals(success, false, "failing weakCompareAndSetAcquire float"); float x = UNSAFE.getFloat(base, offset); assertEquals(x, 1.0f, "failing weakCompareAndSetAcquire float value"); @@ -292,7 +308,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetFloatRelease(base, offset, 1.0f, 2.0f); + success = UNSAFE.compareAndSetFloatMO(UNSAFE_MO_WEAK_CAS_RELEASE, base, offset, 1.0f, 2.0f); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSetRelease float"); @@ -301,7 +317,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetFloatRelease(base, offset, 1.0f, 3.0f); + boolean success = UNSAFE.compareAndSetFloatMO(UNSAFE_MO_WEAK_CAS_RELEASE, base, offset, 1.0f, 3.0f); assertEquals(success, false, "failing weakCompareAndSetRelease float"); float x = UNSAFE.getFloat(base, offset); assertEquals(x, 2.0f, "failing weakCompareAndSetRelease float value"); @@ -310,7 +326,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetFloat(base, offset, 2.0f, 1.0f); + success = UNSAFE.compareAndSetFloatMO(UNSAFE_MO_WEAK_CAS_VOLATILE, base, offset, 2.0f, 1.0f); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSet float"); @@ -319,7 +335,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetFloat(base, offset, 2.0f, 3.0f); + boolean success = UNSAFE.compareAndSetFloatMO(UNSAFE_MO_WEAK_CAS_VOLATILE, base, offset, 2.0f, 3.0f); assertEquals(success, false, "failing weakCompareAndSet float"); float x = UNSAFE.getFloat(base, offset); assertEquals(x, 1.0f, "failing weakCompareAndSet float value"); diff --git a/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestInt.java b/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestInt.java index 93ed85805d5b6..b867eca0e880e 100644 --- a/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestInt.java +++ b/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestInt.java @@ -64,6 +64,10 @@ public class JdkInternalMiscUnsafeAccessTestInt { static int ARRAY_SHIFT; + // copies of Unsafe.MO_RELEASE and other MO constants + static byte UNSAFE_MO_RELEASE, UNSAFE_MO_ACQUIRE, UNSAFE_MO_OPAQUE, UNSAFE_MO_VOLATILE; + static byte UNSAFE_MO_WEAK_CAS_ACQUIRE, UNSAFE_MO_WEAK_CAS_PLAIN, UNSAFE_MO_WEAK_CAS_RELEASE, UNSAFE_MO_WEAK_CAS_VOLATILE; + static { try { Field f = jdk.internal.misc.Unsafe.class.getDeclaredField("theUnsafe"); @@ -91,6 +95,18 @@ public class JdkInternalMiscUnsafeAccessTestInt { ARRAY_OFFSET = UNSAFE.arrayBaseOffset(int[].class); int ascale = UNSAFE.arrayIndexScale(int[].class); ARRAY_SHIFT = 31 - Integer.numberOfLeadingZeros(ascale); + + try { + for (Field umoField : JdkInternalMiscUnsafeAccessTestInt.class.getDeclaredFields()) { + String name = umoField.getName(); + if (!name.startsWith("UNSAFE_MO_")) continue; + Field moField = UNSAFE.getClass().getDeclaredField(name.substring("UNSAFE_".length())); + byte value = (Byte) moField.get(null); + umoField.set(null, value); + } + } catch (Exception e) { + throw new RuntimeException(e); + } } static void weakDelay() { @@ -180,15 +196,15 @@ static void testAccess(Object base, long offset) { // Lazy { - UNSAFE.putIntRelease(base, offset, 0x01234567); - int x = UNSAFE.getIntAcquire(base, offset); + UNSAFE.putIntMO(UNSAFE_MO_RELEASE, base, offset, 0x01234567); + int x = UNSAFE.getIntMO(UNSAFE_MO_ACQUIRE, base, offset); assertEquals(x, 0x01234567, "putRelease int value"); } // Opaque { - UNSAFE.putIntOpaque(base, offset, 0x89ABCDEF); - int x = UNSAFE.getIntOpaque(base, offset); + UNSAFE.putIntMO(UNSAFE_MO_OPAQUE, base, offset, 0x89ABCDEF); + int x = UNSAFE.getIntMO(UNSAFE_MO_OPAQUE, base, offset); assertEquals(x, 0x89ABCDEF, "putOpaque int value"); } @@ -244,28 +260,28 @@ static void testAccess(Object base, long offset) { } { - int r = UNSAFE.compareAndExchangeIntAcquire(base, offset, 0x01234567, 0x89ABCDEF); + int r = UNSAFE.compareAndExchangeIntMO(UNSAFE_MO_ACQUIRE, base, offset, 0x01234567, 0x89ABCDEF); assertEquals(r, 0x01234567, "success compareAndExchangeAcquire int"); int x = UNSAFE.getInt(base, offset); assertEquals(x, 0x89ABCDEF, "success compareAndExchangeAcquire int value"); } { - int r = UNSAFE.compareAndExchangeIntAcquire(base, offset, 0x01234567, 0xCAFEBABE); + int r = UNSAFE.compareAndExchangeIntMO(UNSAFE_MO_ACQUIRE, base, offset, 0x01234567, 0xCAFEBABE); assertEquals(r, 0x89ABCDEF, "failing compareAndExchangeAcquire int"); int x = UNSAFE.getInt(base, offset); assertEquals(x, 0x89ABCDEF, "failing compareAndExchangeAcquire int value"); } { - int r = UNSAFE.compareAndExchangeIntRelease(base, offset, 0x89ABCDEF, 0x01234567); + int r = UNSAFE.compareAndExchangeIntMO(UNSAFE_MO_RELEASE, base, offset, 0x89ABCDEF, 0x01234567); assertEquals(r, 0x89ABCDEF, "success compareAndExchangeRelease int"); int x = UNSAFE.getInt(base, offset); assertEquals(x, 0x01234567, "success compareAndExchangeRelease int value"); } { - int r = UNSAFE.compareAndExchangeIntRelease(base, offset, 0x89ABCDEF, 0xCAFEBABE); + int r = UNSAFE.compareAndExchangeIntMO(UNSAFE_MO_RELEASE, base, offset, 0x89ABCDEF, 0xCAFEBABE); assertEquals(r, 0x01234567, "failing compareAndExchangeRelease int"); int x = UNSAFE.getInt(base, offset); assertEquals(x, 0x01234567, "failing compareAndExchangeRelease int value"); @@ -274,7 +290,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetIntPlain(base, offset, 0x01234567, 0x89ABCDEF); + success = UNSAFE.compareAndSetIntMO(UNSAFE_MO_WEAK_CAS_PLAIN, base, offset, 0x01234567, 0x89ABCDEF); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSetPlain int"); @@ -283,7 +299,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetIntPlain(base, offset, 0x01234567, 0xCAFEBABE); + boolean success = UNSAFE.compareAndSetIntMO(UNSAFE_MO_WEAK_CAS_PLAIN, base, offset, 0x01234567, 0xCAFEBABE); assertEquals(success, false, "failing weakCompareAndSetPlain int"); int x = UNSAFE.getInt(base, offset); assertEquals(x, 0x89ABCDEF, "failing weakCompareAndSetPlain int value"); @@ -292,7 +308,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetIntAcquire(base, offset, 0x89ABCDEF, 0x01234567); + success = UNSAFE.compareAndSetIntMO(UNSAFE_MO_WEAK_CAS_ACQUIRE, base, offset, 0x89ABCDEF, 0x01234567); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSetAcquire int"); @@ -301,7 +317,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetIntAcquire(base, offset, 0x89ABCDEF, 0xCAFEBABE); + boolean success = UNSAFE.compareAndSetIntMO(UNSAFE_MO_WEAK_CAS_ACQUIRE, base, offset, 0x89ABCDEF, 0xCAFEBABE); assertEquals(success, false, "failing weakCompareAndSetAcquire int"); int x = UNSAFE.getInt(base, offset); assertEquals(x, 0x01234567, "failing weakCompareAndSetAcquire int value"); @@ -310,7 +326,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetIntRelease(base, offset, 0x01234567, 0x89ABCDEF); + success = UNSAFE.compareAndSetIntMO(UNSAFE_MO_WEAK_CAS_RELEASE, base, offset, 0x01234567, 0x89ABCDEF); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSetRelease int"); @@ -319,7 +335,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetIntRelease(base, offset, 0x01234567, 0xCAFEBABE); + boolean success = UNSAFE.compareAndSetIntMO(UNSAFE_MO_WEAK_CAS_RELEASE, base, offset, 0x01234567, 0xCAFEBABE); assertEquals(success, false, "failing weakCompareAndSetRelease int"); int x = UNSAFE.getInt(base, offset); assertEquals(x, 0x89ABCDEF, "failing weakCompareAndSetRelease int value"); @@ -328,7 +344,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetInt(base, offset, 0x89ABCDEF, 0x01234567); + success = UNSAFE.compareAndSetIntMO(UNSAFE_MO_WEAK_CAS_VOLATILE, base, offset, 0x89ABCDEF, 0x01234567); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSet int"); @@ -337,7 +353,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetInt(base, offset, 0x89ABCDEF, 0xCAFEBABE); + boolean success = UNSAFE.compareAndSetIntMO(UNSAFE_MO_WEAK_CAS_VOLATILE, base, offset, 0x89ABCDEF, 0xCAFEBABE); assertEquals(success, false, "failing weakCompareAndSet int"); int x = UNSAFE.getInt(base, offset); assertEquals(x, 0x01234567, "failing weakCompareAndSet int value"); diff --git a/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestLong.java b/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestLong.java index 7a6d2bfc033f0..793df4a8e595f 100644 --- a/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestLong.java +++ b/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestLong.java @@ -64,6 +64,10 @@ public class JdkInternalMiscUnsafeAccessTestLong { static int ARRAY_SHIFT; + // copies of Unsafe.MO_RELEASE and other MO constants + static byte UNSAFE_MO_RELEASE, UNSAFE_MO_ACQUIRE, UNSAFE_MO_OPAQUE, UNSAFE_MO_VOLATILE; + static byte UNSAFE_MO_WEAK_CAS_ACQUIRE, UNSAFE_MO_WEAK_CAS_PLAIN, UNSAFE_MO_WEAK_CAS_RELEASE, UNSAFE_MO_WEAK_CAS_VOLATILE; + static { try { Field f = jdk.internal.misc.Unsafe.class.getDeclaredField("theUnsafe"); @@ -91,6 +95,18 @@ public class JdkInternalMiscUnsafeAccessTestLong { ARRAY_OFFSET = UNSAFE.arrayBaseOffset(long[].class); int ascale = UNSAFE.arrayIndexScale(long[].class); ARRAY_SHIFT = 31 - Integer.numberOfLeadingZeros(ascale); + + try { + for (Field umoField : JdkInternalMiscUnsafeAccessTestLong.class.getDeclaredFields()) { + String name = umoField.getName(); + if (!name.startsWith("UNSAFE_MO_")) continue; + Field moField = UNSAFE.getClass().getDeclaredField(name.substring("UNSAFE_".length())); + byte value = (Byte) moField.get(null); + umoField.set(null, value); + } + } catch (Exception e) { + throw new RuntimeException(e); + } } static void weakDelay() { @@ -180,15 +196,15 @@ static void testAccess(Object base, long offset) { // Lazy { - UNSAFE.putLongRelease(base, offset, 0x0123456789ABCDEFL); - long x = UNSAFE.getLongAcquire(base, offset); + UNSAFE.putLongMO(UNSAFE_MO_RELEASE, base, offset, 0x0123456789ABCDEFL); + long x = UNSAFE.getLongMO(UNSAFE_MO_ACQUIRE, base, offset); assertEquals(x, 0x0123456789ABCDEFL, "putRelease long value"); } // Opaque { - UNSAFE.putLongOpaque(base, offset, 0xCAFEBABECAFEBABEL); - long x = UNSAFE.getLongOpaque(base, offset); + UNSAFE.putLongMO(UNSAFE_MO_OPAQUE, base, offset, 0xCAFEBABECAFEBABEL); + long x = UNSAFE.getLongMO(UNSAFE_MO_OPAQUE, base, offset); assertEquals(x, 0xCAFEBABECAFEBABEL, "putOpaque long value"); } @@ -244,28 +260,28 @@ static void testAccess(Object base, long offset) { } { - long r = UNSAFE.compareAndExchangeLongAcquire(base, offset, 0x0123456789ABCDEFL, 0xCAFEBABECAFEBABEL); + long r = UNSAFE.compareAndExchangeLongMO(UNSAFE_MO_ACQUIRE, base, offset, 0x0123456789ABCDEFL, 0xCAFEBABECAFEBABEL); assertEquals(r, 0x0123456789ABCDEFL, "success compareAndExchangeAcquire long"); long x = UNSAFE.getLong(base, offset); assertEquals(x, 0xCAFEBABECAFEBABEL, "success compareAndExchangeAcquire long value"); } { - long r = UNSAFE.compareAndExchangeLongAcquire(base, offset, 0x0123456789ABCDEFL, 0xDEADBEEFDEADBEEFL); + long r = UNSAFE.compareAndExchangeLongMO(UNSAFE_MO_ACQUIRE, base, offset, 0x0123456789ABCDEFL, 0xDEADBEEFDEADBEEFL); assertEquals(r, 0xCAFEBABECAFEBABEL, "failing compareAndExchangeAcquire long"); long x = UNSAFE.getLong(base, offset); assertEquals(x, 0xCAFEBABECAFEBABEL, "failing compareAndExchangeAcquire long value"); } { - long r = UNSAFE.compareAndExchangeLongRelease(base, offset, 0xCAFEBABECAFEBABEL, 0x0123456789ABCDEFL); + long r = UNSAFE.compareAndExchangeLongMO(UNSAFE_MO_RELEASE, base, offset, 0xCAFEBABECAFEBABEL, 0x0123456789ABCDEFL); assertEquals(r, 0xCAFEBABECAFEBABEL, "success compareAndExchangeRelease long"); long x = UNSAFE.getLong(base, offset); assertEquals(x, 0x0123456789ABCDEFL, "success compareAndExchangeRelease long value"); } { - long r = UNSAFE.compareAndExchangeLongRelease(base, offset, 0xCAFEBABECAFEBABEL, 0xDEADBEEFDEADBEEFL); + long r = UNSAFE.compareAndExchangeLongMO(UNSAFE_MO_RELEASE, base, offset, 0xCAFEBABECAFEBABEL, 0xDEADBEEFDEADBEEFL); assertEquals(r, 0x0123456789ABCDEFL, "failing compareAndExchangeRelease long"); long x = UNSAFE.getLong(base, offset); assertEquals(x, 0x0123456789ABCDEFL, "failing compareAndExchangeRelease long value"); @@ -274,7 +290,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetLongPlain(base, offset, 0x0123456789ABCDEFL, 0xCAFEBABECAFEBABEL); + success = UNSAFE.compareAndSetLongMO(UNSAFE_MO_WEAK_CAS_PLAIN, base, offset, 0x0123456789ABCDEFL, 0xCAFEBABECAFEBABEL); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSetPlain long"); @@ -283,7 +299,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetLongPlain(base, offset, 0x0123456789ABCDEFL, 0xDEADBEEFDEADBEEFL); + boolean success = UNSAFE.compareAndSetLongMO(UNSAFE_MO_WEAK_CAS_PLAIN, base, offset, 0x0123456789ABCDEFL, 0xDEADBEEFDEADBEEFL); assertEquals(success, false, "failing weakCompareAndSetPlain long"); long x = UNSAFE.getLong(base, offset); assertEquals(x, 0xCAFEBABECAFEBABEL, "failing weakCompareAndSetPlain long value"); @@ -292,7 +308,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetLongAcquire(base, offset, 0xCAFEBABECAFEBABEL, 0x0123456789ABCDEFL); + success = UNSAFE.compareAndSetLongMO(UNSAFE_MO_WEAK_CAS_ACQUIRE, base, offset, 0xCAFEBABECAFEBABEL, 0x0123456789ABCDEFL); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSetAcquire long"); @@ -301,7 +317,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetLongAcquire(base, offset, 0xCAFEBABECAFEBABEL, 0xDEADBEEFDEADBEEFL); + boolean success = UNSAFE.compareAndSetLongMO(UNSAFE_MO_WEAK_CAS_ACQUIRE, base, offset, 0xCAFEBABECAFEBABEL, 0xDEADBEEFDEADBEEFL); assertEquals(success, false, "failing weakCompareAndSetAcquire long"); long x = UNSAFE.getLong(base, offset); assertEquals(x, 0x0123456789ABCDEFL, "failing weakCompareAndSetAcquire long value"); @@ -310,7 +326,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetLongRelease(base, offset, 0x0123456789ABCDEFL, 0xCAFEBABECAFEBABEL); + success = UNSAFE.compareAndSetLongMO(UNSAFE_MO_WEAK_CAS_RELEASE, base, offset, 0x0123456789ABCDEFL, 0xCAFEBABECAFEBABEL); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSetRelease long"); @@ -319,7 +335,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetLongRelease(base, offset, 0x0123456789ABCDEFL, 0xDEADBEEFDEADBEEFL); + boolean success = UNSAFE.compareAndSetLongMO(UNSAFE_MO_WEAK_CAS_RELEASE, base, offset, 0x0123456789ABCDEFL, 0xDEADBEEFDEADBEEFL); assertEquals(success, false, "failing weakCompareAndSetRelease long"); long x = UNSAFE.getLong(base, offset); assertEquals(x, 0xCAFEBABECAFEBABEL, "failing weakCompareAndSetRelease long value"); @@ -328,7 +344,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetLong(base, offset, 0xCAFEBABECAFEBABEL, 0x0123456789ABCDEFL); + success = UNSAFE.compareAndSetLongMO(UNSAFE_MO_WEAK_CAS_VOLATILE, base, offset, 0xCAFEBABECAFEBABEL, 0x0123456789ABCDEFL); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSet long"); @@ -337,7 +353,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetLong(base, offset, 0xCAFEBABECAFEBABEL, 0xDEADBEEFDEADBEEFL); + boolean success = UNSAFE.compareAndSetLongMO(UNSAFE_MO_WEAK_CAS_VOLATILE, base, offset, 0xCAFEBABECAFEBABEL, 0xDEADBEEFDEADBEEFL); assertEquals(success, false, "failing weakCompareAndSet long"); long x = UNSAFE.getLong(base, offset); assertEquals(x, 0x0123456789ABCDEFL, "failing weakCompareAndSet long value"); diff --git a/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestObject.java b/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestObject.java index 2fe5d7f3ae051..49699fd9f35f3 100644 --- a/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestObject.java +++ b/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestObject.java @@ -64,6 +64,10 @@ public class JdkInternalMiscUnsafeAccessTestObject { static int ARRAY_SHIFT; + // copies of Unsafe.MO_RELEASE and other MO constants + static byte UNSAFE_MO_RELEASE, UNSAFE_MO_ACQUIRE, UNSAFE_MO_OPAQUE, UNSAFE_MO_VOLATILE; + static byte UNSAFE_MO_WEAK_CAS_ACQUIRE, UNSAFE_MO_WEAK_CAS_PLAIN, UNSAFE_MO_WEAK_CAS_RELEASE, UNSAFE_MO_WEAK_CAS_VOLATILE; + static { try { Field f = jdk.internal.misc.Unsafe.class.getDeclaredField("theUnsafe"); @@ -91,6 +95,18 @@ public class JdkInternalMiscUnsafeAccessTestObject { ARRAY_OFFSET = UNSAFE.arrayBaseOffset(Object[].class); int ascale = UNSAFE.arrayIndexScale(Object[].class); ARRAY_SHIFT = 31 - Integer.numberOfLeadingZeros(ascale); + + try { + for (Field umoField : JdkInternalMiscUnsafeAccessTestObject.class.getDeclaredFields()) { + String name = umoField.getName(); + if (!name.startsWith("UNSAFE_MO_")) continue; + Field moField = UNSAFE.getClass().getDeclaredField(name.substring("UNSAFE_".length())); + byte value = (Byte) moField.get(null); + umoField.set(null, value); + } + } catch (Exception e) { + throw new RuntimeException(e); + } } static void weakDelay() { @@ -151,15 +167,15 @@ static void testAccess(Object base, long offset) { // Lazy { - UNSAFE.putReferenceRelease(base, offset, "foo"); - Object x = UNSAFE.getReferenceAcquire(base, offset); + UNSAFE.putReferenceMO(UNSAFE_MO_RELEASE, base, offset, "foo"); + Object x = UNSAFE.getReferenceMO(UNSAFE_MO_ACQUIRE, base, offset); assertEquals(x, "foo", "putRelease Object value"); } // Opaque { - UNSAFE.putReferenceOpaque(base, offset, "bar"); - Object x = UNSAFE.getReferenceOpaque(base, offset); + UNSAFE.putReferenceMO(UNSAFE_MO_OPAQUE, base, offset, "bar"); + Object x = UNSAFE.getReferenceMO(UNSAFE_MO_OPAQUE, base, offset); assertEquals(x, "bar", "putOpaque Object value"); } @@ -197,28 +213,28 @@ static void testAccess(Object base, long offset) { } { - Object r = UNSAFE.compareAndExchangeReferenceAcquire(base, offset, "foo", "bar"); + Object r = UNSAFE.compareAndExchangeReferenceMO(UNSAFE_MO_ACQUIRE, base, offset, "foo", "bar"); assertEquals(r, "foo", "success compareAndExchangeAcquire Object"); Object x = UNSAFE.getReference(base, offset); assertEquals(x, "bar", "success compareAndExchangeAcquire Object value"); } { - Object r = UNSAFE.compareAndExchangeReferenceAcquire(base, offset, "foo", "baz"); + Object r = UNSAFE.compareAndExchangeReferenceMO(UNSAFE_MO_ACQUIRE, base, offset, "foo", "baz"); assertEquals(r, "bar", "failing compareAndExchangeAcquire Object"); Object x = UNSAFE.getReference(base, offset); assertEquals(x, "bar", "failing compareAndExchangeAcquire Object value"); } { - Object r = UNSAFE.compareAndExchangeReferenceRelease(base, offset, "bar", "foo"); + Object r = UNSAFE.compareAndExchangeReferenceMO(UNSAFE_MO_RELEASE, base, offset, "bar", "foo"); assertEquals(r, "bar", "success compareAndExchangeRelease Object"); Object x = UNSAFE.getReference(base, offset); assertEquals(x, "foo", "success compareAndExchangeRelease Object value"); } { - Object r = UNSAFE.compareAndExchangeReferenceRelease(base, offset, "bar", "baz"); + Object r = UNSAFE.compareAndExchangeReferenceMO(UNSAFE_MO_RELEASE, base, offset, "bar", "baz"); assertEquals(r, "foo", "failing compareAndExchangeRelease Object"); Object x = UNSAFE.getReference(base, offset); assertEquals(x, "foo", "failing compareAndExchangeRelease Object value"); @@ -227,7 +243,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetReferencePlain(base, offset, "foo", "bar"); + success = UNSAFE.compareAndSetReferenceMO(UNSAFE_MO_WEAK_CAS_PLAIN, base, offset, "foo", "bar"); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSetPlain Object"); @@ -236,7 +252,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetReferencePlain(base, offset, "foo", "baz"); + boolean success = UNSAFE.compareAndSetReferenceMO(UNSAFE_MO_WEAK_CAS_PLAIN, base, offset, "foo", "baz"); assertEquals(success, false, "failing weakCompareAndSetPlain Object"); Object x = UNSAFE.getReference(base, offset); assertEquals(x, "bar", "failing weakCompareAndSetPlain Object value"); @@ -245,7 +261,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetReferenceAcquire(base, offset, "bar", "foo"); + success = UNSAFE.compareAndSetReferenceMO(UNSAFE_MO_WEAK_CAS_ACQUIRE, base, offset, "bar", "foo"); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSetAcquire Object"); @@ -254,7 +270,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetReferenceAcquire(base, offset, "bar", "baz"); + boolean success = UNSAFE.compareAndSetReferenceMO(UNSAFE_MO_WEAK_CAS_ACQUIRE, base, offset, "bar", "baz"); assertEquals(success, false, "failing weakCompareAndSetAcquire Object"); Object x = UNSAFE.getReference(base, offset); assertEquals(x, "foo", "failing weakCompareAndSetAcquire Object value"); @@ -263,7 +279,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetReferenceRelease(base, offset, "foo", "bar"); + success = UNSAFE.compareAndSetReferenceMO(UNSAFE_MO_WEAK_CAS_RELEASE, base, offset, "foo", "bar"); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSetRelease Object"); @@ -272,7 +288,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetReferenceRelease(base, offset, "foo", "baz"); + boolean success = UNSAFE.compareAndSetReferenceMO(UNSAFE_MO_WEAK_CAS_RELEASE, base, offset, "foo", "baz"); assertEquals(success, false, "failing weakCompareAndSetRelease Object"); Object x = UNSAFE.getReference(base, offset); assertEquals(x, "bar", "failing weakCompareAndSetRelease Object value"); @@ -281,7 +297,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetReference(base, offset, "bar", "foo"); + success = UNSAFE.compareAndSetReferenceMO(UNSAFE_MO_WEAK_CAS_VOLATILE, base, offset, "bar", "foo"); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSet Object"); @@ -290,7 +306,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetReference(base, offset, "bar", "baz"); + boolean success = UNSAFE.compareAndSetReferenceMO(UNSAFE_MO_WEAK_CAS_VOLATILE, base, offset, "bar", "baz"); assertEquals(success, false, "failing weakCompareAndSet Object"); Object x = UNSAFE.getReference(base, offset); assertEquals(x, "foo", "failing weakCompareAndSet Object value"); diff --git a/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestShort.java b/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestShort.java index 47a700d9313c7..205151cf423b4 100644 --- a/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestShort.java +++ b/test/hotspot/jtreg/compiler/unsafe/JdkInternalMiscUnsafeAccessTestShort.java @@ -64,6 +64,10 @@ public class JdkInternalMiscUnsafeAccessTestShort { static int ARRAY_SHIFT; + // copies of Unsafe.MO_RELEASE and other MO constants + static byte UNSAFE_MO_RELEASE, UNSAFE_MO_ACQUIRE, UNSAFE_MO_OPAQUE, UNSAFE_MO_VOLATILE; + static byte UNSAFE_MO_WEAK_CAS_ACQUIRE, UNSAFE_MO_WEAK_CAS_PLAIN, UNSAFE_MO_WEAK_CAS_RELEASE, UNSAFE_MO_WEAK_CAS_VOLATILE; + static { try { Field f = jdk.internal.misc.Unsafe.class.getDeclaredField("theUnsafe"); @@ -91,6 +95,18 @@ public class JdkInternalMiscUnsafeAccessTestShort { ARRAY_OFFSET = UNSAFE.arrayBaseOffset(short[].class); int ascale = UNSAFE.arrayIndexScale(short[].class); ARRAY_SHIFT = 31 - Integer.numberOfLeadingZeros(ascale); + + try { + for (Field umoField : JdkInternalMiscUnsafeAccessTestShort.class.getDeclaredFields()) { + String name = umoField.getName(); + if (!name.startsWith("UNSAFE_MO_")) continue; + Field moField = UNSAFE.getClass().getDeclaredField(name.substring("UNSAFE_".length())); + byte value = (Byte) moField.get(null); + umoField.set(null, value); + } + } catch (Exception e) { + throw new RuntimeException(e); + } } static void weakDelay() { @@ -180,15 +196,15 @@ static void testAccess(Object base, long offset) { // Lazy { - UNSAFE.putShortRelease(base, offset, (short)0x0123); - short x = UNSAFE.getShortAcquire(base, offset); + UNSAFE.putShortMO(UNSAFE_MO_RELEASE, base, offset, (short)0x0123); + short x = UNSAFE.getShortMO(UNSAFE_MO_ACQUIRE, base, offset); assertEquals(x, (short)0x0123, "putRelease short value"); } // Opaque { - UNSAFE.putShortOpaque(base, offset, (short)0x4567); - short x = UNSAFE.getShortOpaque(base, offset); + UNSAFE.putShortMO(UNSAFE_MO_OPAQUE, base, offset, (short)0x4567); + short x = UNSAFE.getShortMO(UNSAFE_MO_OPAQUE, base, offset); assertEquals(x, (short)0x4567, "putOpaque short value"); } @@ -244,28 +260,28 @@ static void testAccess(Object base, long offset) { } { - short r = UNSAFE.compareAndExchangeShortAcquire(base, offset, (short)0x0123, (short)0x4567); + short r = UNSAFE.compareAndExchangeShortMO(UNSAFE_MO_ACQUIRE, base, offset, (short)0x0123, (short)0x4567); assertEquals(r, (short)0x0123, "success compareAndExchangeAcquire short"); short x = UNSAFE.getShort(base, offset); assertEquals(x, (short)0x4567, "success compareAndExchangeAcquire short value"); } { - short r = UNSAFE.compareAndExchangeShortAcquire(base, offset, (short)0x0123, (short)0x89AB); + short r = UNSAFE.compareAndExchangeShortMO(UNSAFE_MO_ACQUIRE, base, offset, (short)0x0123, (short)0x89AB); assertEquals(r, (short)0x4567, "failing compareAndExchangeAcquire short"); short x = UNSAFE.getShort(base, offset); assertEquals(x, (short)0x4567, "failing compareAndExchangeAcquire short value"); } { - short r = UNSAFE.compareAndExchangeShortRelease(base, offset, (short)0x4567, (short)0x0123); + short r = UNSAFE.compareAndExchangeShortMO(UNSAFE_MO_RELEASE, base, offset, (short)0x4567, (short)0x0123); assertEquals(r, (short)0x4567, "success compareAndExchangeRelease short"); short x = UNSAFE.getShort(base, offset); assertEquals(x, (short)0x0123, "success compareAndExchangeRelease short value"); } { - short r = UNSAFE.compareAndExchangeShortRelease(base, offset, (short)0x4567, (short)0x89AB); + short r = UNSAFE.compareAndExchangeShortMO(UNSAFE_MO_RELEASE, base, offset, (short)0x4567, (short)0x89AB); assertEquals(r, (short)0x0123, "failing compareAndExchangeRelease short"); short x = UNSAFE.getShort(base, offset); assertEquals(x, (short)0x0123, "failing compareAndExchangeRelease short value"); @@ -274,7 +290,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetShortPlain(base, offset, (short)0x0123, (short)0x4567); + success = UNSAFE.compareAndSetShortMO(UNSAFE_MO_WEAK_CAS_PLAIN, base, offset, (short)0x0123, (short)0x4567); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSetPlain short"); @@ -283,7 +299,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetShortPlain(base, offset, (short)0x0123, (short)0x89AB); + boolean success = UNSAFE.compareAndSetShortMO(UNSAFE_MO_WEAK_CAS_PLAIN, base, offset, (short)0x0123, (short)0x89AB); assertEquals(success, false, "failing weakCompareAndSetPlain short"); short x = UNSAFE.getShort(base, offset); assertEquals(x, (short)0x4567, "failing weakCompareAndSetPlain short value"); @@ -292,7 +308,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetShortAcquire(base, offset, (short)0x4567, (short)0x0123); + success = UNSAFE.compareAndSetShortMO(UNSAFE_MO_WEAK_CAS_ACQUIRE, base, offset, (short)0x4567, (short)0x0123); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSetAcquire short"); @@ -301,7 +317,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetShortAcquire(base, offset, (short)0x4567, (short)0x89AB); + boolean success = UNSAFE.compareAndSetShortMO(UNSAFE_MO_WEAK_CAS_ACQUIRE, base, offset, (short)0x4567, (short)0x89AB); assertEquals(success, false, "failing weakCompareAndSetAcquire short"); short x = UNSAFE.getShort(base, offset); assertEquals(x, (short)0x0123, "failing weakCompareAndSetAcquire short value"); @@ -310,7 +326,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetShortRelease(base, offset, (short)0x0123, (short)0x4567); + success = UNSAFE.compareAndSetShortMO(UNSAFE_MO_WEAK_CAS_RELEASE, base, offset, (short)0x0123, (short)0x4567); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSetRelease short"); @@ -319,7 +335,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetShortRelease(base, offset, (short)0x0123, (short)0x89AB); + boolean success = UNSAFE.compareAndSetShortMO(UNSAFE_MO_WEAK_CAS_RELEASE, base, offset, (short)0x0123, (short)0x89AB); assertEquals(success, false, "failing weakCompareAndSetRelease short"); short x = UNSAFE.getShort(base, offset); assertEquals(x, (short)0x4567, "failing weakCompareAndSetRelease short value"); @@ -328,7 +344,7 @@ static void testAccess(Object base, long offset) { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetShort(base, offset, (short)0x4567, (short)0x0123); + success = UNSAFE.compareAndSetShortMO(UNSAFE_MO_WEAK_CAS_VOLATILE, base, offset, (short)0x4567, (short)0x0123); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSet short"); @@ -337,7 +353,7 @@ static void testAccess(Object base, long offset) { } { - boolean success = UNSAFE.weakCompareAndSetShort(base, offset, (short)0x4567, (short)0x89AB); + boolean success = UNSAFE.compareAndSetShortMO(UNSAFE_MO_WEAK_CAS_VOLATILE, base, offset, (short)0x4567, (short)0x89AB); assertEquals(success, false, "failing weakCompareAndSet short"); short x = UNSAFE.getShort(base, offset); assertEquals(x, (short)0x0123, "failing weakCompareAndSet short value"); diff --git a/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestBoolean.java b/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestBoolean.java index fcb0b3fbef379..646a303bd7ba5 100644 --- a/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestBoolean.java +++ b/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestBoolean.java @@ -64,6 +64,7 @@ public class SunMiscUnsafeAccessTestBoolean { static int ARRAY_SHIFT; + static { try { Field f = sun.misc.Unsafe.class.getDeclaredField("theUnsafe"); @@ -91,6 +92,7 @@ public class SunMiscUnsafeAccessTestBoolean { ARRAY_OFFSET = UNSAFE.arrayBaseOffset(boolean[].class); int ascale = UNSAFE.arrayIndexScale(boolean[].class); ARRAY_SHIFT = 31 - Integer.numberOfLeadingZeros(ascale); + } static void weakDelay() { diff --git a/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestByte.java b/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestByte.java index 7343463c41d88..6efdf89363843 100644 --- a/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestByte.java +++ b/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestByte.java @@ -64,6 +64,7 @@ public class SunMiscUnsafeAccessTestByte { static int ARRAY_SHIFT; + static { try { Field f = sun.misc.Unsafe.class.getDeclaredField("theUnsafe"); @@ -91,6 +92,7 @@ public class SunMiscUnsafeAccessTestByte { ARRAY_OFFSET = UNSAFE.arrayBaseOffset(byte[].class); int ascale = UNSAFE.arrayIndexScale(byte[].class); ARRAY_SHIFT = 31 - Integer.numberOfLeadingZeros(ascale); + } static void weakDelay() { diff --git a/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestChar.java b/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestChar.java index 4c5e02bea2665..11bb90d89c59c 100644 --- a/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestChar.java +++ b/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestChar.java @@ -64,6 +64,7 @@ public class SunMiscUnsafeAccessTestChar { static int ARRAY_SHIFT; + static { try { Field f = sun.misc.Unsafe.class.getDeclaredField("theUnsafe"); @@ -91,6 +92,7 @@ public class SunMiscUnsafeAccessTestChar { ARRAY_OFFSET = UNSAFE.arrayBaseOffset(char[].class); int ascale = UNSAFE.arrayIndexScale(char[].class); ARRAY_SHIFT = 31 - Integer.numberOfLeadingZeros(ascale); + } static void weakDelay() { diff --git a/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestDouble.java b/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestDouble.java index 5503dcd1840f9..49b0e73759d95 100644 --- a/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestDouble.java +++ b/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestDouble.java @@ -64,6 +64,7 @@ public class SunMiscUnsafeAccessTestDouble { static int ARRAY_SHIFT; + static { try { Field f = sun.misc.Unsafe.class.getDeclaredField("theUnsafe"); @@ -91,6 +92,7 @@ public class SunMiscUnsafeAccessTestDouble { ARRAY_OFFSET = UNSAFE.arrayBaseOffset(double[].class); int ascale = UNSAFE.arrayIndexScale(double[].class); ARRAY_SHIFT = 31 - Integer.numberOfLeadingZeros(ascale); + } static void weakDelay() { diff --git a/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestFloat.java b/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestFloat.java index f050c5bb17b8b..4f85f369ea254 100644 --- a/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestFloat.java +++ b/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestFloat.java @@ -64,6 +64,7 @@ public class SunMiscUnsafeAccessTestFloat { static int ARRAY_SHIFT; + static { try { Field f = sun.misc.Unsafe.class.getDeclaredField("theUnsafe"); @@ -91,6 +92,7 @@ public class SunMiscUnsafeAccessTestFloat { ARRAY_OFFSET = UNSAFE.arrayBaseOffset(float[].class); int ascale = UNSAFE.arrayIndexScale(float[].class); ARRAY_SHIFT = 31 - Integer.numberOfLeadingZeros(ascale); + } static void weakDelay() { diff --git a/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestInt.java b/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestInt.java index 85f82915e03aa..50e8c6e72a120 100644 --- a/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestInt.java +++ b/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestInt.java @@ -64,6 +64,7 @@ public class SunMiscUnsafeAccessTestInt { static int ARRAY_SHIFT; + static { try { Field f = sun.misc.Unsafe.class.getDeclaredField("theUnsafe"); @@ -91,6 +92,7 @@ public class SunMiscUnsafeAccessTestInt { ARRAY_OFFSET = UNSAFE.arrayBaseOffset(int[].class); int ascale = UNSAFE.arrayIndexScale(int[].class); ARRAY_SHIFT = 31 - Integer.numberOfLeadingZeros(ascale); + } static void weakDelay() { diff --git a/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestLong.java b/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestLong.java index 1a73cfe308654..5b0e117e366c9 100644 --- a/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestLong.java +++ b/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestLong.java @@ -64,6 +64,7 @@ public class SunMiscUnsafeAccessTestLong { static int ARRAY_SHIFT; + static { try { Field f = sun.misc.Unsafe.class.getDeclaredField("theUnsafe"); @@ -91,6 +92,7 @@ public class SunMiscUnsafeAccessTestLong { ARRAY_OFFSET = UNSAFE.arrayBaseOffset(long[].class); int ascale = UNSAFE.arrayIndexScale(long[].class); ARRAY_SHIFT = 31 - Integer.numberOfLeadingZeros(ascale); + } static void weakDelay() { diff --git a/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestObject.java b/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestObject.java index 1aab41ddc9b6d..8ee9583999b2f 100644 --- a/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestObject.java +++ b/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestObject.java @@ -64,6 +64,7 @@ public class SunMiscUnsafeAccessTestObject { static int ARRAY_SHIFT; + static { try { Field f = sun.misc.Unsafe.class.getDeclaredField("theUnsafe"); @@ -91,6 +92,7 @@ public class SunMiscUnsafeAccessTestObject { ARRAY_OFFSET = UNSAFE.arrayBaseOffset(Object[].class); int ascale = UNSAFE.arrayIndexScale(Object[].class); ARRAY_SHIFT = 31 - Integer.numberOfLeadingZeros(ascale); + } static void weakDelay() { diff --git a/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestShort.java b/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestShort.java index 123b655220c63..6ff07013c50c9 100644 --- a/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestShort.java +++ b/test/hotspot/jtreg/compiler/unsafe/SunMiscUnsafeAccessTestShort.java @@ -64,6 +64,7 @@ public class SunMiscUnsafeAccessTestShort { static int ARRAY_SHIFT; + static { try { Field f = sun.misc.Unsafe.class.getDeclaredField("theUnsafe"); @@ -91,6 +92,7 @@ public class SunMiscUnsafeAccessTestShort { ARRAY_OFFSET = UNSAFE.arrayBaseOffset(short[].class); int ascale = UNSAFE.arrayIndexScale(short[].class); ARRAY_SHIFT = 31 - Integer.numberOfLeadingZeros(ascale); + } static void weakDelay() { diff --git a/test/hotspot/jtreg/compiler/unsafe/X-UnsafeAccessTest.java.template b/test/hotspot/jtreg/compiler/unsafe/X-UnsafeAccessTest.java.template index f5eadc218e449..34334dd1412fc 100644 --- a/test/hotspot/jtreg/compiler/unsafe/X-UnsafeAccessTest.java.template +++ b/test/hotspot/jtreg/compiler/unsafe/X-UnsafeAccessTest.java.template @@ -68,6 +68,12 @@ public class $Qualifier$UnsafeAccessTest$Type$ { static int ARRAY_SHIFT; +#if[JdkInternalMisc] + // copies of Unsafe.MO_RELEASE and other MO constants + static byte UNSAFE_MO_RELEASE, UNSAFE_MO_ACQUIRE, UNSAFE_MO_OPAQUE, UNSAFE_MO_VOLATILE; + static byte UNSAFE_MO_WEAK_CAS_ACQUIRE, UNSAFE_MO_WEAK_CAS_PLAIN, UNSAFE_MO_WEAK_CAS_RELEASE, UNSAFE_MO_WEAK_CAS_VOLATILE; +#end[JdkInternalMisc] + static { try { Field f = $package$.Unsafe.class.getDeclaredField("theUnsafe"); @@ -95,6 +101,20 @@ public class $Qualifier$UnsafeAccessTest$Type$ { ARRAY_OFFSET = UNSAFE.arrayBaseOffset($type$[].class); int ascale = UNSAFE.arrayIndexScale($type$[].class); ARRAY_SHIFT = 31 - Integer.numberOfLeadingZeros(ascale); + +#if[JdkInternalMisc] + try { + for (Field umoField : $Qualifier$UnsafeAccessTest$Type$.class.getDeclaredFields()) { + String name = umoField.getName(); + if (!name.startsWith("UNSAFE_MO_")) continue; + Field moField = UNSAFE.getClass().getDeclaredField(name.substring("UNSAFE_".length())); + byte value = (Byte) moField.get(null); + umoField.set(null, value); + } + } catch (Exception e) { + throw new RuntimeException(e); + } +#end[JdkInternalMisc] } static void weakDelay() { @@ -199,15 +219,15 @@ public class $Qualifier$UnsafeAccessTest$Type$ { #if[JdkInternalMisc] // Lazy { - UNSAFE.put$MethodAffix$Release(base, offset, $value1$); - $type$ x = UNSAFE.get$MethodAffix$Acquire(base, offset); + UNSAFE.put$MethodAffix$MO(UNSAFE_MO_RELEASE, base, offset, $value1$); + $type$ x = UNSAFE.get$MethodAffix$MO(UNSAFE_MO_ACQUIRE, base, offset); assertEquals(x, $value1$, "putRelease $type$ value"); } // Opaque { - UNSAFE.put$MethodAffix$Opaque(base, offset, $value2$); - $type$ x = UNSAFE.get$MethodAffix$Opaque(base, offset); + UNSAFE.put$MethodAffix$MO(UNSAFE_MO_OPAQUE, base, offset, $value2$); + $type$ x = UNSAFE.get$MethodAffix$MO(UNSAFE_MO_OPAQUE, base, offset); assertEquals(x, $value2$, "putOpaque $type$ value"); } #end[JdkInternalMisc] @@ -288,28 +308,28 @@ public class $Qualifier$UnsafeAccessTest$Type$ { } { - $type$ r = UNSAFE.compareAndExchange$MethodAffix$Acquire(base, offset, $value1$, $value2$); + $type$ r = UNSAFE.compareAndExchange$MethodAffix$MO(UNSAFE_MO_ACQUIRE, base, offset, $value1$, $value2$); assertEquals(r, $value1$, "success compareAndExchangeAcquire $type$"); $type$ x = UNSAFE.get$MethodAffix$(base, offset); assertEquals(x, $value2$, "success compareAndExchangeAcquire $type$ value"); } { - $type$ r = UNSAFE.compareAndExchange$MethodAffix$Acquire(base, offset, $value1$, $value3$); + $type$ r = UNSAFE.compareAndExchange$MethodAffix$MO(UNSAFE_MO_ACQUIRE, base, offset, $value1$, $value3$); assertEquals(r, $value2$, "failing compareAndExchangeAcquire $type$"); $type$ x = UNSAFE.get$MethodAffix$(base, offset); assertEquals(x, $value2$, "failing compareAndExchangeAcquire $type$ value"); } { - $type$ r = UNSAFE.compareAndExchange$MethodAffix$Release(base, offset, $value2$, $value1$); + $type$ r = UNSAFE.compareAndExchange$MethodAffix$MO(UNSAFE_MO_RELEASE, base, offset, $value2$, $value1$); assertEquals(r, $value2$, "success compareAndExchangeRelease $type$"); $type$ x = UNSAFE.get$MethodAffix$(base, offset); assertEquals(x, $value1$, "success compareAndExchangeRelease $type$ value"); } { - $type$ r = UNSAFE.compareAndExchange$MethodAffix$Release(base, offset, $value2$, $value3$); + $type$ r = UNSAFE.compareAndExchange$MethodAffix$MO(UNSAFE_MO_RELEASE, base, offset, $value2$, $value3$); assertEquals(r, $value1$, "failing compareAndExchangeRelease $type$"); $type$ x = UNSAFE.get$MethodAffix$(base, offset); assertEquals(x, $value1$, "failing compareAndExchangeRelease $type$ value"); @@ -318,7 +338,7 @@ public class $Qualifier$UnsafeAccessTest$Type$ { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSet$MethodAffix$Plain(base, offset, $value1$, $value2$); + success = UNSAFE.compareAndSet$MethodAffix$MO(UNSAFE_MO_WEAK_CAS_PLAIN, base, offset, $value1$, $value2$); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSetPlain $type$"); @@ -327,7 +347,7 @@ public class $Qualifier$UnsafeAccessTest$Type$ { } { - boolean success = UNSAFE.weakCompareAndSet$MethodAffix$Plain(base, offset, $value1$, $value3$); + boolean success = UNSAFE.compareAndSet$MethodAffix$MO(UNSAFE_MO_WEAK_CAS_PLAIN, base, offset, $value1$, $value3$); assertEquals(success, false, "failing weakCompareAndSetPlain $type$"); $type$ x = UNSAFE.get$MethodAffix$(base, offset); assertEquals(x, $value2$, "failing weakCompareAndSetPlain $type$ value"); @@ -336,7 +356,7 @@ public class $Qualifier$UnsafeAccessTest$Type$ { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSet$MethodAffix$Acquire(base, offset, $value2$, $value1$); + success = UNSAFE.compareAndSet$MethodAffix$MO(UNSAFE_MO_WEAK_CAS_ACQUIRE, base, offset, $value2$, $value1$); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSetAcquire $type$"); @@ -345,7 +365,7 @@ public class $Qualifier$UnsafeAccessTest$Type$ { } { - boolean success = UNSAFE.weakCompareAndSet$MethodAffix$Acquire(base, offset, $value2$, $value3$); + boolean success = UNSAFE.compareAndSet$MethodAffix$MO(UNSAFE_MO_WEAK_CAS_ACQUIRE, base, offset, $value2$, $value3$); assertEquals(success, false, "failing weakCompareAndSetAcquire $type$"); $type$ x = UNSAFE.get$MethodAffix$(base, offset); assertEquals(x, $value1$, "failing weakCompareAndSetAcquire $type$ value"); @@ -354,7 +374,7 @@ public class $Qualifier$UnsafeAccessTest$Type$ { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSet$MethodAffix$Release(base, offset, $value1$, $value2$); + success = UNSAFE.compareAndSet$MethodAffix$MO(UNSAFE_MO_WEAK_CAS_RELEASE, base, offset, $value1$, $value2$); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSetRelease $type$"); @@ -363,7 +383,7 @@ public class $Qualifier$UnsafeAccessTest$Type$ { } { - boolean success = UNSAFE.weakCompareAndSet$MethodAffix$Release(base, offset, $value1$, $value3$); + boolean success = UNSAFE.compareAndSet$MethodAffix$MO(UNSAFE_MO_WEAK_CAS_RELEASE, base, offset, $value1$, $value3$); assertEquals(success, false, "failing weakCompareAndSetRelease $type$"); $type$ x = UNSAFE.get$MethodAffix$(base, offset); assertEquals(x, $value2$, "failing weakCompareAndSetRelease $type$ value"); @@ -372,7 +392,7 @@ public class $Qualifier$UnsafeAccessTest$Type$ { { boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSet$MethodAffix$(base, offset, $value2$, $value1$); + success = UNSAFE.compareAndSet$MethodAffix$MO(UNSAFE_MO_WEAK_CAS_VOLATILE, base, offset, $value2$, $value1$); if (!success) weakDelay(); } assertEquals(success, true, "success weakCompareAndSet $type$"); @@ -381,7 +401,7 @@ public class $Qualifier$UnsafeAccessTest$Type$ { } { - boolean success = UNSAFE.weakCompareAndSet$MethodAffix$(base, offset, $value2$, $value3$); + boolean success = UNSAFE.compareAndSet$MethodAffix$MO(UNSAFE_MO_WEAK_CAS_VOLATILE, base, offset, $value2$, $value3$); assertEquals(success, false, "failing weakCompareAndSet $type$"); $type$ x = UNSAFE.get$MethodAffix$(base, offset); assertEquals(x, $value1$, "failing weakCompareAndSet $type$ value"); diff --git a/test/hotspot/jtreg/gc/shenandoah/compiler/TestReferenceCAS.java b/test/hotspot/jtreg/gc/shenandoah/compiler/TestReferenceCAS.java index 277da9eeb6307..e133db678e92d 100644 --- a/test/hotspot/jtreg/gc/shenandoah/compiler/TestReferenceCAS.java +++ b/test/hotspot/jtreg/gc/shenandoah/compiler/TestReferenceCAS.java @@ -175,7 +175,7 @@ static void testAccess(Object base, long offset) { String newval = foo; boolean success = false; for (int c = 0; c < WEAK_ATTEMPTS && !success; c++) { - success = UNSAFE.weakCompareAndSetReference(base, offset, "bar", newval); + success = UNSAFE.compareAndSetReferenceMO(Unsafe.MO_WEAK_CAS_VOLATILE, base, offset, "bar", newval); assertEquals(newval, "foo", "must not destroy newval"); } assertEquals(success, true, "weakCompareAndSet Object");