| Quellcodebibliothek Statistik Leitseite products/Sources/formale Sprachen/Java/Openjdk/src/hotspot/share/opto/ (Sun/Oracle ©) Datei vom 29.11.2022 mit Größe 325 kB |
|
SSL library_call.cppInteraktion undPortierbarkeitC |
|
|
/* * Copyright (c) 1999, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as * published by the Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA * or visit www.oracle.com if you need additional information or have any * questions. * */ #include "precompiled.hpp" #include "asm/macroAssembler.hpp" #include "ci/ciUtilities.inline.hpp" #include "classfile/vmIntrinsics.hpp" #include "compiler/compileBroker.hpp" #include "compiler/compileLog.hpp" #include "gc/shared/barrierSet.hpp" #include "jfr/support/jfrIntrinsics.hpp" #include "memory/resourceArea.hpp" #include "oops/klass.inline.hpp" #include "oops/objArrayKlass.hpp" #include "opto/addnode.hpp" #include "opto/arraycopynode.hpp" #include "opto/c2compiler.hpp" #include "opto/castnode.hpp" #include "opto/cfgnode.hpp" #include "opto/convertnode.hpp" #include "opto/countbitsnode.hpp" #include "opto/idealKit.hpp" #include "opto/library_call.hpp" #include "opto/mathexactnode.hpp" #include "opto/mulnode.hpp" #include "opto/narrowptrnode.hpp" #include "opto/opaquenode.hpp" #include "opto/parse.hpp" #include "opto/runtime.hpp" #include "opto/rootnode.hpp" #include "opto/subnode.hpp" #include "prims/unsafe.hpp" #include "runtime/objectMonitor.hpp" #include "runtime/sharedRuntime.hpp" #include "runtime/stubRoutines.hpp" #include "utilities/macros.hpp" #include "utilities/powerOfTwo.hpp" //---------------------------make_vm_intrinsic---------------------------- CallGenerator* Compile::make_vm_intrinsic(ciMethod* m, bool is_virtual) { vmIntrinsicID id = m->intrinsic_id(); assert(id != vmIntrinsics::_none, "must be a VM intrinsic"); if (!m->is_loaded()) { // Do not attempt to inline unloaded methods. return NULL; } C2Compiler* compiler = (C2Compiler*)CompileBroker::compiler(CompLevel_full_optimiza bool is_available = false; { // For calling is_intrinsic_supported and is_intrinsic_disabled_by_flag // the compiler must transition to '_thread_in_vm' state because both // methods access VM-internal data. VM_ENTRY_MARK; methodHandle mh(THREAD, m->get_Method()); is_available = compiler != NULL && compiler->is_intrinsic_supported(mh, is_virtual) && !C->directive()->is_intrinsic_disabled(mh) && !vmIntrinsics::is_disabled_by_flags(mh); } if (is_available) { assert(id <= vmIntrinsics::LAST_COMPILER_INLINE, "caller responsibility"); assert(id != vmIntrinsics::_Object_init && id != vmIntrinsics::_invoke, "enum out of order?" return new LibraryIntrinsic(m, is_virtual, vmIntrinsics::predicates_needed(id), vmIntrinsics::does_virtual_dispatch(id), id); } else { return NULL; } } JVMState* LibraryIntrinsic::generate(JVMState* jvms) { LibraryCallKit kit(jvms, this); Compile* C = kit.C; int nodes = C->unique(); #ifndef PRODUCT if ((C->print_intrinsics() || C->print_inlining()) && Verbose) { char buf[1000]; const char* str = vmIntrinsics::short_name_as_C_string(intrinsic_id(), buf, sizeof(buf tty->print_cr("Intrinsic %s", str); } #endif ciMethod* callee = kit.callee(); const int bci = kit.bci(); #ifdef ASSERT Node* ctrl = kit.control(); #endif // Try to inline the intrinsic. if (callee->check_intrinsic_candidate() && kit.try_to_inline(_last_predicate)) { const char *inline_msg = is_virtual() ? "(intrinsic, virtual)" : "(intrinsic)"; CompileTask::print_inlining_ul(callee, jvms->depth() - 1, bci, inline_msg); if (C->print_intrinsics() || C->print_inlining()) { C->print_inlining(callee, jvms->depth() - 1, bci, inline_msg); } C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_wo if (C->log()) { C->log()->elem("intrinsic id='%s'%s nodes='%d'", vmIntrinsics::name_at(intrinsic_id()), (is_virtual() ? " virtual='1'" : ""), C->unique() - nodes); } // Push the result from the inlined method onto the stack. kit.push_result(); C->print_inlining_update(this); return kit.transfer_exceptions_into_jvms(); } // The intrinsic bailed out assert(ctrl == kit.control(), "Control flow was added although the intrinsic bailed out"); if (jvms->has_method()) { // Not a root compile. const char* msg; if (callee->intrinsic_candidate()) { msg = is_virtual() ? "failed to inline (intrinsic, virtual)" : "failed to inline (intrinsic)" } else { msg = is_virtual() ? "failed to inline (intrinsic, virtual), method not annotated" : "failed to inline (intrinsic), method not annotated"; } CompileTask::print_inlining_ul(callee, jvms->depth() - 1, bci, msg); if (C->print_intrinsics() || C->print_inlining()) { C->print_inlining(callee, jvms->depth() - 1, bci, msg); } } else { // Root compile ResourceMark rm; stringStream msg_stream; msg_stream.print("Did not generate intrinsic %s%s at bci:%d in", vmIntrinsics::name_at(intrinsic_id()), is_virtual() ? " (virtual)" : "", bci); const char *msg = msg_stream.freeze(); log_debug(jit, inlining)("%s", msg); if (C->print_intrinsics() || C->print_inlining()) { tty->print("%s", msg); } } C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_fa C->print_inlining_update(this); return NULL; } Node* LibraryIntrinsic::generate_predicate(JVMState* jvms, int predicate) { LibraryCallKit kit(jvms, this); Compile* C = kit.C; int nodes = C->unique(); _last_predicate = predicate; #ifndef PRODUCT assert(is_predicated() && predicate < predicates_count(), "sanity"); if ((C->print_intrinsics() || C->print_inlining()) && Verbose) { char buf[1000]; const char* str = vmIntrinsics::short_name_as_C_string(intrinsic_id(), buf, sizeof(buf tty->print_cr("Predicate for intrinsic %s", str); } #endif ciMethod* callee = kit.callee(); const int bci = kit.bci(); Node* slow_ctl = kit.try_to_predicate(predicate); if (!kit.failing()) { const char *inline_msg = is_virtual() ? "(intrinsic, virtual, predicate)" : "(intrinsic, predicate)"; CompileTask::print_inlining_ul(callee, jvms->depth() - 1, bci, inline_msg); if (C->print_intrinsics() || C->print_inlining()) { C->print_inlining(callee, jvms->depth() - 1, bci, inline_msg); } C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_wo if (C->log()) { C->log()->elem("predicate_intrinsic id='%s'%s nodes='%d'", vmIntrinsics::name_at(intrinsic_id()), (is_virtual() ? " virtual='1'" : ""), C->unique() - nodes); } return slow_ctl; // Could be NULL if the check folds. } // The intrinsic bailed out if (jvms->has_method()) { // Not a root compile. const char* msg = "failed to generate predicate for intrinsic"; CompileTask::print_inlining_ul(kit.callee(), jvms->depth() - 1, bci, msg); if (C->print_intrinsics() || C->print_inlining()) { C->print_inlining(kit.callee(), jvms->depth() - 1, bci, msg); } } else { // Root compile ResourceMark rm; stringStream msg_stream; msg_stream.print("Did not generate intrinsic %s%s at bci:%d in", vmIntrinsics::name_at(intrinsic_id()), is_virtual() ? " (virtual)" : "", bci); const char *msg = msg_stream.freeze(); log_debug(jit, inlining)("%s", msg); if (C->print_intrinsics() || C->print_inlining()) { C->print_inlining_stream()->print("%s", msg); } } C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_fa return NULL; } bool LibraryCallKit::try_to_inline(int predicate) { // Handle symbolic names for otherwise undistinguished boolean switches: const bool is_store = true; const bool is_compress = true; const bool is_static = true; const bool is_volatile = true; if (!jvms()->has_method()) { // Root JVMState has a null method. assert(map()->memory()->Opcode() == Op_Parm, ""); // Insert the memory aliasing node set_all_memory(reset_memory()); } assert(merged_memory(), ""); switch (intrinsic_id()) { case vmIntrinsics::_hashCode: return inline_native_hashcode(intrinsic()->is_virtual( case vmIntrinsics::_identityHashCode: return inline_native_hashcode(/*!virtual*/ fal case vmIntrinsics::_getClass: return inline_native_getClass(); case vmIntrinsics::_ceil: case vmIntrinsics::_floor: case vmIntrinsics::_rint: case vmIntrinsics::_dsin: case vmIntrinsics::_dcos: case vmIntrinsics::_dtan: case vmIntrinsics::_dabs: case vmIntrinsics::_fabs: case vmIntrinsics::_iabs: case vmIntrinsics::_labs: case vmIntrinsics::_datan2: case vmIntrinsics::_dsqrt: case vmIntrinsics::_dsqrt_strict: case vmIntrinsics::_dexp: case vmIntrinsics::_dlog: case vmIntrinsics::_dlog10: case vmIntrinsics::_dpow: case vmIntrinsics::_dcopySign: case vmIntrinsics::_fcopySign: case vmIntrinsics::_dsignum: case vmIntrinsics::_roundF: case vmIntrinsics::_roundD: case vmIntrinsics::_fsignum: return inline_math_native(intrinsic_id()); case vmIntrinsics::_notify: case vmIntrinsics::_notifyAll: return inline_notify(intrinsic_id()); case vmIntrinsics::_addExactI: return inline_math_addExactI(false /* add */); case vmIntrinsics::_addExactL: return inline_math_addExactL(false /* add */); case vmIntrinsics::_decrementExactI: return inline_math_subtractExactI(true /* decrem case vmIntrinsics::_decrementExactL: return inline_math_subtractExactL(true /* decrem case vmIntrinsics::_incrementExactI: return inline_math_addExactI(true /* increment */ case vmIntrinsics::_incrementExactL: return inline_math_addExactL(true /* increment */ case vmIntrinsics::_multiplyExactI: return inline_math_multiplyExactI(); case vmIntrinsics::_multiplyExactL: return inline_math_multiplyExactL(); case vmIntrinsics::_multiplyHigh: return inline_math_multiplyHigh(); case vmIntrinsics::_unsignedMultiplyHigh: return inline_math_unsignedMultiplyHigh() case vmIntrinsics::_negateExactI: return inline_math_negateExactI(); case vmIntrinsics::_negateExactL: return inline_math_negateExactL(); case vmIntrinsics::_subtractExactI: return inline_math_subtractExactI(false /* subtra case vmIntrinsics::_subtractExactL: return inline_math_subtractExactL(false /* subtra case vmIntrinsics::_arraycopy: return inline_arraycopy(); case vmIntrinsics::_compareToL: return inline_string_compareTo(StrIntrinsicNode::LL case vmIntrinsics::_compareToU: return inline_string_compareTo(StrIntrinsicNode::UU case vmIntrinsics::_compareToLU: return inline_string_compareTo(StrIntrinsicNode::L case vmIntrinsics::_compareToUL: return inline_string_compareTo(StrIntrinsicNode::U case vmIntrinsics::_indexOfL: return inline_string_indexOf(StrIntrinsicNode::LL); case vmIntrinsics::_indexOfU: return inline_string_indexOf(StrIntrinsicNode::UU); case vmIntrinsics::_indexOfUL: return inline_string_indexOf(StrIntrinsicNode::UL); case vmIntrinsics::_indexOfIL: return inline_string_indexOfI(StrIntrinsicNode::LL); case vmIntrinsics::_indexOfIU: return inline_string_indexOfI(StrIntrinsicNode::UU); case vmIntrinsics::_indexOfIUL: return inline_string_indexOfI(StrIntrinsicNode::UL) case vmIntrinsics::_indexOfU_char: return inline_string_indexOfChar(StrIntrinsicNod case vmIntrinsics::_indexOfL_char: return inline_string_indexOfChar(StrIntrinsicNod case vmIntrinsics::_equalsL: return inline_string_equals(StrIntrinsicNode::LL); case vmIntrinsics::_equalsU: return inline_string_equals(StrIntrinsicNode::UU); case vmIntrinsics::_toBytesStringU: return inline_string_toBytesU(); case vmIntrinsics::_getCharsStringU: return inline_string_getCharsU(); case vmIntrinsics::_getCharStringU: return inline_string_char_access(!is_store); case vmIntrinsics::_putCharStringU: return inline_string_char_access( is_store); case vmIntrinsics::_compressStringC: case vmIntrinsics::_compressStringB: return inline_string_copy( is_compress); case vmIntrinsics::_inflateStringC: case vmIntrinsics::_inflateStringB: return inline_string_copy(!is_compress); case vmIntrinsics::_getReference: return inline_unsafe_access(!is_store, T_OBJECT, Re case vmIntrinsics::_getBoolean: return inline_unsafe_access(!is_store, T_BOOLEAN, Rel case vmIntrinsics::_getByte: return inline_unsafe_access(!is_store, T_BYTE, Relaxed, f case vmIntrinsics::_getShort: return inline_unsafe_access(!is_store, T_SHORT, Relaxed case vmIntrinsics::_getChar: return inline_unsafe_access(!is_store, T_CHAR, Relaxed, f case vmIntrinsics::_getInt: return inline_unsafe_access(!is_store, T_INT, Relaxed, fal case vmIntrinsics::_getLong: return inline_unsafe_access(!is_store, T_LONG, Relaxed, f case vmIntrinsics::_getFloat: return inline_unsafe_access(!is_store, T_FLOAT, Relaxed case vmIntrinsics::_getDouble: return inline_unsafe_access(!is_store, T_DOUBLE, Relax case vmIntrinsics::_putReference: return inline_unsafe_access( is_store, T_OBJECT, Rel case vmIntrinsics::_putBoolean: return inline_unsafe_access( is_store, T_BOOLEAN, Rela case vmIntrinsics::_putByte: return inline_unsafe_access( is_store, T_BYTE, Relaxed, fa case vmIntrinsics::_putShort: return inline_unsafe_access( is_store, T_SHORT, Relaxed, case vmIntrinsics::_putChar: return inline_unsafe_access( is_store, T_CHAR, Relaxed, fa case vmIntrinsics::_putInt: return inline_unsafe_access( is_store, T_INT, Relaxed, fals case vmIntrinsics::_putLong: return inline_unsafe_access( is_store, T_LONG, Relaxed, fa case vmIntrinsics::_putFloat: return inline_unsafe_access( is_store, T_FLOAT, Relaxed, case vmIntrinsics::_putDouble: return inline_unsafe_access( is_store, T_DOUBLE, Relaxe case vmIntrinsics::_getReferenceVolatile: return inline_unsafe_access(!is_store, T_O case vmIntrinsics::_getBooleanVolatile: return inline_unsafe_access(!is_store, T_BOO case vmIntrinsics::_getByteVolatile: return inline_unsafe_access(!is_store, T_BYTE, V case vmIntrinsics::_getShortVolatile: return inline_unsafe_access(!is_store, T_SHORT case vmIntrinsics::_getCharVolatile: return inline_unsafe_access(!is_store, T_CHAR, V case vmIntrinsics::_getIntVolatile: return inline_unsafe_access(!is_store, T_INT, Vol case vmIntrinsics::_getLongVolatile: return inline_unsafe_access(!is_store, T_LONG, V case vmIntrinsics::_getFloatVolatile: return inline_unsafe_access(!is_store, T_FLOAT case vmIntrinsics::_getDoubleVolatile: return inline_unsafe_access(!is_store, T_DOUB case vmIntrinsics::_putReferenceVolatile: return inline_unsafe_access( is_store, T_OB case vmIntrinsics::_putBooleanVolatile: return inline_unsafe_access( is_store, T_BOOL case vmIntrinsics::_putByteVolatile: return inline_unsafe_access( is_store, T_BYTE, Vo case vmIntrinsics::_putShortVolatile: return inline_unsafe_access( is_store, T_SHORT, case vmIntrinsics::_putCharVolatile: return inline_unsafe_access( is_store, T_CHAR, Vo case vmIntrinsics::_putIntVolatile: return inline_unsafe_access( is_store, T_INT, Vola case vmIntrinsics::_putLongVolatile: return inline_unsafe_access( is_store, T_LONG, Vo case vmIntrinsics::_putFloatVolatile: return inline_unsafe_access( is_store, T_FLOAT, case vmIntrinsics::_putDoubleVolatile: return inline_unsafe_access( is_store, T_DOUBL case vmIntrinsics::_getShortUnaligned: return inline_unsafe_access(!is_store, T_SHOR case vmIntrinsics::_getCharUnaligned: return inline_unsafe_access(!is_store, T_CHAR, case vmIntrinsics::_getIntUnaligned: return inline_unsafe_access(!is_store, T_INT, Re case vmIntrinsics::_getLongUnaligned: return inline_unsafe_access(!is_store, T_LONG, case vmIntrinsics::_putShortUnaligned: return inline_unsafe_access( is_store, T_SHORT case vmIntrinsics::_putCharUnaligned: return inline_unsafe_access( is_store, T_CHAR, R case vmIntrinsics::_putIntUnaligned: return inline_unsafe_access( is_store, T_INT, Rel case vmIntrinsics::_putLongUnaligned: return inline_unsafe_access( is_store, T_LONG, R case vmIntrinsics::_getReferenceAcquire: return inline_unsafe_access(!is_store, T_OB case vmIntrinsics::_getBooleanAcquire: return inline_unsafe_access(!is_store, T_BOOL case vmIntrinsics::_getByteAcquire: return inline_unsafe_access(!is_store, T_BYTE, Ac case vmIntrinsics::_getShortAcquire: return inline_unsafe_access(!is_store, T_SHORT, case vmIntrinsics::_getCharAcquire: return inline_unsafe_access(!is_store, T_CHAR, Ac case vmIntrinsics::_getIntAcquire: return inline_unsafe_access(!is_store, T_INT, Acqu case vmIntrinsics::_getLongAcquire: return inline_unsafe_access(!is_store, T_LONG, Ac case vmIntrinsics::_getFloatAcquire: return inline_unsafe_access(!is_store, T_FLOAT, case vmIntrinsics::_getDoubleAcquire: return inline_unsafe_access(!is_store, T_DOUBL case vmIntrinsics::_putReferenceRelease: return inline_unsafe_access( is_store, T_OBJ case vmIntrinsics::_putBooleanRelease: return inline_unsafe_access( is_store, T_BOOLE case vmIntrinsics::_putByteRelease: return inline_unsafe_access( is_store, T_BYTE, Rel case vmIntrinsics::_putShortRelease: return inline_unsafe_access( is_store, T_SHORT, R case vmIntrinsics::_putCharRelease: return inline_unsafe_access( is_store, T_CHAR, Rel case vmIntrinsics::_putIntRelease: return inline_unsafe_access( is_store, T_INT, Relea case vmIntrinsics::_putLongRelease: return inline_unsafe_access( is_store, T_LONG, Rel case vmIntrinsics::_putFloatRelease: return inline_unsafe_access( is_store, T_FLOAT, R case vmIntrinsics::_putDoubleRelease: return inline_unsafe_access( is_store, T_DOUBLE case vmIntrinsics::_getReferenceOpaque: return inline_unsafe_access(!is_store, T_OBJ case vmIntrinsics::_getBooleanOpaque: return inline_unsafe_access(!is_store, T_BOOLE case vmIntrinsics::_getByteOpaque: return inline_unsafe_access(!is_store, T_BYTE, Opa case vmIntrinsics::_getShortOpaque: return inline_unsafe_access(!is_store, T_SHORT, O case vmIntrinsics::_getCharOpaque: return inline_unsafe_access(!is_store, T_CHAR, Opa case vmIntrinsics::_getIntOpaque: return inline_unsafe_access(!is_store, T_INT, Opaqu case vmIntrinsics::_getLongOpaque: return inline_unsafe_access(!is_store, T_LONG, Opa case vmIntrinsics::_getFloatOpaque: return inline_unsafe_access(!is_store, T_FLOAT, O case vmIntrinsics::_getDoubleOpaque: return inline_unsafe_access(!is_store, T_DOUBLE case vmIntrinsics::_putReferenceOpaque: return inline_unsafe_access( is_store, T_OBJE case vmIntrinsics::_putBooleanOpaque: return inline_unsafe_access( is_store, T_BOOLEA case vmIntrinsics::_putByteOpaque: return inline_unsafe_access( is_store, T_BYTE, Opaq case vmIntrinsics::_putShortOpaque: return inline_unsafe_access( is_store, T_SHORT, Op case vmIntrinsics::_putCharOpaque: return inline_unsafe_access( is_store, T_CHAR, Opaq case vmIntrinsics::_putIntOpaque: return inline_unsafe_access( is_store, T_INT, Opaque case vmIntrinsics::_putLongOpaque: return inline_unsafe_access( is_store, T_LONG, Opaq case vmIntrinsics::_putFloatOpaque: return inline_unsafe_access( is_store, T_FLOAT, Op case vmIntrinsics::_putDoubleOpaque: return inline_unsafe_access( is_store, T_DOUBLE, case vmIntrinsics::_compareAndSetReference: return inline_unsafe_load_store(T_OBJEC case vmIntrinsics::_compareAndSetByte: return inline_unsafe_load_store(T_BYTE, LS_cm case vmIntrinsics::_compareAndSetShort: return inline_unsafe_load_store(T_SHORT, LS_ case vmIntrinsics::_compareAndSetInt: return inline_unsafe_load_store(T_INT, LS_cmp_ case vmIntrinsics::_compareAndSetLong: return inline_unsafe_load_store(T_LONG, LS_cm case vmIntrinsics::_weakCompareAndSetReferencePlain: return inline_unsafe_load_stor case vmIntrinsics::_weakCompareAndSetReferenceAcquire: return inline_unsafe_load_st case vmIntrinsics::_weakCompareAndSetReferenceRelease: return inline_unsafe_load_st case vmIntrinsics::_weakCompareAndSetReference: return inline_unsafe_load_store(T_O case vmIntrinsics::_weakCompareAndSetBytePlain: return inline_unsafe_load_store(T_B case vmIntrinsics::_weakCompareAndSetByteAcquire: return inline_unsafe_load_store(T case vmIntrinsics::_weakCompareAndSetByteRelease: return inline_unsafe_load_store(T case vmIntrinsics::_weakCompareAndSetByte: return inline_unsafe_load_store(T_BYTE, L case vmIntrinsics::_weakCompareAndSetShortPlain: return inline_unsafe_load_store(T_ case vmIntrinsics::_weakCompareAndSetShortAcquire: return inline_unsafe_load_store( case vmIntrinsics::_weakCompareAndSetShortRelease: return inline_unsafe_load_store( case vmIntrinsics::_weakCompareAndSetShort: return inline_unsafe_load_store(T_SHORT case vmIntrinsics::_weakCompareAndSetIntPlain: return inline_unsafe_load_store(T_IN case vmIntrinsics::_weakCompareAndSetIntAcquire: return inline_unsafe_load_store(T_ case vmIntrinsics::_weakCompareAndSetIntRelease: return inline_unsafe_load_store(T_ case vmIntrinsics::_weakCompareAndSetInt: return inline_unsafe_load_store(T_INT, LS_ case vmIntrinsics::_weakCompareAndSetLongPlain: return inline_unsafe_load_store(T_L case vmIntrinsics::_weakCompareAndSetLongAcquire: return inline_unsafe_load_store(T case vmIntrinsics::_weakCompareAndSetLongRelease: return inline_unsafe_load_store(T case vmIntrinsics::_weakCompareAndSetLong: return inline_unsafe_load_store(T_LONG, L case vmIntrinsics::_compareAndExchangeReference: return inline_unsafe_load_store(T_ case vmIntrinsics::_compareAndExchangeReferenceAcquire: return inline_unsafe_load_s case vmIntrinsics::_compareAndExchangeReferenceRelease: return inline_unsafe_load_s case vmIntrinsics::_compareAndExchangeByte: return inline_unsafe_load_store(T_BYTE, case vmIntrinsics::_compareAndExchangeByteAcquire: return inline_unsafe_load_store( case vmIntrinsics::_compareAndExchangeByteRelease: return inline_unsafe_load_store( case vmIntrinsics::_compareAndExchangeShort: return inline_unsafe_load_store(T_SHOR case vmIntrinsics::_compareAndExchangeShortAcquire: return inline_unsafe_load_store case vmIntrinsics::_compareAndExchangeShortRelease: return inline_unsafe_load_store case vmIntrinsics::_compareAndExchangeInt: return inline_unsafe_load_store(T_INT, LS case vmIntrinsics::_compareAndExchangeIntAcquire: return inline_unsafe_load_store(T case vmIntrinsics::_compareAndExchangeIntRelease: return inline_unsafe_load_store(T case vmIntrinsics::_compareAndExchangeLong: return inline_unsafe_load_store(T_LONG, case vmIntrinsics::_compareAndExchangeLongAcquire: return inline_unsafe_load_store( case vmIntrinsics::_compareAndExchangeLongRelease: return inline_unsafe_load_store( case vmIntrinsics::_getAndAddByte: return inline_unsafe_load_store(T_BYTE, LS_get_ad case vmIntrinsics::_getAndAddShort: return inline_unsafe_load_store(T_SHORT, LS_get_ case vmIntrinsics::_getAndAddInt: return inline_unsafe_load_store(T_INT, LS_get_add, case vmIntrinsics::_getAndAddLong: return inline_unsafe_load_store(T_LONG, LS_get_ad case vmIntrinsics::_getAndSetByte: return inline_unsafe_load_store(T_BYTE, LS_get_se case vmIntrinsics::_getAndSetShort: return inline_unsafe_load_store(T_SHORT, LS_get_ case vmIntrinsics::_getAndSetInt: return inline_unsafe_load_store(T_INT, LS_get_set, case vmIntrinsics::_getAndSetLong: return inline_unsafe_load_store(T_LONG, LS_get_se case vmIntrinsics::_getAndSetReference: return inline_unsafe_load_store(T_OBJECT, LS case vmIntrinsics::_loadFence: case vmIntrinsics::_storeFence: case vmIntrinsics::_storeStoreFence: case vmIntrinsics::_fullFence: return inline_unsafe_fence(intrinsic_id()); case vmIntrinsics::_onSpinWait: return inline_onspinwait(); case vmIntrinsics::_currentCarrierThread: return inline_native_currentCarrierThread case vmIntrinsics::_currentThread: return inline_native_currentThread(); case vmIntrinsics::_setCurrentThread: return inline_native_setCurrentThread(); case vmIntrinsics::_scopedValueCache: return inline_native_scopedValueCache(); case vmIntrinsics::_setScopedValueCache: return inline_native_setScopedValueCache() #ifdef JFR_HAVE_INTRINSICS case vmIntrinsics::_counterTime: return inline_native_time_funcs(CAST_FROM_FN_PTR(a case vmIntrinsics::_getEventWriter: return inline_native_getEventWriter(); #endif case vmIntrinsics::_currentTimeMillis: return inline_native_time_funcs(CAST_FROM_FN case vmIntrinsics::_nanoTime: return inline_native_time_funcs(CAST_FROM_FN_PTR(addr case vmIntrinsics::_writeback0: return inline_unsafe_writeback0(); case vmIntrinsics::_writebackPreSync0: return inline_unsafe_writebackSync0(true); case vmIntrinsics::_writebackPostSync0: return inline_unsafe_writebackSync0(false); case vmIntrinsics::_allocateInstance: return inline_unsafe_allocate(); case vmIntrinsics::_copyMemory: return inline_unsafe_copyMemory(); case vmIntrinsics::_getLength: return inline_native_getLength(); case vmIntrinsics::_copyOf: return inline_array_copyOf(false); case vmIntrinsics::_copyOfRange: return inline_array_copyOf(true); case vmIntrinsics::_equalsB: return inline_array_equals(StrIntrinsicNode::LL); case vmIntrinsics::_equalsC: return inline_array_equals(StrIntrinsicNode::UU); case vmIntrinsics::_Preconditions_checkIndex: return inline_preconditions_checkInde case vmIntrinsics::_Preconditions_checkLongIndex: return inline_preconditions_check case vmIntrinsics::_clone: return inline_native_clone(intrinsic()->is_virtual()); case vmIntrinsics::_allocateUninitializedArray: return inline_unsafe_newArray(true) case vmIntrinsics::_newArray: return inline_unsafe_newArray(false); case vmIntrinsics::_isAssignableFrom: return inline_native_subtype_check(); case vmIntrinsics::_isInstance: case vmIntrinsics::_getModifiers: case vmIntrinsics::_isInterface: case vmIntrinsics::_isArray: case vmIntrinsics::_isPrimitive: case vmIntrinsics::_isHidden: case vmIntrinsics::_getSuperclass: case vmIntrinsics::_getClassAccessFlags: return inline_native_Class_query(intrinsic case vmIntrinsics::_floatToRawIntBits: case vmIntrinsics::_floatToIntBits: case vmIntrinsics::_intBitsToFloat: case vmIntrinsics::_doubleToRawLongBits: case vmIntrinsics::_doubleToLongBits: case vmIntrinsics::_longBitsToDouble: case vmIntrinsics::_floatToFloat16: case vmIntrinsics::_float16ToFloat: return inline_fp_conversions(intrinsic_id()); case vmIntrinsics::_floatIsFinite: case vmIntrinsics::_floatIsInfinite: case vmIntrinsics::_doubleIsFinite: case vmIntrinsics::_doubleIsInfinite: return inline_fp_range_check(intrinsic_id()); case vmIntrinsics::_numberOfLeadingZeros_i: case vmIntrinsics::_numberOfLeadingZeros_l: case vmIntrinsics::_numberOfTrailingZeros_i: case vmIntrinsics::_numberOfTrailingZeros_l: case vmIntrinsics::_bitCount_i: case vmIntrinsics::_bitCount_l: case vmIntrinsics::_reverse_i: case vmIntrinsics::_reverse_l: case vmIntrinsics::_reverseBytes_i: case vmIntrinsics::_reverseBytes_l: case vmIntrinsics::_reverseBytes_s: case vmIntrinsics::_reverseBytes_c: return inline_number_methods(intrinsic_id()); case vmIntrinsics::_compress_i: case vmIntrinsics::_compress_l: case vmIntrinsics::_expand_i: case vmIntrinsics::_expand_l: return inline_bitshuffle_methods(intrinsic_id()); case vmIntrinsics::_compareUnsigned_i: case vmIntrinsics::_compareUnsigned_l: return inline_compare_unsigned(intrinsic_id( case vmIntrinsics::_divideUnsigned_i: case vmIntrinsics::_divideUnsigned_l: case vmIntrinsics::_remainderUnsigned_i: case vmIntrinsics::_remainderUnsigned_l: return inline_divmod_methods(intrinsic_id( case vmIntrinsics::_getCallerClass: return inline_native_Reflection_getCallerClass( case vmIntrinsics::_Reference_get: return inline_reference_get(); case vmIntrinsics::_Reference_refersTo0: return inline_reference_refersTo0(false); case vmIntrinsics::_PhantomReference_refersTo0: return inline_reference_refersTo0(t case vmIntrinsics::_Class_cast: return inline_Class_cast(); case vmIntrinsics::_aescrypt_encryptBlock: case vmIntrinsics::_aescrypt_decryptBlock: return inline_aescrypt_Block(intrinsic_i case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt: case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt: return inline_cipherBlockChaining_AESCrypt(intrinsic_id()); case vmIntrinsics::_electronicCodeBook_encryptAESCrypt: case vmIntrinsics::_electronicCodeBook_decryptAESCrypt: return inline_electronicCodeBook_AESCrypt(intrinsic_id()); case vmIntrinsics::_counterMode_AESCrypt: return inline_counterMode_AESCrypt(intrinsic_id()); case vmIntrinsics::_galoisCounterMode_AESCrypt: return inline_galoisCounterMode_AESCrypt(); case vmIntrinsics::_md5_implCompress: case vmIntrinsics::_sha_implCompress: case vmIntrinsics::_sha2_implCompress: case vmIntrinsics::_sha5_implCompress: case vmIntrinsics::_sha3_implCompress: return inline_digestBase_implCompress(intrinsic_id()); case vmIntrinsics::_digestBase_implCompressMB: return inline_digestBase_implCompressMB(predicate); case vmIntrinsics::_multiplyToLen: return inline_multiplyToLen(); case vmIntrinsics::_squareToLen: return inline_squareToLen(); case vmIntrinsics::_mulAdd: return inline_mulAdd(); case vmIntrinsics::_montgomeryMultiply: return inline_montgomeryMultiply(); case vmIntrinsics::_montgomerySquare: return inline_montgomerySquare(); case vmIntrinsics::_bigIntegerRightShiftWorker: return inline_bigIntegerShift(true); case vmIntrinsics::_bigIntegerLeftShiftWorker: return inline_bigIntegerShift(false); case vmIntrinsics::_vectorizedMismatch: return inline_vectorizedMismatch(); case vmIntrinsics::_ghash_processBlocks: return inline_ghash_processBlocks(); case vmIntrinsics::_chacha20Block: return inline_chacha20Block(); case vmIntrinsics::_base64_encodeBlock: return inline_base64_encodeBlock(); case vmIntrinsics::_base64_decodeBlock: return inline_base64_decodeBlock(); case vmIntrinsics::_poly1305_processBlocks: return inline_poly1305_processBlocks(); case vmIntrinsics::_encodeISOArray: case vmIntrinsics::_encodeByteISOArray: return inline_encodeISOArray(false); case vmIntrinsics::_encodeAsciiArray: return inline_encodeISOArray(true); case vmIntrinsics::_updateCRC32: return inline_updateCRC32(); case vmIntrinsics::_updateBytesCRC32: return inline_updateBytesCRC32(); case vmIntrinsics::_updateByteBufferCRC32: return inline_updateByteBufferCRC32(); case vmIntrinsics::_updateBytesCRC32C: return inline_updateBytesCRC32C(); case vmIntrinsics::_updateDirectByteBufferCRC32C: return inline_updateDirectByteBufferCRC32C(); case vmIntrinsics::_updateBytesAdler32: return inline_updateBytesAdler32(); case vmIntrinsics::_updateByteBufferAdler32: return inline_updateByteBufferAdler32(); case vmIntrinsics::_profileBoolean: return inline_profileBoolean(); case vmIntrinsics::_isCompileConstant: return inline_isCompileConstant(); case vmIntrinsics::_countPositives: return inline_countPositives(); case vmIntrinsics::_fmaD: case vmIntrinsics::_fmaF: return inline_fma(intrinsic_id()); case vmIntrinsics::_isDigit: case vmIntrinsics::_isLowerCase: case vmIntrinsics::_isUpperCase: case vmIntrinsics::_isWhitespace: return inline_character_compare(intrinsic_id()); case vmIntrinsics::_min: case vmIntrinsics::_max: case vmIntrinsics::_min_strict: case vmIntrinsics::_max_strict: return inline_min_max(intrinsic_id()); case vmIntrinsics::_maxF: case vmIntrinsics::_minF: case vmIntrinsics::_maxD: case vmIntrinsics::_minD: case vmIntrinsics::_maxF_strict: case vmIntrinsics::_minF_strict: case vmIntrinsics::_maxD_strict: case vmIntrinsics::_minD_strict: return inline_fp_min_max(intrinsic_id()); case vmIntrinsics::_VectorUnaryOp: return inline_vector_nary_operation(1); case vmIntrinsics::_VectorBinaryOp: return inline_vector_nary_operation(2); case vmIntrinsics::_VectorTernaryOp: return inline_vector_nary_operation(3); case vmIntrinsics::_VectorFromBitsCoerced: return inline_vector_frombits_coerced(); case vmIntrinsics::_VectorShuffleIota: return inline_vector_shuffle_iota(); case vmIntrinsics::_VectorMaskOp: return inline_vector_mask_operation(); case vmIntrinsics::_VectorShuffleToVector: return inline_vector_shuffle_to_vector(); case vmIntrinsics::_VectorLoadOp: return inline_vector_mem_operation(/*is_store=*/false); case vmIntrinsics::_VectorLoadMaskedOp: return inline_vector_mem_masked_operation(/*is_store*/false); case vmIntrinsics::_VectorStoreOp: return inline_vector_mem_operation(/*is_store=*/true); case vmIntrinsics::_VectorStoreMaskedOp: return inline_vector_mem_masked_operation(/*is_store=*/true); case vmIntrinsics::_VectorGatherOp: return inline_vector_gather_scatter(/*is_scatter*/ false); case vmIntrinsics::_VectorScatterOp: return inline_vector_gather_scatter(/*is_scatter*/ true); case vmIntrinsics::_VectorReductionCoerced: return inline_vector_reduction(); case vmIntrinsics::_VectorTest: return inline_vector_test(); case vmIntrinsics::_VectorBlend: return inline_vector_blend(); case vmIntrinsics::_VectorRearrange: return inline_vector_rearrange(); case vmIntrinsics::_VectorCompare: return inline_vector_compare(); case vmIntrinsics::_VectorBroadcastInt: return inline_vector_broadcast_int(); case vmIntrinsics::_VectorConvert: return inline_vector_convert(); case vmIntrinsics::_VectorInsert: return inline_vector_insert(); case vmIntrinsics::_VectorExtract: return inline_vector_extract(); case vmIntrinsics::_VectorCompressExpand: return inline_vector_compress_expand(); case vmIntrinsics::_IndexVector: return inline_index_vector(); case vmIntrinsics::_getObjectSize: return inline_getObjectSize(); case vmIntrinsics::_blackhole: return inline_blackhole(); default: // If you get here, it may be that someone has added a new intrinsic // to the list in vmIntrinsics.hpp without implementing it here. #ifndef PRODUCT if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) { tty->print_cr("*** Warning: Unimplemented intrinsic %s(%d)", vmIntrinsics::name_at(intrinsic_id()), vmIntrinsics::as_int(intrinsic_id())); } #endif return false; } } Node* LibraryCallKit::try_to_predicate(int predicate) { if (!jvms()->has_method()) { // Root JVMState has a null method. assert(map()->memory()->Opcode() == Op_Parm, ""); // Insert the memory aliasing node set_all_memory(reset_memory()); } assert(merged_memory(), ""); switch (intrinsic_id()) { case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt: return inline_cipherBlockChaining_AESCrypt_predicate(false); case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt: return inline_cipherBlockChaining_AESCrypt_predicate(true); case vmIntrinsics::_electronicCodeBook_encryptAESCrypt: return inline_electronicCodeBook_AESCrypt_predicate(false); case vmIntrinsics::_electronicCodeBook_decryptAESCrypt: return inline_electronicCodeBook_AESCrypt_predicate(true); case vmIntrinsics::_counterMode_AESCrypt: return inline_counterMode_AESCrypt_predicate(); case vmIntrinsics::_digestBase_implCompressMB: return inline_digestBase_implCompressMB_predicate(predicate); case vmIntrinsics::_galoisCounterMode_AESCrypt: return inline_galoisCounterMode_AESCrypt_predicate(); default: // If you get here, it may be that someone has added a new intrinsic // to the list in vmIntrinsics.hpp without implementing it here. #ifndef PRODUCT if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) { tty->print_cr("*** Warning: Unimplemented predicate for intrinsic %s(%d)", vmIntrinsics::name_at(intrinsic_id()), vmIntrinsics::as_int(intrinsic_id())); } #endif Node* slow_ctl = control(); set_control(top()); // No fast path intrinsic return slow_ctl; } } //------------------------------set_result------------------------------- // Helper function for finishing intrinsics. void LibraryCallKit::set_result(RegionNode* region, PhiNode* value) { record_for_igvn(region); set_control(_gvn.transform(region)); set_result( _gvn.transform(value)); assert(value->type()->basic_type() == result()->bottom_type()->basic_type(), "sanity"); } //------------------------------generate_guard--------------------------- // Helper function for generating guarded fast-slow graph structures. // The given 'test', if true, guards a slow path. If the test fails // then a fast path can be taken. (We generally hope it fails.) // In all cases, GraphKit::control() is updated to the fast path. // The returned value represents the control for the slow path. // The return value is never 'top'; it is either a valid control // or NULL if it is obvious that the slow path can never be taken. // Also, if region and the slow control are not NULL, the slow edge // is appended to the region. Node* LibraryCallKit::generate_guard(Node* test, RegionNode* region, float true_prob) { if (stopped()) { // Already short circuited. return NULL; } // Build an if node and its projections. // If test is true we take the slow path, which we assume is uncommon. if (_gvn.type(test) == TypeInt::ZERO) { // The slow branch is never taken. No need to build this guard. return NULL; } IfNode* iff = create_and_map_if(control(), test, true_prob, COUNT_UNKNOWN); Node* if_slow = _gvn.transform(new IfTrueNode(iff)); if (if_slow == top()) { // The slow branch is never taken. No need to build this guard. return NULL; } if (region != NULL) region->add_req(if_slow); Node* if_fast = _gvn.transform(new IfFalseNode(iff)); set_control(if_fast); return if_slow; } inline Node* LibraryCallKit::generate_slow_guard(Node* test, RegionNode* region) { return generate_guard(test, region, PROB_UNLIKELY_MAG(3)); } inline Node* LibraryCallKit::generate_fair_guard(Node* test, RegionNode* region) { return generate_guard(test, region, PROB_FAIR); } inline Node* LibraryCallKit::generate_negative_guard(Node* index, RegionNode* region, Node* *pos_index) { if (stopped()) return NULL; // already stopped if (_gvn.type(index)->higher_equal(TypeInt::POS)) // [0,maxint] return NULL; // index is already adequately typed Node* cmp_lt = _gvn.transform(new CmpINode(index, intcon(0))); Node* bol_lt = _gvn.transform(new BoolNode(cmp_lt, BoolTest::lt)); Node* is_neg = generate_guard(bol_lt, region, PROB_MIN); if (is_neg != NULL && pos_index != NULL) { // Emulate effect of Parse::adjust_map_after_if. Node* ccast = new CastIINode(index, TypeInt::POS); ccast->set_req(0, control()); (*pos_index) = _gvn.transform(ccast); } return is_neg; } // Make sure that 'position' is a valid limit index, in [0..length]. // There are two equivalent plans for checking this: // A. (offset + copyLength) unsigned<= arrayLength // B. offset <= (arrayLength - copyLength) // We require that all of the values above, except for the sum and // difference, are already known to be non-negative. // Plan A is robust in the face of overflow, if offset and copyLength // are both hugely positive. // // Plan B is less direct and intuitive, but it does not overflow at // all, since the difference of two non-negatives is always // representable. Whenever Java methods must perform the equivalent // check they generally use Plan B instead of Plan A. // For the moment we use Plan A. inline Node* LibraryCallKit::generate_limit_guard(Node* offset, Node* subseq_length, Node* array_length, RegionNode* region) { if (stopped()) return NULL; // already stopped bool zero_offset = _gvn.type(offset) == TypeInt::ZERO; if (zero_offset && subseq_length->eqv_uncast(array_length)) return NULL; // common case of whole-array copy Node* last = subseq_length; if (!zero_offset) // last += offset last = _gvn.transform(new AddINode(last, offset)); Node* cmp_lt = _gvn.transform(new CmpUNode(array_length, last)); Node* bol_lt = _gvn.transform(new BoolNode(cmp_lt, BoolTest::lt)); Node* is_over = generate_guard(bol_lt, region, PROB_MIN); return is_over; } // Emit range checks for the given String.value byte array void LibraryCallKit::generate_string_range_check(Node* array, Node* offset, Node* coun if (stopped()) { return; // already stopped } RegionNode* bailout = new RegionNode(1); record_for_igvn(bailout); if (char_count) { // Convert char count to byte count count = _gvn.transform(new LShiftINode(count, intcon(1))); } // Offset and count must not be negative generate_negative_guard(offset, bailout); generate_negative_guard(count, bailout); // Offset + count must not exceed length of array generate_limit_guard(offset, count, load_array_length(array), bailout); if (bailout->req() > 1) { PreserveJVMState pjvms(this); set_control(_gvn.transform(bailout)); uncommon_trap(Deoptimization::Reason_intrinsic, Deoptimization::Action_maybe_recompile); } } Node* LibraryCallKit::current_thread_helper(Node*& tls_output, ByteSize handle_o bool is_immutable) { ciKlass* thread_klass = env()->Thread_klass(); const Type* thread_type = TypeOopPtr::make_from_klass(thread_klass)->cast_to_ptr_type(TypePtr::NotNull); Node* thread = _gvn.transform(new ThreadLocalNode()); Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(handle_offset)); tls_output = thread; Node* thread_obj_handle = (is_immutable ? LoadNode::make(_gvn, NULL, immutable_memory(), p, p->bottom_type()->is_ptr(), TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered) : make_load(NULL, p, p->bottom_type()->is_ptr(), T_ADDRESS, MemNode::unordered)); thread_obj_handle = _gvn.transform(thread_obj_handle); DecoratorSet decorators = IN_NATIVE; if (is_immutable) { decorators |= C2_IMMUTABLE_MEMORY; } return access_load(thread_obj_handle, thread_type, T_OBJECT, decorators); } //--------------------------generate_current_thread-------------------- Node* LibraryCallKit::generate_current_thread(Node* &tls_output) { return current_thread_helper(tls_output, JavaThread::threadObj_offset(), /*is_immutable*/false); } //--------------------------generate_virtual_thread-------------------- Node* LibraryCallKit::generate_virtual_thread(Node* tls_output) { return current_thread_helper(tls_output, JavaThread::vthread_offset(), !C->method()->changes_current_thread()); } //------------------------------make_string_method_node------------------------ // Helper method for String intrinsic functions. This version is called with // str1 and str2 pointing to byte[] nodes containing Latin1 or UTF16 encoded // characters (depending on 'is_byte'). cnt1 and cnt2 are pointing to Int nodes // containing the lengths of str1 and str2. Node* LibraryCallKit::make_string_method_node(int opcode, Node* str1_start, Node* cnt1 Node* result = NULL; switch (opcode) { case Op_StrIndexOf: result = new StrIndexOfNode(control(), memory(TypeAryPtr::BYTES), str1_start, cnt1, str2_start, cnt2, ae); break; case Op_StrComp: result = new StrCompNode(control(), memory(TypeAryPtr::BYTES), str1_start, cnt1, str2_start, cnt2, ae); break; case Op_StrEquals: // We already know that cnt1 == cnt2 here (checked in 'inline_string_equals'). // Use the constant length if there is one because optimized match rule may exist. result = new StrEqualsNode(control(), memory(TypeAryPtr::BYTES), str1_start, str2_start, cnt2->is_Con() ? cnt2 : cnt1, ae); break; default: ShouldNotReachHere(); return NULL; } // All these intrinsics have checks. C->set_has_split_ifs(true); // Has chance for split-if optimization clear_upper_avx(); return _gvn.transform(result); } //------------------------------inline_string_compareTo------------------------ bool LibraryCallKit::inline_string_compareTo(StrIntrinsicNode::ArgEnc ae) { Node* arg1 = argument(0); Node* arg2 = argument(1); arg1 = must_be_not_null(arg1, true); arg2 = must_be_not_null(arg2, true); // Get start addr and length of first argument Node* arg1_start = array_element_address(arg1, intcon(0), T_BYTE); Node* arg1_cnt = load_array_length(arg1); // Get start addr and length of second argument Node* arg2_start = array_element_address(arg2, intcon(0), T_BYTE); Node* arg2_cnt = load_array_length(arg2); Node* result = make_string_method_node(Op_StrComp, arg1_start, arg1_cnt, arg2_start, ar set_result(result); return true; } //------------------------------inline_string_equals------------------------ bool LibraryCallKit::inline_string_equals(StrIntrinsicNode::ArgEnc ae) { Node* arg1 = argument(0); Node* arg2 = argument(1); // paths (plus control) merge RegionNode* region = new RegionNode(3); Node* phi = new PhiNode(region, TypeInt::BOOL); if (!stopped()) { arg1 = must_be_not_null(arg1, true); arg2 = must_be_not_null(arg2, true); // Get start addr and length of first argument Node* arg1_start = array_element_address(arg1, intcon(0), T_BYTE); Node* arg1_cnt = load_array_length(arg1); // Get start addr and length of second argument Node* arg2_start = array_element_address(arg2, intcon(0), T_BYTE); Node* arg2_cnt = load_array_length(arg2); // Check for arg1_cnt != arg2_cnt Node* cmp = _gvn.transform(new CmpINode(arg1_cnt, arg2_cnt)); Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::ne)); Node* if_ne = generate_slow_guard(bol, NULL); if (if_ne != NULL) { phi->init_req(2, intcon(0)); region->init_req(2, if_ne); } // Check for count == 0 is done by assembler code for StrEquals. if (!stopped()) { Node* equals = make_string_method_node(Op_StrEquals, arg1_start, arg1_cnt, arg2_start, phi->init_req(1, equals); region->init_req(1, control()); } } // post merge set_control(_gvn.transform(region)); record_for_igvn(region); set_result(_gvn.transform(phi)); return true; } //------------------------------inline_array_equals---------------------------- bool LibraryCallKit::inline_array_equals(StrIntrinsicNode::ArgEnc ae) { assert(ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::LL, "unsupported array types Node* arg1 = argument(0); Node* arg2 = argument(1); const TypeAryPtr* mtype = (ae == StrIntrinsicNode::UU) ? TypeAryPtr::CHARS : TypeAryPtr::B set_result(_gvn.transform(new AryEqNode(control(), memory(mtype), arg1, arg2, ae))); clear_upper_avx(); return true; } //------------------------------inline_countPositives--------------------------- bool LibraryCallKit::inline_countPositives() { if (too_many_traps(Deoptimization::Reason_intrinsic)) { return false; } assert(callee()->signature()->size() == 3, "countPositives has 3 parameters"); // no receiver since it is static method Node* ba = argument(0); Node* offset = argument(1); Node* len = argument(2); ba = must_be_not_null(ba, true); // Range checks generate_string_range_check(ba, offset, len, false); if (stopped()) { return true; } Node* ba_start = array_element_address(ba, offset, T_BYTE); Node* result = new CountPositivesNode(control(), memory(TypeAryPtr::BYTES), ba_start, l set_result(_gvn.transform(result)); return true; } bool LibraryCallKit::inline_preconditions_checkIndex(BasicType bt) { Node* index = argument(0); Node* length = bt == T_INT ? argument(1) : argument(2); if (too_many_traps(Deoptimization::Reason_intrinsic) || too_many_traps(Deoptimizati return false; } // check that length is positive Node* len_pos_cmp = _gvn.transform(CmpNode::make(length, integercon(0, bt), bt)); Node* len_pos_bol = _gvn.transform(new BoolNode(len_pos_cmp, BoolTest::ge)); { BuildCutout unless(this, len_pos_bol, PROB_MAX); uncommon_trap(Deoptimization::Reason_intrinsic, Deoptimization::Action_make_not_entrant); } if (stopped()) { // Length is known to be always negative during compilation and the IR graph so far constructed i return true; } // length is now known positive, add a cast node to make this explicit jlong upper_bound = _gvn.type(length)->is_integer(bt)->hi_as_long(); Node* casted_length = ConstraintCastNode::make(control(), length, TypeInteger::make(0 casted_length = _gvn.transform(casted_length); replace_in_map(length, casted_length); length = casted_length; // Use an unsigned comparison for the range check itself Node* rc_cmp = _gvn.transform(CmpNode::make(index, length, bt, true)); BoolTest::mask btest = BoolTest::lt; Node* rc_bool = _gvn.transform(new BoolNode(rc_cmp, btest)); RangeCheckNode* rc = new RangeCheckNode(control(), rc_bool, PROB_MAX, COUNT_UNKNOWN); _gvn.set_type(rc, rc->Value(&_gvn)); if (!rc_bool->is_Con()) { record_for_igvn(rc); } set_control(_gvn.transform(new IfTrueNode(rc))); { PreserveJVMState pjvms(this); set_control(_gvn.transform(new IfFalseNode(rc))); uncommon_trap(Deoptimization::Reason_range_check, Deoptimization::Action_make_not_entrant); } if (stopped()) { // Range check is known to always fail during compilation and the IR graph so far constructed is g return true; } // index is now known to be >= 0 and < length, cast it Node* result = ConstraintCastNode::make(control(), index, TypeInteger::make(0, upper_b result = _gvn.transform(result); set_result(result); replace_in_map(index, result); return true; } //------------------------------inline_string_indexOf------------------------ bool LibraryCallKit::inline_string_indexOf(StrIntrinsicNode::ArgEnc ae) { if (!Matcher::match_rule_supported(Op_StrIndexOf)) { return false; } Node* src = argument(0); Node* tgt = argument(1); // Make the merge point RegionNode* result_rgn = new RegionNode(4); Node* result_phi = new PhiNode(result_rgn, TypeInt::INT); src = must_be_not_null(src, true); tgt = must_be_not_null(tgt, true); // Get start addr and length of source string Node* src_start = array_element_address(src, intcon(0), T_BYTE); Node* src_count = load_array_length(src); // Get start addr and length of substring Node* tgt_start = array_element_address(tgt, intcon(0), T_BYTE); Node* tgt_count = load_array_length(tgt); if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) { // Divide src size by 2 if String is UTF16 encoded src_count = _gvn.transform(new RShiftINode(src_count, intcon(1))); } if (ae == StrIntrinsicNode::UU) { // Divide substring size by 2 if String is UTF16 encoded tgt_count = _gvn.transform(new RShiftINode(tgt_count, intcon(1))); } Node* result = make_indexOf_node(src_start, src_count, tgt_start, tgt_count, result_rgn if (result != NULL) { result_phi->init_req(3, result); result_rgn->init_req(3, control()); } set_control(_gvn.transform(result_rgn)); record_for_igvn(result_rgn); set_result(_gvn.transform(result_phi)); return true; } //-----------------------------inline_string_indexOf----------------------- bool LibraryCallKit::inline_string_indexOfI(StrIntrinsicNode::ArgEnc ae) { if (too_many_traps(Deoptimization::Reason_intrinsic)) { return false; } if (!Matcher::match_rule_supported(Op_StrIndexOf)) { return false; } assert(callee()->signature()->size() == 5, "String.indexOf() has 5 arguments"); Node* src = argument(0); // byte[] Node* src_count = argument(1); // char count Node* tgt = argument(2); // byte[] Node* tgt_count = argument(3); // char count Node* from_index = argument(4); // char index src = must_be_not_null(src, true); tgt = must_be_not_null(tgt, true); // Multiply byte array index by 2 if String is UTF16 encoded Node* src_offset = (ae == StrIntrinsicNode::LL) ? from_index : _gvn.transform(new LShiftIN src_count = _gvn.transform(new SubINode(src_count, from_index)); Node* src_start = array_element_address(src, src_offset, T_BYTE); Node* tgt_start = array_element_address(tgt, intcon(0), T_BYTE); // Range checks generate_string_range_check(src, src_offset, src_count, ae != StrIntrinsicNode::LL); generate_string_range_check(tgt, intcon(0), tgt_count, ae == StrIntrinsicNode::UU); if (stopped()) { return true; } RegionNode* region = new RegionNode(5); Node* phi = new PhiNode(region, TypeInt::INT); Node* result = make_indexOf_node(src_start, src_count, tgt_start, tgt_count, region, phi if (result != NULL) { // The result is index relative to from_index if substring was found, -1 otherwise. // Generate code which will fold into cmove. Node* cmp = _gvn.transform(new CmpINode(result, intcon(0))); Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::lt)); Node* if_lt = generate_slow_guard(bol, NULL); if (if_lt != NULL) { // result == -1 phi->init_req(3, result); region->init_req(3, if_lt); } if (!stopped()) { result = _gvn.transform(new AddINode(result, from_index)); phi->init_req(4, result); region->init_req(4, control()); } } set_control(_gvn.transform(region)); record_for_igvn(region); set_result(_gvn.transform(phi)); clear_upper_avx(); return true; } // Create StrIndexOfNode with fast path checks Node* LibraryCallKit::make_indexOf_node(Node* src_start, Node* src_count, Node* tgt_st RegionNode* region, Node* phi, StrIntrinsicNode::ArgEnc ae) { // Check for substr count > string count Node* cmp = _gvn.transform(new CmpINode(tgt_count, src_count)); Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::gt)); Node* if_gt = generate_slow_guard(bol, NULL); if (if_gt != NULL) { phi->init_req(1, intcon(-1)); region->init_req(1, if_gt); } if (!stopped()) { // Check for substr count == 0 cmp = _gvn.transform(new CmpINode(tgt_count, intcon(0))); bol = _gvn.transform(new BoolNode(cmp, BoolTest::eq)); Node* if_zero = generate_slow_guard(bol, NULL); if (if_zero != NULL) { phi->init_req(2, intcon(0)); region->init_req(2, if_zero); } } if (!stopped()) { return make_string_method_node(Op_StrIndexOf, src_start, src_count, tgt_start, tgt_co } return NULL; } //-----------------------------inline_string_indexOfChar----------------------- bool LibraryCallKit::inline_string_indexOfChar(StrIntrinsicNode::ArgEnc ae) { if (too_many_traps(Deoptimization::Reason_intrinsic)) { return false; } if (!Matcher::match_rule_supported(Op_StrIndexOfChar)) { return false; } assert(callee()->signature()->size() == 4, "String.indexOfChar() has 4 arguments"); Node* src = argument(0); // byte[] Node* tgt = argument(1); // tgt is int ch Node* from_index = argument(2); Node* max = argument(3); src = must_be_not_null(src, true); Node* src_offset = ae == StrIntrinsicNode::L ? from_index : _gvn.transform(new LShiftINode Node* src_start = array_element_address(src, src_offset, T_BYTE); Node* src_count = _gvn.transform(new SubINode(max, from_index)); // Range checks generate_string_range_check(src, src_offset, src_count, ae == StrIntrinsicNode::U); if (stopped()) { return true; } RegionNode* region = new RegionNode(3); Node* phi = new PhiNode(region, TypeInt::INT); Node* result = new StrIndexOfCharNode(control(), memory(TypeAryPtr::BYTES), src_start, C->set_has_split_ifs(true); // Has chance for split-if optimization _gvn.transform(result); Node* cmp = _gvn.transform(new CmpINode(result, intcon(0))); Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::lt)); Node* if_lt = generate_slow_guard(bol, NULL); if (if_lt != NULL) { // result == -1 phi->init_req(2, result); region->init_req(2, if_lt); } if (!stopped()) { result = _gvn.transform(new AddINode(result, from_index)); phi->init_req(1, result); region->init_req(1, control()); } set_control(_gvn.transform(region)); record_for_igvn(region); set_result(_gvn.transform(phi)); return true; } //---------------------------inline_string_copy--------------------- // compressIt == true --> generate a compressed copy operation (compress char[]/byte[] to byte // int StringUTF16.compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) // int StringUTF16.compress(byte[] src, int srcOff, byte[] dst, int dstOff, int len) // compressIt == false --> generate an inflated copy operation (inflate byte[] to char[]/byte[ // void StringLatin1.inflate(byte[] src, int srcOff, char[] dst, int dstOff, int len) // void StringLatin1.inflate(byte[] src, int srcOff, byte[] dst, int dstOff, int len) bool LibraryCallKit::inline_string_copy(bool compress) { if (too_many_traps(Deoptimization::Reason_intrinsic)) { return false; } int nargs = 5; // 2 oops, 3 ints assert(callee()->signature()->size() == nargs, "string copy has 5 arguments"); Node* src = argument(0); Node* src_offset = argument(1); Node* dst = argument(2); Node* dst_offset = argument(3); Node* length = argument(4); // Check for allocation before we add nodes that would confuse // tightly_coupled_allocation() AllocateArrayNode* alloc = tightly_coupled_allocation(dst); // Figure out the size and type of the elements we will be copying. const Type* src_type = src->Value(&_gvn); const Type* dst_type = dst->Value(&_gvn); BasicType src_elem = src_type->isa_aryptr()->elem()->array_element_basic_type(); BasicType dst_elem = dst_type->isa_aryptr()->elem()->array_element_basic_type(); assert((compress && dst_elem == T_BYTE && (src_elem == T_BYTE || src_elem == T_CHAR)) || (!compress && src_elem == T_BYTE && (dst_elem == T_BYTE || dst_elem == T_CHAR)), "Unsupported array types for inline_string_copy"); src = must_be_not_null(src, true); dst = must_be_not_null(dst, true); // Convert char[] offsets to byte[] offsets bool convert_src = (compress && src_elem == T_BYTE); bool convert_dst = (!compress && dst_elem == T_BYTE); if (convert_src) { src_offset = _gvn.transform(new LShiftINode(src_offset, intcon(1))); } else if (convert_dst) { dst_offset = _gvn.transform(new LShiftINode(dst_offset, intcon(1))); } // Range checks generate_string_range_check(src, src_offset, length, convert_src); generate_string_range_check(dst, dst_offset, length, convert_dst); if (stopped()) { return true; } Node* src_start = array_element_address(src, src_offset, src_elem); Node* dst_start = array_element_address(dst, dst_offset, dst_elem); // 'src_start' points to src array + scaled offset // 'dst_start' points to dst array + scaled offset Node* count = NULL; if (compress) { count = compress_string(src_start, TypeAryPtr::get_array_body_type(src_elem), dst_st } else { inflate_string(src_start, dst_start, TypeAryPtr::get_array_body_type(dst_elem), len } if (alloc != NULL) { if (alloc->maybe_set_complete(&_gvn)) { // "You break it, you buy it." InitializeNode* init = alloc->initialization(); assert(init->is_complete(), "we just did this"); init->set_complete_with_arraycopy(); assert(dst->is_CheckCastPP(), "sanity"); assert(dst->in(0)->in(0) == init, "dest pinned"); } // Do not let stores that initialize this object be reordered with // a subsequent store that would make this object accessible by // other threads. // Record what AllocateNode this StoreStore protects so that // escape analysis can go from the MemBarStoreStoreNode to the // AllocateNode and eliminate the MemBarStoreStoreNode if possible // based on the escape status of the AllocateNode. insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddre } if (compress) { set_result(_gvn.transform(count)); } clear_upper_avx(); return true; } #ifdef _LP64 #define XTOP ,top() /*additional argument*/ #else //_LP64 #define XTOP /*no additional argument*/ #endif //_LP64 //------------------------inline_string_toBytesU-------------------------- // public static byte[] StringUTF16.toBytes(char[] value, int off, int len) bool LibraryCallKit::inline_string_toBytesU() { if (too_many_traps(Deoptimization::Reason_intrinsic)) { return false; } // Get the arguments. Node* value = argument(0); Node* offset = argument(1); Node* length = argument(2); Node* newcopy = NULL; // Set the original stack and the reexecute bit for the interpreter to reexecute // the bytecode that invokes StringUTF16.toBytes() if deoptimization happens. { PreserveReexecuteState preexecs(this); jvms()->set_should_reexecute(true); // Check if a null path was taken unconditionally. value = null_check(value); RegionNode* bailout = new RegionNode(1); record_for_igvn(bailout); // Range checks generate_negative_guard(offset, bailout); generate_negative_guard(length, bailout); generate_limit_guard(offset, length, load_array_length(value), bailout); // Make sure that resulting byte[] length does not overflow Integer.MAX_VALUE generate_limit_guard(length, intcon(0), intcon(max_jint/2), bailout); if (bailout->req() > 1) { PreserveJVMState pjvms(this); set_control(_gvn.transform(bailout)); uncommon_trap(Deoptimization::Reason_intrinsic, Deoptimization::Action_maybe_recompile); } if (stopped()) { return true; } Node* size = _gvn.transform(new LShiftINode(length, intcon(1))); Node* klass_node = makecon(TypeKlassPtr::make(ciTypeArrayKlass::make(T_BYTE))); newcopy = new_array(klass_node, size, 0); // no arguments to push AllocateArrayNode* alloc = tightly_coupled_allocation(newcopy); guarantee(alloc != NULL, "created above"); // Calculate starting addresses. Node* src_start = array_element_address(value, offset, T_CHAR); Node* dst_start = basic_plus_adr(newcopy, arrayOopDesc::base_offset_in_bytes(T_BYTE) // Check if src array address is aligned to HeapWordSize (dst is always aligned) const TypeInt* toffset = gvn().type(offset)->is_int(); bool aligned = toffset->is_con() && ((toffset->get_con() * type2aelembytes(T_CHAR)) % HeapWor // Figure out which arraycopy runtime method to call (disjoint, uninitialized). const char* copyfunc_name = "arraycopy"; address copyfunc_addr = StubRoutines::select_arraycopy_function(T_CHAR, aligned, true Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::fast_arraycopy_Type(), copyfunc_addr, copyfunc_name, TypeRawPtr::BOTTOM, src_start, dst_start, ConvI2X(length) XTOP); // Do not let reads from the cloned object float above the arraycopy. if (alloc->maybe_set_complete(&_gvn)) { // "You break it, you buy it." InitializeNode* init = alloc->initialization(); assert(init->is_complete(), "we just did this"); init->set_complete_with_arraycopy(); assert(newcopy->is_CheckCastPP(), "sanity"); assert(newcopy->in(0)->in(0) == init, "dest pinned"); } // Do not let stores that initialize this object be reordered with // a subsequent store that would make this object accessible by // other threads. // Record what AllocateNode this StoreStore protects so that // escape analysis can go from the MemBarStoreStoreNode to the // AllocateNode and eliminate the MemBarStoreStoreNode if possible // based on the escape status of the AllocateNode. insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddre } // original reexecute is set back here C->set_has_split_ifs(true); // Has chance for split-if optimization if (!stopped()) { set_result(newcopy); } clear_upper_avx(); return true; } //------------------------inline_string_getCharsU-------------------------- // public void StringUTF16.getChars(byte[] src, int srcBegin, int srcEnd, char dst[], int ds bool LibraryCallKit::inline_string_getCharsU() { if (too_many_traps(Deoptimization::Reason_intrinsic)) { return false; } // Get the arguments. Node* src = argument(0); Node* src_begin = argument(1); Node* src_end = argument(2); // exclusive offset (i < src_end) Node* dst = argument(3); Node* dst_begin = argument(4); // Check for allocation before we add nodes that would confuse // tightly_coupled_allocation() AllocateArrayNode* alloc = tightly_coupled_allocation(dst); // Check if a null path was taken unconditionally. src = null_check(src); dst = null_check(dst); if (stopped()) { return true; } // Get length and convert char[] offset to byte[] offset Node* length = _gvn.transform(new SubINode(src_end, src_begin)); src_begin = _gvn.transform(new LShiftINode(src_begin, intcon(1))); // Range checks generate_string_range_check(src, src_begin, length, true); generate_string_range_check(dst, dst_begin, length, false); if (stopped()) { return true; } if (!stopped()) { // Calculate starting addresses. Node* src_start = array_element_address(src, src_begin, T_BYTE); Node* dst_start = array_element_address(dst, dst_begin, T_CHAR); // Check if array addresses are aligned to HeapWordSize const TypeInt* tsrc = gvn().type(src_begin)->is_int(); const TypeInt* tdst = gvn().type(dst_begin)->is_int(); bool aligned = tsrc->is_con() && ((tsrc->get_con() * type2aelembytes(T_BYTE)) % HeapWordSize = tdst->is_con() && ((tdst->get_con() * type2aelembytes(T_CHAR)) % HeapWordSize == 0); // Figure out which arraycopy runtime method to call (disjoint, uninitialized). const char* copyfunc_name = "arraycopy"; address copyfunc_addr = StubRoutines::select_arraycopy_function(T_CHAR, aligned, true Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::fast_arraycopy_Type(), copyfunc_addr, copyfunc_name, TypeRawPtr::BOTTOM, src_start, dst_start, ConvI2X(length) XTOP); // Do not let reads from the cloned object float above the arraycopy. if (alloc != NULL) { if (alloc->maybe_set_complete(&_gvn)) { // "You break it, you buy it." InitializeNode* init = alloc->initialization(); assert(init->is_complete(), "we just did this"); init->set_complete_with_arraycopy(); assert(dst->is_CheckCastPP(), "sanity"); assert(dst->in(0)->in(0) == init, "dest pinned"); } // Do not let stores that initialize this object be reordered with // a subsequent store that would make this object accessible by // other threads. // Record what AllocateNode this StoreStore protects so that // escape analysis can go from the MemBarStoreStoreNode to the // AllocateNode and eliminate the MemBarStoreStoreNode if possible // based on the escape status of the AllocateNode. insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddre } else { insert_mem_bar(Op_MemBarCPUOrder); } } C->set_has_split_ifs(true); // Has chance for split-if optimization return true; } //----------------------inline_string_char_access---------------------------- // Store/Load char to/from byte[] array. // static void StringUTF16.putChar(byte[] val, int index, int c) // static char StringUTF16.getChar(byte[] val, int index) bool LibraryCallKit::inline_string_char_access(bool is_store) { Node* value = argument(0); Node* index = argument(1); Node* ch = is_store ? argument(2) : NULL; // This intrinsic accesses byte[] array as char[] array. Computing the offsets // correctly requires matched array shapes. assert (arrayOopDesc::base_offset_in_bytes(T_CHAR) == arrayOopDesc::base_offset_in_ "sanity: byte[] and char[] bases agree"); assert (type2aelembytes(T_CHAR) == type2aelembytes(T_BYTE)*2, "sanity: byte[] and char[] scales agree"); // Bail when getChar over constants is requested: constant folding would // reject folding mismatched char access over byte[]. A normal inlining for getChar // Java method would constant fold nicely instead. if (!is_store && value->is_Con() && index->is_Con()) { return false; } // Save state and restore on bailout uint old_sp = sp(); SafePointNode* old_map = clone_map(); value = must_be_not_null(value, true); Node* adr = array_element_address(value, index, T_CHAR); if (adr->is_top()) { set_map(old_map); set_sp(old_sp); return false; } old_map->destruct(&_gvn); if (is_store) { access_store_at(value, adr, TypeAryPtr::BYTES, ch, TypeInt::CHAR, T_CHAR, IN_HEAP | MO_U } else { ch = access_load_at(value, adr, TypeAryPtr::BYTES, TypeInt::CHAR, T_CHAR, IN_HEAP | MO_UN set_result(ch); } return true; } //--------------------------round_double_node-------------------------------- // Round a double node if necessary. Node* LibraryCallKit::round_double_node(Node* n) { if (Matcher::strict_fp_requires_explicit_rounding) { #ifdef IA32 if (UseSSE < 2) { n = _gvn.transform(new RoundDoubleNode(NULL, n)); } #else Unimplemented(); #endif // IA32 } return n; } //------------------------------inline_math----------------------------------- // public static double Math.abs(double) // public static double Math.sqrt(double) // public static double Math.log(double) // public static double Math.log10(double) // public static double Math.round(double) bool LibraryCallKit::inline_double_math(vmIntrinsics::ID id) { Node* arg = round_double_node(argument(0)); Node* n = NULL; switch (id) { case vmIntrinsics::_dabs: n = new AbsDNode( arg); break; case vmIntrinsics::_dsqrt: case vmIntrinsics::_dsqrt_strict: n = new SqrtDNode(C, control(), arg); break; case vmIntrinsics::_ceil: n = RoundDoubleModeNode::make(_gvn, arg, RoundDoubleModeNode case vmIntrinsics::_floor: n = RoundDoubleModeNode::make(_gvn, arg, RoundDoubleModeNod case vmIntrinsics::_rint: n = RoundDoubleModeNode::make(_gvn, arg, RoundDoubleModeNode case vmIntrinsics::_roundD: n = new RoundDNode(arg); break; case vmIntrinsics::_dcopySign: n = CopySignDNode::make(_gvn, arg, round_double_node(ar case vmIntrinsics::_dsignum: n = SignumDNode::make(_gvn, arg); break; default: fatal_unexpected_iid(id); break; } set_result(_gvn.transform(n)); return true; } //------------------------------inline_math----------------------------------- // public static float Math.abs(float) // public static int Math.abs(int) // public static long Math.abs(long) bool LibraryCallKit::inline_math(vmIntrinsics::ID id) { Node* arg = argument(0); Node* n = NULL; switch (id) { case vmIntrinsics::_fabs: n = new AbsFNode( arg); break; case vmIntrinsics::_iabs: n = new AbsINode( arg); break; case vmIntrinsics::_labs: n = new AbsLNode( arg); break; case vmIntrinsics::_fcopySign: n = new CopySignFNode(arg, argument(1)); break; case vmIntrinsics::_fsignum: n = SignumFNode::make(_gvn, arg); break; case vmIntrinsics::_roundF: n = new RoundFNode(arg); break; default: fatal_unexpected_iid(id); break; } set_result(_gvn.transform(n)); return true; } //------------------------------runtime_math----------------------------- bool LibraryCallKit::runtime_math(const TypeFunc* call_type, address funcAddr, const ch assert(call_type == OptoRuntime::Math_DD_D_Type() || call_type == OptoRuntime::Math_D_ "must be (DD)D or (D)D type"); // Inputs Node* a = round_double_node(argument(0)); Node* b = (call_type == OptoRuntime::Math_DD_D_Type()) ? round_double_node(argument(2)) const TypePtr* no_memory_effects = NULL; Node* trig = make_runtime_call(RC_LEAF, call_type, funcAddr, funcName, no_memory_effects, a, top(), b, b ? top() : NULL); Node* value = _gvn.transform(new ProjNode(trig, TypeFunc::Parms+0)); #ifdef ASSERT Node* value_top = _gvn.transform(new ProjNode(trig, TypeFunc::Parms+1)); assert(value_top == top(), "second value must be top"); #endif set_result(value); return true; } //------------------------------inline_math_pow----------------------------- bool LibraryCallKit::inline_math_pow() { Node* exp = round_double_node(argument(2)); const TypeD* d = _gvn.type(exp)->isa_double_constant(); if (d != NULL) { if (d->getd() == 2.0) { // Special case: pow(x, 2.0) => x * x Node* base = round_double_node(argument(0)); set_result(_gvn.transform(new MulDNode(base, base))); return true; } else if (d->getd() == 0.5 && Matcher::match_rule_supported(Op_SqrtD)) { // Special case: pow(x, 0.5) => sqrt(x) Node* base = round_double_node(argument(0)); Node* zero = _gvn.zerocon(T_DOUBLE); RegionNode* region = new RegionNode(3); Node* phi = new PhiNode(region, Type::DOUBLE); Node* cmp = _gvn.transform(new CmpDNode(base, zero)); // According to the API specs, pow(-0.0, 0.5) = 0.0 and sqrt(-0.0) = -0.0. // So pow(-0.0, 0.5) shouldn't be replaced with sqrt(-0.0). // -0.0/+0.0 are both excluded since floating-point comparison doesn't distinguish -0.0 fro Node* test = _gvn.transform(new BoolNode(cmp, BoolTest::le)); Node* if_pow = generate_slow_guard(test, NULL); Node* value_sqrt = _gvn.transform(new SqrtDNode(C, control(), base)); phi->init_req(1, value_sqrt); region->init_req(1, control()); if (if_pow != NULL) { set_control(if_pow); address target = StubRoutines::dpow() != NULL ? StubRoutines::dpow() : CAST_FROM_FN_PTR(address, SharedRuntime::dpow); const TypePtr* no_memory_effects = NULL; Node* trig = make_runtime_call(RC_LEAF, OptoRuntime::Math_DD_D_Type(), target, "POW", no_memory_effects, base, top(), exp, top()); Node* value_pow = _gvn.transform(new ProjNode(trig, TypeFunc::Parms+0)); #ifdef ASSERT Node* value_top = _gvn.transform(new ProjNode(trig, TypeFunc::Parms+1)); assert(value_top == top(), "second value must be top"); #endif phi->init_req(2, value_pow); region->init_req(2, _gvn.transform(new ProjNode(trig, TypeFunc::Control))); } C->set_has_split_ifs(true); // Has chance for split-if optimization set_control(_gvn.transform(region)); record_for_igvn(region); set_result(_gvn.transform(phi)); return true; } } return StubRoutines::dpow() != NULL ? runtime_math(OptoRuntime::Math_DD_D_Type(), StubRoutines::dpow(), "dpow") : runtime_math(OptoRuntime::Math_DD_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntim } //------------------------------inline_math_native----------------------------- bool LibraryCallKit::inline_math_native(vmIntrinsics::ID id) { switch (id) { case vmIntrinsics::_dsin: return StubRoutines::dsin() != NULL ? runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dsin(), "dsin") : runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime case vmIntrinsics::_dcos: return StubRoutines::dcos() != NULL ? runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dcos(), "dcos") : runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime case vmIntrinsics::_dtan: return StubRoutines::dtan() != NULL ? runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dtan(), "dtan") : runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime case vmIntrinsics::_dexp: return StubRoutines::dexp() != NULL ? runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dexp(), "dexp") : runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime case vmIntrinsics::_dlog: return StubRoutines::dlog() != NULL ? runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dlog(), "dlog") : runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime case vmIntrinsics::_dlog10: return StubRoutines::dlog10() != NULL ? runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dlog10(), "dlog10") : runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime case vmIntrinsics::_roundD: return Matcher::match_rule_supported(Op_RoundD) ? inline_ case vmIntrinsics::_ceil: case vmIntrinsics::_floor: case vmIntrinsics::_rint: return Matcher::match_rule_supported(Op_RoundDoubleMode) ? case vmIntrinsics::_dsqrt: case vmIntrinsics::_dsqrt_strict: return Matcher::match_rule_supported(Op_SqrtD) ? inline_double_math(id) : false; case vmIntrinsics::_dabs: return Matcher::has_match_rule(Op_AbsD) ? inline_double_mat case vmIntrinsics::_fabs: return Matcher::match_rule_supported(Op_AbsF) ? inline_math case vmIntrinsics::_iabs: return Matcher::match_rule_supported(Op_AbsI) ? inline_math case vmIntrinsics::_labs: return Matcher::match_rule_supported(Op_AbsL) ? inline_math case vmIntrinsics::_dpow: return inline_math_pow(); case vmIntrinsics::_dcopySign: return inline_double_math(id); case vmIntrinsics::_fcopySign: return inline_math(id); case vmIntrinsics::_dsignum: return Matcher::match_rule_supported(Op_SignumD) ? inlin case vmIntrinsics::_fsignum: return Matcher::match_rule_supported(Op_SignumF) ? inlin case vmIntrinsics::_roundF: return Matcher::match_rule_supported(Op_RoundF) ? inline_ // These intrinsics are not yet correctly implemented case vmIntrinsics::_datan2: return false; default: fatal_unexpected_iid(id); return false; } } //----------------------------inline_notify-----------------------------------* bool LibraryCallKit::inline_notify(vmIntrinsics::ID id) { const TypeFunc* ftype = OptoRuntime::monitor_notify_Type(); address func; if (id == vmIntrinsics::_notify) { func = OptoRuntime::monitor_notify_Java(); } else { func = OptoRuntime::monitor_notifyAll_Java(); } Node* call = make_runtime_call(RC_NO_LEAF, ftype, func, NULL, TypeRawPtr::BOTTOM, argume make_slow_call_ex(call, env()->Throwable_klass(), false); return true; } //----------------------------inline_min_max----------------------------------- bool LibraryCallKit::inline_min_max(vmIntrinsics::ID id) { set_result(generate_min_max(id, argument(0), argument(1))); return true; } void LibraryCallKit::inline_math_mathExact(Node* math, Node *test) { Node* bol = _gvn.transform( new BoolNode(test, BoolTest::overflow) ); IfNode* check = create_and_map_if(control(), bol, PROB_UNLIKELY_MAG(3), COUNT_UNKNOWN) Node* fast_path = _gvn.transform( new IfFalseNode(check)); Node* slow_path = _gvn.transform( new IfTrueNode(check) ); { PreserveJVMState pjvms(this); PreserveReexecuteState preexecs(this); jvms()->set_should_reexecute(true); set_control(slow_path); set_i_o(i_o()); uncommon_trap(Deoptimization::Reason_intrinsic, Deoptimization::Action_none); } set_control(fast_path); set_result(math); } template <typename OverflowOp> bool LibraryCallKit::inline_math_overflow(Node* arg1, Node* arg2) { typedef typename OverflowOp::MathOp MathOp; MathOp* mathOp = new MathOp(arg1, arg2); Node* operation = _gvn.transform( mathOp ); Node* ofcheck = _gvn.transform( new OverflowOp(arg1, arg2) ); inline_math_mathExact(operation, ofcheck); return true; } bool LibraryCallKit::inline_math_addExactI(bool is_increment) { return inline_math_overflow<OverflowAddINode>(argument(0), is_increment ? intcon(1) : ar } bool LibraryCallKit::inline_math_addExactL(bool is_increment) { return inline_math_overflow<OverflowAddLNode>(argument(0), is_increment ? longcon(1) : a } bool LibraryCallKit::inline_math_subtractExactI(bool is_decrement) { return inline_math_overflow<OverflowSubINode>(argument(0), is_decrement ? intcon(1) : ar } bool LibraryCallKit::inline_math_subtractExactL(bool is_decrement) { return inline_math_overflow<OverflowSubLNode>(argument(0), is_decrement ? longcon(1) : a } bool LibraryCallKit::inline_math_negateExactI() { return inline_math_overflow<OverflowSubINode>(intcon(0), argument(0)); } bool LibraryCallKit::inline_math_negateExactL() { return inline_math_overflow<OverflowSubLNode>(longcon(0), argument(0)); } bool LibraryCallKit::inline_math_multiplyExactI() { return inline_math_overflow<OverflowMulINode>(argument(0), argument(1)); } bool LibraryCallKit::inline_math_multiplyExactL() { return inline_math_overflow<OverflowMulLNode>(argument(0), argument(2)); } bool LibraryCallKit::inline_math_multiplyHigh() { set_result(_gvn.transform(new MulHiLNode(argument(0), argument(2)))); return true; } bool LibraryCallKit::inline_math_unsignedMultiplyHigh() { set_result(_gvn.transform(new UMulHiLNode(argument(0), argument(2)))); return true; } Node* LibraryCallKit::generate_min_max(vmIntrinsics::ID id, Node* x0, Node* y0) { Node* result_val = NULL; switch (id) { case vmIntrinsics::_min: case vmIntrinsics::_min_strict: result_val = _gvn.transform(new MinINode(x0, y0)); break; case vmIntrinsics::_max: case vmIntrinsics::_max_strict: result_val = _gvn.transform(new MaxINode(x0, y0)); break; default: fatal_unexpected_iid(id); break; } return result_val; } inline int LibraryCallKit::classify_unsafe_addr(Node* &base, Node* &offset, BasicType type) { const TypePtr* base_type = TypePtr::NULL_PTR; if (base != NULL) base_type = _gvn.type(base)->isa_ptr(); if (base_type == NULL) { // Unknown type. return Type::AnyPtr; } else if (base_type == TypePtr::NULL_PTR) { // Since this is a NULL+long form, we have to switch to a rawptr. base = _gvn.transform(new CastX2PNode(offset)); offset = MakeConX(0); return Type::RawPtr; } else if (base_type->base() == Type::RawPtr) { return Type::RawPtr; } else if (base_type->isa_oopptr()) { // Base is never null => always a heap address. if (!TypePtr::NULL_PTR->higher_equal(base_type)) { return Type::OopPtr; } // Offset is small => always a heap address. const TypeX* offset_type = _gvn.type(offset)->isa_intptr_t(); if (offset_type != NULL && base_type->offset() == 0 && // (should always be?) offset_type->_lo >= 0 && !MacroAssembler::needs_explicit_null_check(offset_type->_hi)) { return Type::OopPtr; } else if (type == T_OBJECT) { // off heap access to an oop doesn't make any sense. Has to be on // heap. return Type::OopPtr; } // Otherwise, it might either be oop+off or NULL+addr. return Type::AnyPtr; } else { // No information: return Type::AnyPtr; } } Node* LibraryCallKit::make_unsafe_address(Node*& base, Node* offset, BasicType typ Node* uncasted_base = base; int kind = classify_unsafe_addr(uncasted_base, offset, type); if (kind == Type::RawPtr) { return basic_plus_adr(top(), uncasted_base, offset); } else if (kind == Type::AnyPtr) { assert(base == uncasted_base, "unexpected base change"); if (can_cast) { if (!_gvn.type(base)->speculative_maybe_null() && !too_many_traps(Deoptimization::Reason_speculate_null_check)) { // According to profiling, this access is always on // heap. Casting the base to not null and thus avoiding membars // around the access should allow better optimizations Node* null_ctl = top(); base = null_check_oop(base, &null_ctl, true, true, true); assert(null_ctl->is_top(), "no null control here"); return basic_plus_adr(base, offset); } else if (_gvn.type(base)->speculative_always_null() && !too_many_traps(Deoptimization::Reason_speculate_null_assert)) { // According to profiling, this access is always off // heap. base = null_assert(base); Node* raw_base = _gvn.transform(new CastX2PNode(offset)); offset = MakeConX(0); return basic_plus_adr(top(), raw_base, offset); } } // We don't know if it's an on heap or off heap access. Fall back // to raw memory access. Node* raw = _gvn.transform(new CheckCastPPNode(control(), base, TypeRawPtr::BOTTOM)); return basic_plus_adr(top(), raw, offset); } else { assert(base == uncasted_base, "unexpected base change"); // We know it's an on heap access so base can't be null if (TypePtr::NULL_PTR->higher_equal(_gvn.type(base))) { base = must_be_not_null(base, true); } return basic_plus_adr(base, offset); } } //--------------------------inline_number_methods----------------------------- // inline int Integer.numberOfLeadingZeros(int) // inline int Long.numberOfLeadingZeros(long) // // inline int Integer.numberOfTrailingZeros(int) // inline int Long.numberOfTrailingZeros(long) // // inline int Integer.bitCount(int) // inline int Long.bitCount(long) // // inline char Character.reverseBytes(char) // inline short Short.reverseBytes(short) // inline int Integer.reverseBytes(int) // inline long Long.reverseBytes(long) bool LibraryCallKit::inline_number_methods(vmIntrinsics::ID id) { Node* arg = argument(0); Node* n = NULL; switch (id) { case vmIntrinsics::_numberOfLeadingZeros_i: n = new CountLeadingZerosINode( arg); break case vmIntrinsics::_numberOfLeadingZeros_l: n = new CountLeadingZerosLNode( arg); break case vmIntrinsics::_numberOfTrailingZeros_i: n = new CountTrailingZerosINode(arg); bre case vmIntrinsics::_numberOfTrailingZeros_l: n = new CountTrailingZerosLNode(arg); bre case vmIntrinsics::_bitCount_i: n = new PopCountINode( arg); break; case vmIntrinsics::_bitCount_l: n = new PopCountLNode( arg); break; case vmIntrinsics::_reverseBytes_c: n = new ReverseBytesUSNode(0, arg); break; case vmIntrinsics::_reverseBytes_s: n = new ReverseBytesSNode( 0, arg); break; case vmIntrinsics::_reverseBytes_i: n = new ReverseBytesINode( 0, arg); break; case vmIntrinsics::_reverseBytes_l: n = new ReverseBytesLNode( 0, arg); break; case vmIntrinsics::_reverse_i: n = new ReverseINode(0, arg); break; case vmIntrinsics::_reverse_l: n = new ReverseLNode(0, arg); break; default: fatal_unexpected_iid(id); break; } set_result(_gvn.transform(n)); return true; } //--------------------------inline_bitshuffle_methods--------------------------- // inline int Integer.compress(int, int) // inline int Integer.expand(int, int) // inline long Long.compress(long, long) // inline long Long.expand(long, long) bool LibraryCallKit::inline_bitshuffle_methods(vmIntrinsics::ID id) { Node* n = NULL; switch (id) { case vmIntrinsics::_compress_i: n = new CompressBitsNode(argument(0), argument(1), Type case vmIntrinsics::_expand_i: n = new ExpandBitsNode(argument(0), argument(1), TypeInt: case vmIntrinsics::_compress_l: n = new CompressBitsNode(argument(0), argument(2), Type case vmIntrinsics::_expand_l: n = new ExpandBitsNode(argument(0), argument(2), TypeLong default: fatal_unexpected_iid(id); break; } set_result(_gvn.transform(n)); return true; } //--------------------------inline_number_methods----------------------------- // inline int Integer.compareUnsigned(int, int) // inline int Long.compareUnsigned(long, long) bool LibraryCallKit::inline_compare_unsigned(vmIntrinsics::ID id) { Node* arg1 = argument(0); Node* arg2 = (id == vmIntrinsics::_compareUnsigned_l) ? argument(2) : argument(1); Node* n = NULL; switch (id) { case vmIntrinsics::_compareUnsigned_i: n = new CmpU3Node(arg1, arg2); break; case vmIntrinsics::_compareUnsigned_l: n = new CmpUL3Node(arg1, arg2); break; default: fatal_unexpected_iid(id); break; } set_result(_gvn.transform(n)); return true; } //--------------------------inline_unsigned_divmod_methods---------------------- // inline int Integer.divideUnsigned(int, int) // inline int Integer.remainderUnsigned(int, int) // inline long Long.divideUnsigned(long, long) // inline long Long.remainderUnsigned(long, long) bool LibraryCallKit::inline_divmod_methods(vmIntrinsics::ID id) { Node* n = NULL; switch (id) { case vmIntrinsics::_divideUnsigned_i: { zero_check_int(argument(1)); // Compile-time detect of null-exception if (stopped()) { return true; // keep the graph constructed so far } n = new UDivINode(control(), argument(0), argument(1)); break; } case vmIntrinsics::_divideUnsigned_l: { zero_check_long(argument(2)); // Compile-time detect of null-exception if (stopped()) { return true; // keep the graph constructed so far } n = new UDivLNode(control(), argument(0), argument(2)); break; } case vmIntrinsics::_remainderUnsigned_i: { zero_check_int(argument(1)); // Compile-time detect of null-exception if (stopped()) { return true; // keep the graph constructed so far } n = new UModINode(control(), argument(0), argument(1)); break; } case vmIntrinsics::_remainderUnsigned_l: { zero_check_long(argument(2)); // Compile-time detect of null-exception if (stopped()) { return true; // keep the graph constructed so far } n = new UModLNode(control(), argument(0), argument(2)); break; } default: fatal_unexpected_iid(id); break; } set_result(_gvn.transform(n)); return true; } //----------------------------inline_unsafe_access---------------------------- const TypeOopPtr* LibraryCallKit::sharpen_unsafe_type(Compile::AliasType* alias_typ // Attempt to infer a sharper value type from the offset and base type. ciKlass* sharpened_klass = NULL; // See if it is an instance field, with an object type. if (alias_type->field() != NULL) { if (alias_type->field()->type()->is_klass()) { sharpened_klass = alias_type->field()->type()->as_klass(); } } const TypeOopPtr* result = NULL; // See if it is a narrow oop array. if (adr_type->isa_aryptr()) { if (adr_type->offset() >= objArrayOopDesc::base_offset_in_bytes()) { const TypeOopPtr* elem_type = adr_type->is_aryptr()->elem()->make_oopptr(); if (elem_type != NULL && elem_type->is_loaded()) { // Sharpen the value type. result = elem_type; } } } // The sharpened class might be unloaded if there is no class loader // contraint in place. if (result == NULL && sharpened_klass != NULL && sharpened_klass->is_loaded()) { // Sharpen the value type. result = TypeOopPtr::make_from_klass(sharpened_klass); } if (result != NULL) { #ifndef PRODUCT if (C->print_intrinsics() || C->print_inlining()) { tty->print(" from base type: "); adr_type->dump(); tty->cr(); tty->print(" sharpened value: "); result->dump(); tty->cr(); } #endif } return result; } DecoratorSet LibraryCallKit::mo_decorator_for_access_kind(AccessKind kind) { switch (kind) { case Relaxed: return MO_UNORDERED; case Opaque: return MO_RELAXED; case Acquire: return MO_ACQUIRE; case Release: return MO_RELEASE; case Volatile: return MO_SEQ_CST; default: ShouldNotReachHere(); return 0; } } bool LibraryCallKit::inline_unsafe_access(bool is_store, const BasicType type, const Ac if (callee()->is_static()) return false; // caller must have the capability! DecoratorSet decorators = C2_UNSAFE_ACCESS; guarantee(!is_store || kind != Acquire, "Acquire accesses can be produced only for loads"); guarantee( is_store || kind != Release, "Release accesses can be produced only for stores"); assert(type != T_OBJECT || !unaligned, "unaligned access not supported with object type"); if (is_reference_type(type)) { decorators |= ON_UNKNOWN_OOP_REF; } if (unaligned) { decorators |= C2_UNALIGNED; } #ifndef PRODUCT { ResourceMark rm; // Check the signatures. ciSignature* sig = callee()->signature(); #ifdef ASSERT if (!is_store) { // Object getReference(Object base, int/long offset), etc. BasicType rtype = sig->return_type()->basic_type(); assert(rtype == type, "getter must return the expected value"); assert(sig->count() == 2, "oop getter has 2 arguments"); assert(sig->type_at(0)->basic_type() == T_OBJECT, "getter base is object"); assert(sig->type_at(1)->basic_type() == T_LONG, "getter offset is correct"); } else { // void putReference(Object base, int/long offset, Object x), etc. assert(sig->return_type()->basic_type() == T_VOID, "putter must not return a value"); assert(sig->count() == 3, "oop putter has 3 arguments"); assert(sig->type_at(0)->basic_type() == T_OBJECT, "putter base is object"); assert(sig->type_at(1)->basic_type() == T_LONG, "putter offset is correct"); BasicType vtype = sig->type_at(sig->count()-1)->basic_type(); assert(vtype == type, "putter must accept the expected value"); } #endif // ASSERT } #endif //PRODUCT C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe". Node* receiver = argument(0); // type: oop // Build address expression. Node* heap_base_oop = top(); // The base is either a Java object or a value produced by Unsafe.staticFieldBase Node* base = argument(1); // type: oop // The offset is a value produced by Unsafe.staticFieldOffset or Unsafe.objectFieldOffset Node* offset = argument(2); // type: long // We currently rely on the cookies produced by Unsafe.xxxFieldOffset // to be plain byte offsets, which are also the same as those accepted // by oopDesc::field_addr. assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled"); // 32-bit machines ignore the high half! offset = ConvL2X(offset); // Save state and restore on bailout uint old_sp = sp(); SafePointNode* old_map = clone_map(); Node* adr = make_unsafe_address(base, offset, type, kind == Relaxed); if (_gvn.type(base)->isa_ptr() == TypePtr::NULL_PTR) { if (type != T_OBJECT) { decorators |= IN_NATIVE; // off-heap primitive access } else { set_map(old_map); set_sp(old_sp); return false; // off-heap oop accesses are not supported } } else { heap_base_oop = base; // on-heap or mixed access } // Can base be NULL? Otherwise, always on-heap access. bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(_gvn.type(base)); if (!can_access_non_heap) { decorators |= IN_HEAP; } Node* val = is_store ? argument(4) : NULL; const TypePtr* adr_type = _gvn.type(adr)->isa_ptr(); if (adr_type == TypePtr::NULL_PTR) { set_map(old_map); set_sp(old_sp); return false; // off-heap access with zero address } // Try to categorize the address. Compile::AliasType* alias_type = C->alias_type(adr_type); assert(alias_type->index() != Compile::AliasIdxBot, "no bare pointers here"); if (alias_type->adr_type() == TypeInstPtr::KLASS || alias_type->adr_type() == TypeAryPtr::RANGE) { set_map(old_map); set_sp(old_sp); return false; // not supported } bool mismatched = false; BasicType bt = alias_type->basic_type(); if (bt != T_ILLEGAL) { assert(alias_type->adr_type()->is_oopptr(), "should be on-heap access"); if (bt == T_BYTE && adr_type->isa_aryptr()) { // Alias type doesn't differentiate between byte[] and boolean[]). // Use address type to get the element type. bt = adr_type->is_aryptr()->elem()->array_element_basic_type(); } if (is_reference_type(bt, true)) { // accessing an array field with getReference is not a mismatch bt = T_OBJECT; } if ((bt == T_OBJECT) != (type == T_OBJECT)) { // Don't intrinsify mismatched object accesses set_map(old_map); set_sp(old_sp); return false; } mismatched = (bt != type); } else if (alias_type->adr_type()->isa_oopptr()) { mismatched = true; // conservatively mark all "wide" on-heap accesses as mismatched } old_map->destruct(&_gvn); assert(!mismatched || alias_type->adr_type()->is_oopptr(), "off-heap access can't be mism if (mismatched) { decorators |= C2_MISMATCHED; } // First guess at the value type. const Type *value_type = Type::get_const_basic_type(type); // Figure out the memory ordering. decorators |= mo_decorator_for_access_kind(kind); if (!is_store && type == T_OBJECT) { const TypeOopPtr* tjp = sharpen_unsafe_type(alias_type, adr_type); if (tjp != NULL) { value_type = tjp; } } receiver = null_check(receiver); if (stopped()) { return true; } // Heap pointers get a null-check from the interpreter, // as a courtesy. However, this is not guaranteed by Unsafe, // and it is not possible to fully distinguish unintended nulls // from intended ones in this API. if (!is_store) { Node* p = NULL; // Try to constant fold a load from a constant field ciField* field = alias_type->field(); if (heap_base_oop != top() && field != NULL && field->is_constant() && !mismatched) { // final or stable field p = make_constant_from_field(field, heap_base_oop); } if (p == NULL) { // Could not constant fold the load p = access_load_at(heap_base_oop, adr, adr_type, value_type, type, decorators); // Normalize the value returned by getBoolean in the following cases if (type == T_BOOLEAN && (mismatched || heap_base_oop == top() || // - heap_base_oop is NULL or (can_access_non_heap && field == NULL)) // - heap_base_oop is potentially NULL // and the unsafe access is made to large offset // (i.e., larger than the maximum offset necessary for any // field access) ) { IdealKit ideal = IdealKit(this); #define __ ideal. IdealVariable normalized_result(ideal); __ declarations_done(); __ set(normalized_result, p); __ if_then(p, BoolTest::ne, ideal.ConI(0)); __ set(normalized_result, ideal.ConI(1)); ideal.end_if(); final_sync(ideal); p = __ value(normalized_result); #undef __ } } if (type == T_ADDRESS) { p = gvn().transform(new CastP2XNode(NULL, p)); p = ConvX2UL(p); } // The load node has the control of the preceding MemBarCPUOrder. All // following nodes will have the control of the MemBarCPUOrder inserted at // the end of this method. So, pushing the load onto the stack at a later // point is fine. set_result(p); } else { if (bt == T_ADDRESS) { // Repackage the long as a pointer. val = ConvL2X(val); val = gvn().transform(new CastX2PNode(val)); } access_store_at(heap_base_oop, adr, adr_type, val, value_type, type, decorators); } return true; } //----------------------------inline_unsafe_load_store-------------------------- // This method serves a couple of different customers (depending on LoadStoreKind): // // LS_cmp_swap: // // boolean compareAndSetReference(Object o, long offset, Object expected, Object x); // boolean compareAndSetInt( Object o, long offset, int expected, int x); // boolean compareAndSetLong( Object o, long offset, long expected, long x); // // LS_cmp_swap_weak: // // boolean weakCompareAndSetReference( Object o, long offset, Object expected, Object x); // boolean weakCompareAndSetReferencePlain( Object o, long offset, Object expected, Objec // boolean weakCompareAndSetReferenceAcquire(Object o, long offset, Object expected, Obj // boolean weakCompareAndSetReferenceRelease(Object o, long offset, Object expected, Obj // // boolean weakCompareAndSetInt( Object o, long offset, int expected, int x); // boolean weakCompareAndSetIntPlain( Object o, long offset, int expected, int x); // boolean weakCompareAndSetIntAcquire( Object o, long offset, int expected, int x); // boolean weakCompareAndSetIntRelease( Object o, long offset, int expected, int x); // // boolean weakCompareAndSetLong( Object o, long offset, long expected, long x); // boolean weakCompareAndSetLongPlain( Object o, long offset, long expected, long x); // boolean weakCompareAndSetLongAcquire( Object o, long offset, long expected, long x); // boolean weakCompareAndSetLongRelease( Object o, long offset, long expected, long x); // // LS_cmp_exchange: // // Object compareAndExchangeReferenceVolatile(Object o, long offset, Object expected, Ob // Object compareAndExchangeReferenceAcquire( Object o, long offset, Object expected, Obj // Object compareAndExchangeReferenceRelease( Object o, long offset, Object expected, Obj // // Object compareAndExchangeIntVolatile( Object o, long offset, Object expected, Object x) // Object compareAndExchangeIntAcquire( Object o, long offset, Object expected, Object x); // Object compareAndExchangeIntRelease( Object o, long offset, Object expected, Object x); // // Object compareAndExchangeLongVolatile( Object o, long offset, Object expected, Object x // Object compareAndExchangeLongAcquire( Object o, long offset, Object expected, Object x) // Object compareAndExchangeLongRelease( Object o, long offset, Object expected, Object x) // // LS_get_add: // // int getAndAddInt( Object o, long offset, int delta) // long getAndAddLong(Object o, long offset, long delta) // // LS_get_set: // // int getAndSet(Object o, long offset, int newValue) // long getAndSet(Object o, long offset, long newValue) // Object getAndSet(Object o, long offset, Object newValue) // bool LibraryCallKit::inline_unsafe_load_store(const BasicType type, const LoadStoreKi // This basic scheme here is the same as inline_unsafe_access, but // differs in enough details that combining them would make the code // overly confusing. (This is a true fact! I originally combined // them, but even I was confused by it!) As much code/comments as // possible are retained from inline_unsafe_access though to make // the correspondences clearer. - dl if (callee()->is_static()) return false; // caller must have the capability! DecoratorSet decorators = C2_UNSAFE_ACCESS; decorators |= mo_decorator_for_access_kind(access_kind); #ifndef PRODUCT BasicType rtype; { ResourceMark rm; // Check the signatures. ciSignature* sig = callee()->signature(); rtype = sig->return_type()->basic_type(); switch(kind) { case LS_get_add: case LS_get_set: { // Check the signatures. #ifdef ASSERT assert(rtype == type, "get and set must return the expected type"); assert(sig->count() == 3, "get and set has 3 arguments"); assert(sig->type_at(0)->basic_type() == T_OBJECT, "get and set base is object"); assert(sig->type_at(1)->basic_type() == T_LONG, "get and set offset is long"); assert(sig->type_at(2)->basic_type() == type, "get and set must take expected type as new valu assert(access_kind == Volatile, "mo is not passed to intrinsic nodes in current implementati #endif // ASSERT break; } case LS_cmp_swap: case LS_cmp_swap_weak: { // Check the signatures. #ifdef ASSERT assert(rtype == T_BOOLEAN, "CAS must return boolean"); assert(sig->count() == 4, "CAS has 4 arguments"); assert(sig->type_at(0)->basic_type() == T_OBJECT, "CAS base is object"); assert(sig->type_at(1)->basic_type() == T_LONG, "CAS offset is long"); #endif // ASSERT break; } case LS_cmp_exchange: { // Check the signatures. #ifdef ASSERT assert(rtype == type, "CAS must return the expected type"); assert(sig->count() == 4, "CAS has 4 arguments"); assert(sig->type_at(0)->basic_type() == T_OBJECT, "CAS base is object"); assert(sig->type_at(1)->basic_type() == T_LONG, "CAS offset is long"); #endif // ASSERT break; } default: ShouldNotReachHere(); } } #endif //PRODUCT C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe". // Get arguments: Node* receiver = NULL; Node* base = NULL; Node* offset = NULL; Node* oldval = NULL; Node* newval = NULL; switch(kind) { case LS_cmp_swap: case LS_cmp_swap_weak: case LS_cmp_exchange: { const bool two_slot_type = type2size[type] == 2; receiver = argument(0); // type: oop base = argument(1); // type: oop offset = argument(2); // type: long oldval = argument(4); // type: oop, int, or long newval = argument(two_slot_type ? 6 : 5); // type: oop, int, or long break; } case LS_get_add: case LS_get_set: { receiver = argument(0); // type: oop base = argument(1); // type: oop offset = argument(2); // type: long oldval = NULL; newval = argument(4); // type: oop, int, or long break; } default: ShouldNotReachHere(); } // Build field offset expression. // We currently rely on the cookies produced by Unsafe.xxxFieldOffset // to be plain byte offsets, which are also the same as those accepted // by oopDesc::field_addr. assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled" // 32-bit machines ignore the high half of long offsets offset = ConvL2X(offset); // Save state and restore on bailout uint old_sp = sp(); SafePointNode* old_map = clone_map(); Node* adr = make_unsafe_address(base, offset,type, false); const TypePtr *adr_type = _gvn.type(adr)->isa_ptr(); Compile::AliasType* alias_type = C->alias_type(adr_type); BasicType bt = alias_type->basic_type(); if (bt != T_ILLEGAL && (is_reference_type(bt) != (type == T_OBJECT))) { // Don't intrinsify mismatched object accesses. set_map(old_map); set_sp(old_sp); return false; } old_map->destruct(&_gvn); // For CAS, unlike inline_unsafe_access, there seems no point in // trying to refine types. Just use the coarse types here. assert(alias_type->index() != Compile::AliasIdxBot, "no bare pointers here"); const Type *value_type = Type::get_const_basic_type(type); switch (kind) { case LS_get_set: case LS_cmp_exchange: { if (type == T_OBJECT) { const TypeOopPtr* tjp = sharpen_unsafe_type(alias_type, adr_type); if (tjp != NULL) { value_type = tjp; } } break; } case LS_cmp_swap: case LS_cmp_swap_weak: case LS_get_add: break; default: ShouldNotReachHere(); } // Null check receiver. receiver = null_check(receiver); if (stopped()) { return true; } int alias_idx = C->get_alias_index(adr_type); if (is_reference_type(type)) { decorators |= IN_HEAP | ON_UNKNOWN_OOP_REF; // Transformation of a value which could be NULL pointer (CastPP #NULL) // could be delayed during Parse (for example, in adjust_map_after_if()). // Execute transformation here to avoid barrier generation in such case. if (_gvn.type(newval) == TypePtr::NULL_PTR) newval = _gvn.makecon(TypePtr::NULL_PTR); if (oldval != NULL && _gvn.type(oldval) == TypePtr::NULL_PTR) { // Refine the value to a null constant, when it is known to be null oldval = _gvn.makecon(TypePtr::NULL_PTR); } } Node* result = NULL; switch (kind) { case LS_cmp_exchange: { result = access_atomic_cmpxchg_val_at(base, adr, adr_type, alias_idx, oldval, newval, value_type, type, decorators); break; } case LS_cmp_swap_weak: decorators |= C2_WEAK_CMPXCHG; case LS_cmp_swap: { result = access_atomic_cmpxchg_bool_at(base, adr, adr_type, alias_idx, oldval, newval, value_type, type, decorators); break; } case LS_get_set: { result = access_atomic_xchg_at(base, adr, adr_type, alias_idx, newval, value_type, type, decorators); break; } case LS_get_add: { result = access_atomic_add_at(base, adr, adr_type, alias_idx, newval, value_type, type, decorators); break; } default: ShouldNotReachHere(); } assert(type2size[result->bottom_type()->basic_type()] == type2size[rtype], "result typ set_result(result); return true; } bool LibraryCallKit::inline_unsafe_fence(vmIntrinsics::ID id) { // Regardless of form, don't allow previous ld/st to move down, // then issue acquire, release, or volatile mem_bar. insert_mem_bar(Op_MemBarCPUOrder); switch(id) { case vmIntrinsics::_loadFence: insert_mem_bar(Op_LoadFence); return true; case vmIntrinsics::_storeFence: insert_mem_bar(Op_StoreFence); return true; case vmIntrinsics::_storeStoreFence: insert_mem_bar(Op_StoreStoreFence); return true; case vmIntrinsics::_fullFence: insert_mem_bar(Op_MemBarVolatile); return true; default: fatal_unexpected_iid(id); return false; } } bool LibraryCallKit::inline_onspinwait() { insert_mem_bar(Op_OnSpinWait); return true; } bool LibraryCallKit::klass_needs_init_guard(Node* kls) { if (!kls->is_Con()) { return true; } const TypeInstKlassPtr* klsptr = kls->bottom_type()->isa_instklassptr(); if (klsptr == NULL) { return true; } ciInstanceKlass* ik = klsptr->instance_klass(); // don't need a guard for a klass that is already initialized return !ik->is_initialized(); } //----------------------------inline_unsafe_writeback0------------------------- // public native void Unsafe.writeback0(long address) bool LibraryCallKit::inline_unsafe_writeback0() { if (!Matcher::has_match_rule(Op_CacheWB)) { return false; } #ifndef PRODUCT assert(Matcher::has_match_rule(Op_CacheWBPreSync), "found match rule for CacheWB but no assert(Matcher::has_match_rule(Op_CacheWBPostSync), "found match rule for CacheWB but n ciSignature* sig = callee()->signature(); assert(sig->type_at(0)->basic_type() == T_LONG, "Unsafe_writeback0 address is long!"); #endif null_check_receiver(); // null-check, then ignore Node *addr = argument(1); addr = new CastX2PNode(addr); addr = _gvn.transform(addr); Node *flush = new CacheWBNode(control(), memory(TypeRawPtr::BOTTOM), addr); flush = _gvn.transform(flush); set_memory(flush, TypeRawPtr::BOTTOM); return true; } //----------------------------inline_unsafe_writeback0------------------------- // public native void Unsafe.writeback0(long address) bool LibraryCallKit::inline_unsafe_writebackSync0(bool is_pre) { if (is_pre && !Matcher::has_match_rule(Op_CacheWBPreSync)) { return false; } if (!is_pre && !Matcher::has_match_rule(Op_CacheWBPostSync)) { return false; } #ifndef PRODUCT assert(Matcher::has_match_rule(Op_CacheWB), (is_pre ? "found match rule for CacheWBPreSync but not CacheWB" : "found match rule for CacheWBPostSync but not CacheWB")); #endif null_check_receiver(); // null-check, then ignore Node *sync; if (is_pre) { sync = new CacheWBPreSyncNode(control(), memory(TypeRawPtr::BOTTOM)); } else { sync = new CacheWBPostSyncNode(control(), memory(TypeRawPtr::BOTTOM)); } sync = _gvn.transform(sync); set_memory(sync, TypeRawPtr::BOTTOM); return true; } //----------------------------inline_unsafe_allocate--------------------------- // public native Object Unsafe.allocateInstance(Class<?> cls); bool LibraryCallKit::inline_unsafe_allocate() { if (callee()->is_static()) return false; // caller must have the capability! null_check_receiver(); // null-check, then ignore Node* cls = null_check(argument(1)); if (stopped()) return true; Node* kls = load_klass_from_mirror(cls, false, NULL, 0); kls = null_check(kls); if (stopped()) return true; // argument was like int.class Node* test = NULL; if (LibraryCallKit::klass_needs_init_guard(kls)) { // Note: The argument might still be an illegal value like // Serializable.class or Object[].class. The runtime will handle it. // But we must make an explicit check for initialization. Node* insp = basic_plus_adr(kls, in_bytes(InstanceKlass::init_state_offset())); // Use T_BOOLEAN for InstanceKlass::_init_state so the compiler // can generate code to load it as unsigned byte. Node* inst = make_load(NULL, insp, TypeInt::UBYTE, T_BOOLEAN, MemNode::unordered); Node* bits = intcon(InstanceKlass::fully_initialized); test = _gvn.transform(new SubINode(inst, bits)); // The 'test' is non-zero if we need to take a slow path. } Node* obj = new_instance(kls, test); set_result(obj); return true; } //------------------------inline_native_time_funcs-------------- // inline code for System.currentTimeMillis() and System.nanoTime() // these have the same type and signature bool LibraryCallKit::inline_native_time_funcs(address funcAddr, const char* funcName) const TypeFunc* tf = OptoRuntime::void_long_Type(); const TypePtr* no_memory_effects = NULL; Node* time = make_runtime_call(RC_LEAF, tf, funcAddr, funcName, no_memory_effects); Node* value = _gvn.transform(new ProjNode(time, TypeFunc::Parms+0)); #ifdef ASSERT Node* value_top = _gvn.transform(new ProjNode(time, TypeFunc::Parms+1)); assert(value_top == top(), "second value must be top"); #endif set_result(value); return true; } #ifdef JFR_HAVE_INTRINSICS /** * if oop->klass != null * // normal class * epoch = _epoch_state ? 2 : 1 * if oop->klass->trace_id & ((epoch << META_SHIFT) | epoch)) != epoch { * ... // enter slow path when the klass is first recorded or the epoch of JFR shifts * } * id = oop->klass->trace_id >> TRACE_ID_SHIFT // normal class path * else * // primitive class * if oop->array_klass != null * id = (oop->array_klass->trace_id >> TRACE_ID_SHIFT) + 1 // primitive class path * else * id = LAST_TYPE_ID + 1 // void class path * if (!signaled) * signaled = true */ bool LibraryCallKit::inline_native_classID() { Node* cls = argument(0); IdealKit ideal(this); #define __ ideal. IdealVariable result(ideal); __ declarations_done(); Node* kls = _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), basic_plus_adr(cls, java_lang_Class::klass_offset()), TypeRawPtr::BOTTOM, TypeInstKlassPtr::OBJECT_OR_NULL)); __ if_then(kls, BoolTest::ne, null()); { Node* kls_trace_id_addr = basic_plus_adr(kls, in_bytes(KLASS_TRACE_ID_OFFSET)); Node* kls_trace_id_raw = ideal.load(ideal.ctrl(), kls_trace_id_addr,TypeLong::LONG, T Node* epoch_address = makecon(TypeRawPtr::make(JfrIntrinsicSupport::epoch_address() Node* epoch = ideal.load(ideal.ctrl(), epoch_address, TypeInt::BOOL, T_BOOLEAN, Compile epoch = _gvn.transform(new LShiftLNode(longcon(1), epoch)); Node* mask = _gvn.transform(new LShiftLNode(epoch, intcon(META_SHIFT))); mask = _gvn.transform(new OrLNode(mask, epoch)); Node* kls_trace_id_raw_and_mask = _gvn.transform(new AndLNode(kls_trace_id_raw, mask) float unlikely = PROB_UNLIKELY(0.999); __ if_then(kls_trace_id_raw_and_mask, BoolTest::ne, epoch, unlikely); { sync_kit(ideal); make_runtime_call(RC_LEAF, OptoRuntime::class_id_load_barrier_Type(), CAST_FROM_FN_PTR(address, JfrIntrinsicSupport::load_barrier), "class id load barrier", TypePtr::BOTTOM, kls); ideal.sync_kit(this); } __ end_if(); ideal.set(result, _gvn.transform(new URShiftLNode(kls_trace_id_raw, ideal.ConI(TRAC } __ else_(); { Node* array_kls = _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), basic_plus_adr(cls, java_lang_Class::array_klass_offset()), TypeRawPtr::BOTTOM, TypeInstKlassPtr::OBJECT_OR_NULL)); __ if_then(array_kls, BoolTest::ne, null()); { Node* array_kls_trace_id_addr = basic_plus_adr(array_kls, in_bytes(KLASS_TRACE_ID_OF Node* array_kls_trace_id_raw = ideal.load(ideal.ctrl(), array_kls_trace_id_addr, Type Node* array_kls_trace_id = _gvn.transform(new URShiftLNode(array_kls_trace_id_raw, id ideal.set(result, _gvn.transform(new AddLNode(array_kls_trace_id, longcon(1)))); } __ else_(); { // void class case ideal.set(result, _gvn.transform(longcon(LAST_TYPE_ID + 1))); } __ end_if(); Node* signaled_flag_address = makecon(TypeRawPtr::make(JfrIntrinsicSupport::signal_ Node* signaled = ideal.load(ideal.ctrl(), signaled_flag_address, TypeInt::BOOL, T_BOOL __ if_then(signaled, BoolTest::ne, ideal.ConI(1)); { ideal.store(ideal.ctrl(), signaled_flag_address, ideal.ConI(1), T_BOOLEAN, Compile:: } __ end_if(); } __ end_if(); final_sync(ideal); set_result(ideal.value(result)); #undef __ return true; } /* * The intrinsic is a model of this pseudo-code: * * JfrThreadLocal* const tl = Thread::jfr_thread_local() * jobject h_event_writer = tl->java_event_writer(); * if (h_event_writer == NULL) { * return NULL; * } * oop threadObj = Thread::threadObj(); * oop vthread = java_lang_Thread::vthread(threadObj); * traceid tid; * bool excluded; * if (vthread != threadObj) { // i.e. current thread is virtual * tid = java_lang_Thread::tid(vthread); * u2 vthread_epoch_raw = java_lang_Thread::jfr_epoch(vthread); * excluded = vthread_epoch_raw & excluded_mask; * if (!excluded) { * traceid current_epoch = JfrTraceIdEpoch::current_generation(); * u2 vthread_epoch = vthread_epoch_raw & epoch_mask; * if (vthread_epoch != current_epoch) { * write_checkpoint(); * } * } * } else { * tid = java_lang_Thread::tid(threadObj); * u2 thread_epoch_raw = java_lang_Thread::jfr_epoch(threadObj); * excluded = thread_epoch_raw & excluded_mask; * } * oop event_writer = JNIHandles::resolve_non_null(h_event_writer); * traceid tid_in_event_writer = getField(event_writer, "threadID"); * if (tid_in_event_writer != tid) { * setField(event_writer, "threadID", tid); * setField(event_writer, "excluded", excluded); * } * return event_writer */ bool LibraryCallKit::inline_native_getEventWriter() { enum { _true_path = 1, _false_path = 2, PATH_LIMIT }; // Save input memory and i_o state. Node* input_memory_state = reset_memory(); set_all_memory(input_memory_state); Node* input_io_state = i_o(); Node* excluded_mask = _gvn.intcon(32768); Node* epoch_mask = _gvn.intcon(32767); // TLS Node* tls_ptr = _gvn.transform(new ThreadLocalNode()); // Load the address of java event writer jobject handle from the jfr_thread_local structure. Node* jobj_ptr = basic_plus_adr(top(), tls_ptr, in_bytes(THREAD_LOCAL_WRITER_OFFSET_J // Load the eventwriter jobject handle. Node* jobj = make_load(control(), jobj_ptr, TypeRawPtr::BOTTOM, T_ADDRESS, MemNode::uno // Null check the jobject handle. Node* jobj_cmp_null = _gvn.transform(new CmpPNode(jobj, null())); Node* test_jobj_not_equal_null = _gvn.transform(new BoolNode(jobj_cmp_null, BoolTest: IfNode* iff_jobj_not_equal_null = create_and_map_if(control(), test_jobj_not_equal_n // False path, jobj is null. Node* jobj_is_null = _gvn.transform(new IfFalseNode(iff_jobj_not_equal_null)); // True path, jobj is not null. Node* jobj_is_not_null = _gvn.transform(new IfTrueNode(iff_jobj_not_equal_null)); set_control(jobj_is_not_null); // Load the threadObj for the CarrierThread. Node* threadObj = generate_current_thread(tls_ptr); // Load the vthread. Node* vthread = generate_virtual_thread(tls_ptr); // If vthread != threadObj, this is a virtual thread. Node* vthread_cmp_threadObj = _gvn.transform(new CmpPNode(vthread, threadObj)); Node* test_vthread_not_equal_threadObj = _gvn.transform(new BoolNode(vthread_cmp_thr IfNode* iff_vthread_not_equal_threadObj = create_and_map_if(jobj_is_not_null, test_vthread_not_equal_threadObj, PROB_FAIR, CO // False branch, fallback to threadObj. Node* vthread_equal_threadObj = _gvn.transform(new IfFalseNode(iff_vthread_not_equal set_control(vthread_equal_threadObj); // Load the tid field from the vthread object. Node* thread_obj_tid = load_field_from_object(threadObj, "tid", "J"); // Load the raw epoch value from the threadObj. Node* threadObj_epoch_offset = basic_plus_adr(threadObj, java_lang_Thread::jfr_epoch Node* threadObj_epoch_raw = access_load_at(threadObj, threadObj_epoch_offset, TypeRaw IN_HEAP | MO_UNORDERED | C2_MISMATCHED | C2_CONTROL_DEPENDENT_LOAD); // Mask off the excluded information from the epoch. Node * threadObj_is_excluded = _gvn.transform(new AndINode(threadObj_epoch_raw, exclud // True branch, this is a virtual thread. Node* vthread_not_equal_threadObj = _gvn.transform(new IfTrueNode(iff_vthread_not_eq set_control(vthread_not_equal_threadObj); // Load the tid field from the vthread object. Node* vthread_tid = load_field_from_object(vthread, "tid", "J"); // Load the raw epoch value from the vthread. Node* vthread_epoch_offset = basic_plus_adr(vthread, java_lang_Thread::jfr_epoch_off Node* vthread_epoch_raw = access_load_at(vthread, vthread_epoch_offset, TypeRawPtr::B IN_HEAP | MO_UNORDERED | C2_MISMATCHED | C2_CONTROL_DEPENDENT_LOAD); // Mask off the excluded information from the epoch. Node * vthread_is_excluded = _gvn.transform(new AndINode(vthread_epoch_raw, _gvn.trans // Branch on excluded to conditionalize updating the epoch for the virtual thread. Node* is_excluded_cmp = _gvn.transform(new CmpINode(vthread_is_excluded, _gvn.transfo Node* test_not_excluded = _gvn.transform(new BoolNode(is_excluded_cmp, BoolTest::ne)) IfNode* iff_not_excluded = create_and_map_if(control(), test_not_excluded, PROB_MAX, C // False branch, vthread is excluded, no need to write epoch info. Node* excluded = _gvn.transform(new IfFalseNode(iff_not_excluded)); // True branch, vthread is included, update epoch info. Node* included = _gvn.transform(new IfTrueNode(iff_not_excluded)); set_control(included); // Get epoch value. Node* epoch = _gvn.transform(new AndINode(vthread_epoch_raw, _gvn.transform(epoch_mas // Load the current epoch generation. The value is unsigned 16-bit, so we type it as T_CHAR. Node* epoch_generation_address = makecon(TypeRawPtr::make(JfrIntrinsicSupport::epoc Node* current_epoch_generation = make_load(control(), epoch_generation_address, TypeI // Compare the epoch in the vthread to the current epoch generation. Node* const epoch_cmp = _gvn.transform(new CmpUNode(current_epoch_generation, epoch)); Node* test_epoch_not_equal = _gvn.transform(new BoolNode(epoch_cmp, BoolTest::ne)); IfNode* iff_epoch_not_equal = create_and_map_if(control(), test_epoch_not_equal, PROB // False path, epoch is equal, checkpoint information is valid. Node* epoch_is_equal = _gvn.transform(new IfFalseNode(iff_epoch_not_equal)); // True path, epoch is not equal, write a checkpoint for the vthread. Node* epoch_is_not_equal = _gvn.transform(new IfTrueNode(iff_epoch_not_equal)); set_control(epoch_is_not_equal); // Make a runtime call, which can safepoint, to write a checkpoint for the vthread for this epoch // The call also updates the native thread local thread id and the vthread with the current epoch Node* call_write_checkpoint = make_runtime_call(RC_NO_LEAF, OptoRuntime::jfr_write_checkpoint_Type(), StubRoutines::jfr_write_checkpoint(), "write_checkpoint", TypePtr::BOTTOM); Node* call_write_checkpoint_control = _gvn.transform(new ProjNode(call_write_checkpo // vthread epoch != current epoch RegionNode* epoch_compare_rgn = new RegionNode(PATH_LIMIT); record_for_igvn(epoch_compare_rgn); PhiNode* epoch_compare_mem = new PhiNode(epoch_compare_rgn, Type::MEMORY, TypePtr::BOT record_for_igvn(epoch_compare_mem); PhiNode* epoch_compare_io = new PhiNode(epoch_compare_rgn, Type::ABIO); record_for_igvn(epoch_compare_io); // Update control and phi nodes. epoch_compare_rgn->init_req(_true_path, call_write_checkpoint_control); epoch_compare_rgn->init_req(_false_path, epoch_is_equal); epoch_compare_mem->init_req(_true_path, _gvn.transform(reset_memory())); epoch_compare_mem->init_req(_false_path, input_memory_state); epoch_compare_io->init_req(_true_path, i_o()); epoch_compare_io->init_req(_false_path, input_io_state); // excluded != true RegionNode* exclude_compare_rgn = new RegionNode(PATH_LIMIT); record_for_igvn(exclude_compare_rgn); PhiNode* exclude_compare_mem = new PhiNode(exclude_compare_rgn, Type::MEMORY, TypePtr: record_for_igvn(exclude_compare_mem); PhiNode* exclude_compare_io = new PhiNode(exclude_compare_rgn, Type::ABIO); record_for_igvn(exclude_compare_io); // Update control and phi nodes. exclude_compare_rgn->init_req(_true_path, _gvn.transform(epoch_compare_rgn)); exclude_compare_rgn->init_req(_false_path, excluded); exclude_compare_mem->init_req(_true_path, _gvn.transform(epoch_compare_mem)); exclude_compare_mem->init_req(_false_path, input_memory_state); exclude_compare_io->init_req(_true_path, _gvn.transform(epoch_compare_io)); exclude_compare_io->init_req(_false_path, input_io_state); // vthread != threadObj RegionNode* vthread_compare_rgn = new RegionNode(PATH_LIMIT); record_for_igvn(vthread_compare_rgn); PhiNode* vthread_compare_mem = new PhiNode(vthread_compare_rgn, Type::MEMORY, TypePtr: |