| Quellcodebibliothek Statistik Leitseite products/Sources/formale Sprachen/Java/Openjdk/src/hotspot/cpu/x86/ (Sun/Oracle ©) Datei vom 13.11.2022 mit Größe 161 kB |
|
SSL stubGenerator_x86_32.cpp Sprache: C |
|||||||||
|
/*
* Copyright (c) 1999, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as * published by the Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA * or visit www.oracle.com if you need additional information or have any * questions. * */ #include "precompiled.hpp" #include "asm/macroAssembler.hpp" #include "asm/macroAssembler.inline.hpp" #include "compiler/oopMap.hpp" #include "gc/shared/barrierSet.hpp" #include "gc/shared/barrierSetAssembler.hpp" #include "gc/shared/barrierSetNMethod.hpp" #include "interpreter/interpreter.hpp" #include "memory/universe.hpp" #include "nativeInst_x86.hpp" #include "oops/instanceOop.hpp" #include "oops/method.hpp" #include "oops/objArrayKlass.hpp" #include "oops/oop.inline.hpp" #include "prims/methodHandles.hpp" #include "runtime/frame.inline.hpp" #include "runtime/handles.inline.hpp" #include "runtime/javaThread.hpp" #include "runtime/sharedRuntime.hpp" #include "runtime/stubCodeGenerator.hpp" #include "runtime/stubRoutines.hpp" #ifdef COMPILER2 #include "opto/runtime.hpp" #endif // Declaration and definition of StubGenerator (no .hpp file). // For a more detailed description of the stub routine structure // see the comment in stubRoutines.hpp #define __ _masm-> #define a__ ((Assembler*)_masm)-> #ifdef PRODUCT #define BLOCK_COMMENT(str) /* nothing */ #else #define BLOCK_COMMENT(str) __ block_comment(str) #endif #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") const int MXCSR_MASK = 0xFFC0; // Mask out any pending exceptions const int FPU_CNTRL_WRD_MASK = 0xFFFF; ATTRIBUTE_ALIGNED(16) uint32_t KEY_SHUFFLE_MASK[] = { 0x00010203UL, 0x04050607UL, 0x08090A0BUL, 0x0C0D0E0FUL, }; ATTRIBUTE_ALIGNED(16) uint32_t COUNTER_SHUFFLE_MASK[] = { 0x0C0D0E0FUL, 0x08090A0BUL, 0x04050607UL, 0x00010203UL, }; ATTRIBUTE_ALIGNED(16) uint32_t GHASH_BYTE_SWAP_MASK[] = { 0x0C0D0E0FUL, 0x08090A0BUL, 0x04050607UL, 0x00010203UL, }; ATTRIBUTE_ALIGNED(16) uint32_t GHASH_LONG_SWAP_MASK[] = { 0x0B0A0908UL, 0x0F0E0D0CUL, 0x03020100UL, 0x07060504UL, }; // ------------------------------------------------------------------------------ // Stub Code definitions class StubGenerator: public StubCodeGenerator { private: #ifdef PRODUCT #define inc_counter_np(counter) ((void)0) #else void inc_counter_np_(int& counter) { __ incrementl(ExternalAddress((address)&counter)); } #define inc_counter_np(counter) \ BLOCK_COMMENT("inc_counter " #counter); \ inc_counter_np_(counter); #endif //PRODUCT void inc_copy_counter_np(BasicType t) { #ifndef PRODUCT switch (t) { case T_BYTE: inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); return; case T_SHORT: inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); return; case T_INT: inc_counter_np(SharedRuntime::_jint_array_copy_ctr); return; case T_LONG: inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); return; case T_OBJECT: inc_counter_np(SharedRuntime::_oop_array_copy_ctr); return; default: ShouldNotReachHere(); } #endif //PRODUCT } //------------------------------------------------------------------------------ // Call stubs are used to call Java from C // // [ return_from_Java ] <--- rsp // [ argument word n ] // ... // -N [ argument word 1 ] // -7 [ Possible padding for stack alignment ] // -6 [ Possible padding for stack alignment ] // -5 [ Possible padding for stack alignment ] // -4 [ mxcsr save ] <--- rsp_after_call // -3 [ saved rbx, ] // -2 [ saved rsi ] // -1 [ saved rdi ] // 0 [ saved rbp, ] <--- rbp, // 1 [ return address ] // 2 [ ptr. to call wrapper ] // 3 [ result ] // 4 [ result_type ] // 5 [ method ] // 6 [ entry_point ] // 7 [ parameters ] // 8 [ parameter_size ] // 9 [ thread ] address generate_call_stub(address& return_address) { StubCodeMark mark(this, "StubRoutines", "call_stub"); address start = __ pc(); // stub code parameters / addresses assert(frame::entry_frame_call_wrapper_offset == 2, "adjust this code"); bool sse_save = false; const Address rsp_after_call(rbp, -4 * wordSize); // same as in generate_catch_exception() const int locals_count_in_bytes (4*wordSize); const Address mxcsr_save (rbp, -4 * wordSize); const Address saved_rbx (rbp, -3 * wordSize); const Address saved_rsi (rbp, -2 * wordSize); const Address saved_rdi (rbp, -1 * wordSize); const Address result (rbp, 3 * wordSize); const Address result_type (rbp, 4 * wordSize); const Address method (rbp, 5 * wordSize); const Address entry_point (rbp, 6 * wordSize); const Address parameters (rbp, 7 * wordSize); const Address parameter_size(rbp, 8 * wordSize); const Address thread (rbp, 9 * wordSize); // same as in generate_catch_exception()! sse_save = UseSSE > 0; // stub code __ enter(); __ movptr(rcx, parameter_size); // parameter counter __ shlptr(rcx, Interpreter::logStackElementSize); // convert parameter count to bytes __ addptr(rcx, locals_count_in_bytes); // reserve space for register saves __ subptr(rsp, rcx); __ andptr(rsp, -(StackAlignmentInBytes)); // Align stack // save rdi, rsi, & rbx, according to C calling conventions __ movptr(saved_rdi, rdi); __ movptr(saved_rsi, rsi); __ movptr(saved_rbx, rbx); // save and initialize %mxcsr if (sse_save) { Label skip_ldmx; __ stmxcsr(mxcsr_save); __ movl(rax, mxcsr_save); __ andl(rax, MXCSR_MASK); // Only check control and mask bits ExternalAddress mxcsr_std(StubRoutines::x86::addr_mxcsr_std()); __ cmp32(rax, mxcsr_std); __ jcc(Assembler::equal, skip_ldmx); __ ldmxcsr(mxcsr_std); __ bind(skip_ldmx); } // make sure the control word is correct. __ fldcw(ExternalAddress(StubRoutines::x86::addr_fpu_cntrl_wrd_std())); #ifdef ASSERT // make sure we have no pending exceptions { Label L; __ movptr(rcx, thread); __ cmpptr(Address(rcx, Thread::pending_exception_offset()), NULL_WORD); __ jcc(Assembler::equal, L); __ stop("StubRoutines::call_stub: entered with pending exception"); __ bind(L); } #endif // pass parameters if any BLOCK_COMMENT("pass parameters if any"); Label parameters_done; __ movl(rcx, parameter_size); // parameter counter __ testl(rcx, rcx); __ jcc(Assembler::zero, parameters_done); // parameter passing loop Label loop; // Copy Java parameters in reverse order (receiver last) // Note that the argument order is inverted in the process // source is rdx[rcx: N-1..0] // dest is rsp[rbx: 0..N-1] __ movptr(rdx, parameters); // parameter pointer __ xorptr(rbx, rbx); __ BIND(loop); // get parameter __ movptr(rax, Address(rdx, rcx, Interpreter::stackElementScale(), -wordSize)); __ movptr(Address(rsp, rbx, Interpreter::stackElementScale(), Interpreter::expr_offset_in_bytes(0)), rax); // store parameter __ increment(rbx); __ decrement(rcx); __ jcc(Assembler::notZero, loop); // call Java function __ BIND(parameters_done); __ movptr(rbx, method); // get Method* __ movptr(rax, entry_point); // get entry_point __ mov(rsi, rsp); // set sender sp BLOCK_COMMENT("call Java function"); __ call(rax); BLOCK_COMMENT("call_stub_return_address:"); return_address = __ pc(); #ifdef COMPILER2 { Label L_skip; if (UseSSE >= 2) { __ verify_FPU(0, "call_stub_return"); } else { for (int i = 1; i < 8; i++) { __ ffree(i); } // UseSSE <= 1 so double result should be left on TOS __ movl(rsi, result_type); __ cmpl(rsi, T_DOUBLE); __ jcc(Assembler::equal, L_skip); if (UseSSE == 0) { // UseSSE == 0 so float result should be left on TOS __ cmpl(rsi, T_FLOAT); __ jcc(Assembler::equal, L_skip); } __ ffree(0); } __ BIND(L_skip); } #endif // COMPILER2 // store result depending on type // (everything that is not T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT) __ movptr(rdi, result); Label is_long, is_float, is_double, exit; __ movl(rsi, result_type); __ cmpl(rsi, T_LONG); __ jcc(Assembler::equal, is_long); __ cmpl(rsi, T_FLOAT); __ jcc(Assembler::equal, is_float); __ cmpl(rsi, T_DOUBLE); __ jcc(Assembler::equal, is_double); // handle T_INT case __ movl(Address(rdi, 0), rax); __ BIND(exit); // check that FPU stack is empty __ verify_FPU(0, "generate_call_stub"); // pop parameters __ lea(rsp, rsp_after_call); // restore %mxcsr if (sse_save) { __ ldmxcsr(mxcsr_save); } // restore rdi, rsi and rbx, __ movptr(rbx, saved_rbx); __ movptr(rsi, saved_rsi); __ movptr(rdi, saved_rdi); __ addptr(rsp, 4*wordSize); // return __ pop(rbp); __ ret(0); // handle return types different from T_INT __ BIND(is_long); __ movl(Address(rdi, 0 * wordSize), rax); __ movl(Address(rdi, 1 * wordSize), rdx); __ jmp(exit); __ BIND(is_float); // interpreter uses xmm0 for return values if (UseSSE >= 1) { __ movflt(Address(rdi, 0), xmm0); } else { __ fstp_s(Address(rdi, 0)); } __ jmp(exit); __ BIND(is_double); // interpreter uses xmm0 for return values if (UseSSE >= 2) { __ movdbl(Address(rdi, 0), xmm0); } else { __ fstp_d(Address(rdi, 0)); } __ jmp(exit); return start; } //------------------------------------------------------------------------------ // Return point for a Java call if there's an exception thrown in Java code. // The exception is caught and transformed into a pending exception stored in // JavaThread that can be tested from within the VM. // // Note: Usually the parameters are removed by the callee. In case of an exception // crossing an activation frame boundary, that is not the case if the callee // is compiled code => need to setup the rsp. // // rax,: exception oop address generate_catch_exception() { StubCodeMark mark(this, "StubRoutines", "catch_exception"); const Address rsp_after_call(rbp, -4 * wordSize); // same as in generate_call_stub()! const Address thread (rbp, 9 * wordSize); // same as in generate_call_stub()! address start = __ pc(); // get thread directly __ movptr(rcx, thread); #ifdef ASSERT // verify that threads correspond { Label L; __ get_thread(rbx); __ cmpptr(rbx, rcx); __ jcc(Assembler::equal, L); __ stop("StubRoutines::catch_exception: threads must correspond"); __ bind(L); } #endif // set pending exception __ verify_oop(rax); __ movptr(Address(rcx, Thread::pending_exception_offset()), rax); __ lea(Address(rcx, Thread::exception_file_offset()), ExternalAddress((address)__FILE__), noreg); __ movl(Address(rcx, Thread::exception_line_offset()), __LINE__ ); // complete return to VM assert(StubRoutines::_call_stub_return_address != NULL, "_call_stub_return_address __ jump(RuntimeAddress(StubRoutines::_call_stub_return_address)); return start; } //------------------------------------------------------------------------------ // Continuation point for runtime calls returning with a pending exception. // The pending exception check happened in the runtime or native call stub. // The pending exception in Thread is converted into a Java-level exception. // // Contract with Java-level exception handlers: // rax: exception // rdx: throwing pc // // NOTE: At entry of this stub, exception-pc must be on stack !! address generate_forward_exception() { StubCodeMark mark(this, "StubRoutines", "forward exception"); address start = __ pc(); const Register thread = rcx; // other registers used in this stub const Register exception_oop = rax; const Register handler_addr = rbx; const Register exception_pc = rdx; // Upon entry, the sp points to the return address returning into Java // (interpreted or compiled) code; i.e., the return address becomes the // throwing pc. // // Arguments pushed before the runtime call are still on the stack but // the exception handler will reset the stack pointer -> ignore them. // A potential result in registers can be ignored as well. #ifdef ASSERT // make sure this code is only executed if there is a pending exception { Label L; __ get_thread(thread); __ cmpptr(Address(thread, Thread::pending_exception_offset()), NULL_WORD); __ jcc(Assembler::notEqual, L); __ stop("StubRoutines::forward exception: no pending exception (1)"); __ bind(L); } #endif // compute exception handler into rbx, __ get_thread(thread); __ movptr(exception_pc, Address(rsp, 0)); BLOCK_COMMENT("call exception_handler_for_return_address"); __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_ret __ mov(handler_addr, rax); // setup rax & rdx, remove return address & clear pending exception __ get_thread(thread); __ pop(exception_pc); __ movptr(exception_oop, Address(thread, Thread::pending_exception_offset())); __ movptr(Address(thread, Thread::pending_exception_offset()), NULL_WORD); #ifdef ASSERT // make sure exception is set { Label L; __ testptr(exception_oop, exception_oop); __ jcc(Assembler::notEqual, L); __ stop("StubRoutines::forward exception: no pending exception (2)"); __ bind(L); } #endif // Verify that there is really a valid exception in RAX. __ verify_oop(exception_oop); // continue at exception handler (return address removed) // rax: exception // rbx: exception handler // rdx: throwing pc __ jmp(handler_addr); return start; } //------------------------------------------------------------------------------ // Support for void verify_mxcsr() // // This routine is used with -Xcheck:jni to verify that native // JNI code does not return to Java code without restoring the // MXCSR register to our expected state. address generate_verify_mxcsr() { StubCodeMark mark(this, "StubRoutines", "verify_mxcsr"); address start = __ pc(); const Address mxcsr_save(rsp, 0); if (CheckJNICalls && UseSSE > 0 ) { Label ok_ret; ExternalAddress mxcsr_std(StubRoutines::x86::addr_mxcsr_std()); __ push(rax); __ subptr(rsp, wordSize); // allocate a temp location __ stmxcsr(mxcsr_save); __ movl(rax, mxcsr_save); __ andl(rax, MXCSR_MASK); __ cmp32(rax, mxcsr_std); __ jcc(Assembler::equal, ok_ret); __ warn("MXCSR changed by native JNI code."); __ ldmxcsr(mxcsr_std); __ bind(ok_ret); __ addptr(rsp, wordSize); __ pop(rax); } __ ret(0); return start; } //--------------------------------------------------------------------------- // Support for void verify_fpu_cntrl_wrd() // // This routine is used with -Xcheck:jni to verify that native // JNI code does not return to Java code without restoring the // FP control word to our expected state. address generate_verify_fpu_cntrl_wrd() { StubCodeMark mark(this, "StubRoutines", "verify_spcw"); address start = __ pc(); const Address fpu_cntrl_wrd_save(rsp, 0); if (CheckJNICalls) { Label ok_ret; __ push(rax); __ subptr(rsp, wordSize); // allocate a temp location __ fnstcw(fpu_cntrl_wrd_save); __ movl(rax, fpu_cntrl_wrd_save); __ andl(rax, FPU_CNTRL_WRD_MASK); ExternalAddress fpu_std(StubRoutines::x86::addr_fpu_cntrl_wrd_std()); __ cmp32(rax, fpu_std); __ jcc(Assembler::equal, ok_ret); __ warn("Floating point control word changed by native JNI code."); __ fldcw(fpu_std); __ bind(ok_ret); __ addptr(rsp, wordSize); __ pop(rax); } __ ret(0); return start; } //--------------------------------------------------------------------------- // Wrapper for slow-case handling of double-to-integer conversion // d2i or f2i fast case failed either because it is nan or because // of under/overflow. // Input: FPU TOS: float value // Output: rax, (rdx): integer (long) result address generate_d2i_wrapper(BasicType t, address fcn) { StubCodeMark mark(this, "StubRoutines", "d2i_wrapper"); address start = __ pc(); // Capture info about frame layout enum layout { FPUState_off = 0, rbp_off = FPUStateSizeInWords, rdi_off, rsi_off, rcx_off, rbx_off, saved_argument_off, saved_argument_off2, // 2nd half of double framesize }; assert(FPUStateSizeInWords == 27, "update stack layout"); // Save outgoing argument to stack across push_FPU_state() __ subptr(rsp, wordSize * 2); __ fstp_d(Address(rsp, 0)); // Save CPU & FPU state __ push(rbx); __ push(rcx); __ push(rsi); __ push(rdi); __ push(rbp); __ push_FPU_state(); // push_FPU_state() resets the FP top of stack // Load original double into FP top of stack __ fld_d(Address(rsp, saved_argument_off * wordSize)); // Store double into stack as outgoing argument __ subptr(rsp, wordSize*2); __ fst_d(Address(rsp, 0)); // Prepare FPU for doing math in C-land __ empty_FPU_stack(); // Call the C code to massage the double. Result in EAX if (t == T_INT) { BLOCK_COMMENT("SharedRuntime::d2i"); } else if (t == T_LONG) { BLOCK_COMMENT("SharedRuntime::d2l"); } __ call_VM_leaf( fcn, 2 ); // Restore CPU & FPU state __ pop_FPU_state(); __ pop(rbp); __ pop(rdi); __ pop(rsi); __ pop(rcx); __ pop(rbx); __ addptr(rsp, wordSize * 2); __ ret(0); return start; } //------------------------------------------------------------------------------ address generate_vector_mask(const char *stub_name, int32_t mask) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", stub_name); address start = __ pc(); for (int i = 0; i < 16; i++) { __ emit_data(mask, relocInfo::none, 0); } return start; } address generate_count_leading_zeros_lut(const char *stub_name) { __ align64(); StubCodeMark mark(this, "StubRoutines", stub_name); address start = __ pc(); __ emit_data(0x02020304, relocInfo::none, 0); __ emit_data(0x01010101, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); __ emit_data(0x02020304, relocInfo::none, 0); __ emit_data(0x01010101, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); __ emit_data(0x02020304, relocInfo::none, 0); __ emit_data(0x01010101, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); __ emit_data(0x02020304, relocInfo::none, 0); __ emit_data(0x01010101, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); return start; } address generate_popcount_avx_lut(const char *stub_name) { __ align64(); StubCodeMark mark(this, "StubRoutines", stub_name); address start = __ pc(); __ emit_data(0x02010100, relocInfo::none, 0); __ emit_data(0x03020201, relocInfo::none, 0); __ emit_data(0x03020201, relocInfo::none, 0); __ emit_data(0x04030302, relocInfo::none, 0); __ emit_data(0x02010100, relocInfo::none, 0); __ emit_data(0x03020201, relocInfo::none, 0); __ emit_data(0x03020201, relocInfo::none, 0); __ emit_data(0x04030302, relocInfo::none, 0); __ emit_data(0x02010100, relocInfo::none, 0); __ emit_data(0x03020201, relocInfo::none, 0); __ emit_data(0x03020201, relocInfo::none, 0); __ emit_data(0x04030302, relocInfo::none, 0); __ emit_data(0x02010100, relocInfo::none, 0); __ emit_data(0x03020201, relocInfo::none, 0); __ emit_data(0x03020201, relocInfo::none, 0); __ emit_data(0x04030302, relocInfo::none, 0); return start; } address generate_iota_indices(const char *stub_name) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", stub_name); address start = __ pc(); // B __ emit_data(0x03020100, relocInfo::none, 0); __ emit_data(0x07060504, relocInfo::none, 0); __ emit_data(0x0B0A0908, relocInfo::none, 0); __ emit_data(0x0F0E0D0C, relocInfo::none, 0); __ emit_data(0x13121110, relocInfo::none, 0); __ emit_data(0x17161514, relocInfo::none, 0); __ emit_data(0x1B1A1918, relocInfo::none, 0); __ emit_data(0x1F1E1D1C, relocInfo::none, 0); __ emit_data(0x23222120, relocInfo::none, 0); __ emit_data(0x27262524, relocInfo::none, 0); __ emit_data(0x2B2A2928, relocInfo::none, 0); __ emit_data(0x2F2E2D2C, relocInfo::none, 0); __ emit_data(0x33323130, relocInfo::none, 0); __ emit_data(0x37363534, relocInfo::none, 0); __ emit_data(0x3B3A3938, relocInfo::none, 0); __ emit_data(0x3F3E3D3C, relocInfo::none, 0); // W __ emit_data(0x00010000, relocInfo::none, 0); __ emit_data(0x00030002, relocInfo::none, 0); __ emit_data(0x00050004, relocInfo::none, 0); __ emit_data(0x00070006, relocInfo::none, 0); __ emit_data(0x00090008, relocInfo::none, 0); __ emit_data(0x000B000A, relocInfo::none, 0); __ emit_data(0x000D000C, relocInfo::none, 0); __ emit_data(0x000F000E, relocInfo::none, 0); __ emit_data(0x00110010, relocInfo::none, 0); __ emit_data(0x00130012, relocInfo::none, 0); __ emit_data(0x00150014, relocInfo::none, 0); __ emit_data(0x00170016, relocInfo::none, 0); __ emit_data(0x00190018, relocInfo::none, 0); __ emit_data(0x001B001A, relocInfo::none, 0); __ emit_data(0x001D001C, relocInfo::none, 0); __ emit_data(0x001F001E, relocInfo::none, 0); // D __ emit_data(0x00000000, relocInfo::none, 0); __ emit_data(0x00000001, relocInfo::none, 0); __ emit_data(0x00000002, relocInfo::none, 0); __ emit_data(0x00000003, relocInfo::none, 0); __ emit_data(0x00000004, relocInfo::none, 0); __ emit_data(0x00000005, relocInfo::none, 0); __ emit_data(0x00000006, relocInfo::none, 0); __ emit_data(0x00000007, relocInfo::none, 0); __ emit_data(0x00000008, relocInfo::none, 0); __ emit_data(0x00000009, relocInfo::none, 0); __ emit_data(0x0000000A, relocInfo::none, 0); __ emit_data(0x0000000B, relocInfo::none, 0); __ emit_data(0x0000000C, relocInfo::none, 0); __ emit_data(0x0000000D, relocInfo::none, 0); __ emit_data(0x0000000E, relocInfo::none, 0); __ emit_data(0x0000000F, relocInfo::none, 0); // Q __ emit_data(0x00000000, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); __ emit_data(0x00000001, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); __ emit_data(0x00000002, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); __ emit_data(0x00000003, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); __ emit_data(0x00000004, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); __ emit_data(0x00000005, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); __ emit_data(0x00000006, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); __ emit_data(0x00000007, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); // D - FP __ emit_data(0x00000000, relocInfo::none, 0); // 0.0f __ emit_data(0x3F800000, relocInfo::none, 0); // 1.0f __ emit_data(0x40000000, relocInfo::none, 0); // 2.0f __ emit_data(0x40400000, relocInfo::none, 0); // 3.0f __ emit_data(0x40800000, relocInfo::none, 0); // 4.0f __ emit_data(0x40A00000, relocInfo::none, 0); // 5.0f __ emit_data(0x40C00000, relocInfo::none, 0); // 6.0f __ emit_data(0x40E00000, relocInfo::none, 0); // 7.0f __ emit_data(0x41000000, relocInfo::none, 0); // 8.0f __ emit_data(0x41100000, relocInfo::none, 0); // 9.0f __ emit_data(0x41200000, relocInfo::none, 0); // 10.0f __ emit_data(0x41300000, relocInfo::none, 0); // 11.0f __ emit_data(0x41400000, relocInfo::none, 0); // 12.0f __ emit_data(0x41500000, relocInfo::none, 0); // 13.0f __ emit_data(0x41600000, relocInfo::none, 0); // 14.0f __ emit_data(0x41700000, relocInfo::none, 0); // 15.0f // Q - FP __ emit_data(0x00000000, relocInfo::none, 0); // 0.0d __ emit_data(0x00000000, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); // 1.0d __ emit_data(0x3FF00000, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); // 2.0d __ emit_data(0x40000000, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); // 3.0d __ emit_data(0x40080000, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); // 4.0d __ emit_data(0x40100000, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); // 5.0d __ emit_data(0x40140000, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); // 6.0d __ emit_data(0x40180000, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); // 7.0d __ emit_data(0x401c0000, relocInfo::none, 0); return start; } address generate_vector_reverse_bit_lut(const char *stub_name) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", stub_name); address start = __ pc(); __ emit_data(0x0C040800, relocInfo::none, 0); __ emit_data(0x0E060A02, relocInfo::none, 0); __ emit_data(0x0D050901, relocInfo::none, 0); __ emit_data(0x0F070B03, relocInfo::none, 0); __ emit_data(0x0C040800, relocInfo::none, 0); __ emit_data(0x0E060A02, relocInfo::none, 0); __ emit_data(0x0D050901, relocInfo::none, 0); __ emit_data(0x0F070B03, relocInfo::none, 0); __ emit_data(0x0C040800, relocInfo::none, 0); __ emit_data(0x0E060A02, relocInfo::none, 0); __ emit_data(0x0D050901, relocInfo::none, 0); __ emit_data(0x0F070B03, relocInfo::none, 0); __ emit_data(0x0C040800, relocInfo::none, 0); __ emit_data(0x0E060A02, relocInfo::none, 0); __ emit_data(0x0D050901, relocInfo::none, 0); __ emit_data(0x0F070B03, relocInfo::none, 0); return start; } address generate_vector_reverse_byte_perm_mask_long(const char *stub_name) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", stub_name); address start = __ pc(); __ emit_data(0x04050607, relocInfo::none, 0); __ emit_data(0x00010203, relocInfo::none, 0); __ emit_data(0x0C0D0E0F, relocInfo::none, 0); __ emit_data(0x08090A0B, relocInfo::none, 0); __ emit_data(0x04050607, relocInfo::none, 0); __ emit_data(0x00010203, relocInfo::none, 0); __ emit_data(0x0C0D0E0F, relocInfo::none, 0); __ emit_data(0x08090A0B, relocInfo::none, 0); __ emit_data(0x04050607, relocInfo::none, 0); __ emit_data(0x00010203, relocInfo::none, 0); __ emit_data(0x0C0D0E0F, relocInfo::none, 0); __ emit_data(0x08090A0B, relocInfo::none, 0); __ emit_data(0x04050607, relocInfo::none, 0); __ emit_data(0x00010203, relocInfo::none, 0); __ emit_data(0x0C0D0E0F, relocInfo::none, 0); __ emit_data(0x08090A0B, relocInfo::none, 0); return start; } address generate_vector_reverse_byte_perm_mask_int(const char *stub_name) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", stub_name); address start = __ pc(); __ emit_data(0x00010203, relocInfo::none, 0); __ emit_data(0x04050607, relocInfo::none, 0); __ emit_data(0x08090A0B, relocInfo::none, 0); __ emit_data(0x0C0D0E0F, relocInfo::none, 0); __ emit_data(0x00010203, relocInfo::none, 0); __ emit_data(0x04050607, relocInfo::none, 0); __ emit_data(0x08090A0B, relocInfo::none, 0); __ emit_data(0x0C0D0E0F, relocInfo::none, 0); __ emit_data(0x00010203, relocInfo::none, 0); __ emit_data(0x04050607, relocInfo::none, 0); __ emit_data(0x08090A0B, relocInfo::none, 0); __ emit_data(0x0C0D0E0F, relocInfo::none, 0); __ emit_data(0x00010203, relocInfo::none, 0); __ emit_data(0x04050607, relocInfo::none, 0); __ emit_data(0x08090A0B, relocInfo::none, 0); __ emit_data(0x0C0D0E0F, relocInfo::none, 0); return start; } address generate_vector_reverse_byte_perm_mask_short(const char *stub_name) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", stub_name); address start = __ pc(); __ emit_data(0x02030001, relocInfo::none, 0); __ emit_data(0x06070405, relocInfo::none, 0); __ emit_data(0x0A0B0809, relocInfo::none, 0); __ emit_data(0x0E0F0C0D, relocInfo::none, 0); __ emit_data(0x02030001, relocInfo::none, 0); __ emit_data(0x06070405, relocInfo::none, 0); __ emit_data(0x0A0B0809, relocInfo::none, 0); __ emit_data(0x0E0F0C0D, relocInfo::none, 0); __ emit_data(0x02030001, relocInfo::none, 0); __ emit_data(0x06070405, relocInfo::none, 0); __ emit_data(0x0A0B0809, relocInfo::none, 0); __ emit_data(0x0E0F0C0D, relocInfo::none, 0); __ emit_data(0x02030001, relocInfo::none, 0); __ emit_data(0x06070405, relocInfo::none, 0); __ emit_data(0x0A0B0809, relocInfo::none, 0); __ emit_data(0x0E0F0C0D, relocInfo::none, 0); return start; } address generate_vector_byte_shuffle_mask(const char *stub_name) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", stub_name); address start = __ pc(); __ emit_data(0x70707070, relocInfo::none, 0); __ emit_data(0x70707070, relocInfo::none, 0); __ emit_data(0x70707070, relocInfo::none, 0); __ emit_data(0x70707070, relocInfo::none, 0); __ emit_data(0xF0F0F0F0, relocInfo::none, 0); __ emit_data(0xF0F0F0F0, relocInfo::none, 0); __ emit_data(0xF0F0F0F0, relocInfo::none, 0); __ emit_data(0xF0F0F0F0, relocInfo::none, 0); return start; } address generate_vector_mask_long_double(const char *stub_name, int32_t maskhi, int32_ __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", stub_name); address start = __ pc(); for (int i = 0; i < 8; i++) { __ emit_data(masklo, relocInfo::none, 0); __ emit_data(maskhi, relocInfo::none, 0); } return start; } //------------------------------------------------------------------------------ address generate_vector_byte_perm_mask(const char *stub_name) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", stub_name); address start = __ pc(); __ emit_data(0x00000001, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); __ emit_data(0x00000003, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); __ emit_data(0x00000005, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); __ emit_data(0x00000007, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); __ emit_data(0x00000002, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); __ emit_data(0x00000004, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); __ emit_data(0x00000006, relocInfo::none, 0); __ emit_data(0x00000000, relocInfo::none, 0); return start; } address generate_vector_custom_i32(const char *stub_name, Assembler::AvxVectorLen len int32_t val0, int32_t val1, int32_t val2, int32_t val3, int32_t val4 = 0, int32_t val5 = 0, int32_t val6 = 0, int32_t val7 = 0, int32_t val8 = 0, int32_t val9 = 0, int32_t val10 = 0, int32_t val11 = 0, int32_t val12 = 0, int32_t val13 = 0, int32_t val14 = 0, int32_t val15 = 0) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", stub_name); address start = __ pc(); assert(len != Assembler::AVX_NoVec, "vector len must be specified"); __ emit_data(val0, relocInfo::none, 0); __ emit_data(val1, relocInfo::none, 0); __ emit_data(val2, relocInfo::none, 0); __ emit_data(val3, relocInfo::none, 0); if (len >= Assembler::AVX_256bit) { __ emit_data(val4, relocInfo::none, 0); __ emit_data(val5, relocInfo::none, 0); __ emit_data(val6, relocInfo::none, 0); __ emit_data(val7, relocInfo::none, 0); if (len >= Assembler::AVX_512bit) { __ emit_data(val8, relocInfo::none, 0); __ emit_data(val9, relocInfo::none, 0); __ emit_data(val10, relocInfo::none, 0); __ emit_data(val11, relocInfo::none, 0); __ emit_data(val12, relocInfo::none, 0); __ emit_data(val13, relocInfo::none, 0); __ emit_data(val14, relocInfo::none, 0); __ emit_data(val15, relocInfo::none, 0); } } return start; } //------------------------------------------------------------------------------ // Non-destructive plausibility checks for oops address generate_verify_oop() { StubCodeMark mark(this, "StubRoutines", "verify_oop"); address start = __ pc(); // Incoming arguments on stack after saving rax,: // // [tos ]: saved rdx // [tos + 1]: saved EFLAGS // [tos + 2]: return address // [tos + 3]: char* error message // [tos + 4]: oop object to verify // [tos + 5]: saved rax, - saved by caller and bashed Label exit, error; __ pushf(); __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr())); __ push(rdx); // save rdx // make sure object is 'reasonable' __ movptr(rax, Address(rsp, 4 * wordSize)); // get object __ testptr(rax, rax); __ jcc(Assembler::zero, exit); // if obj is NULL it is ok // Check if the oop is in the right area of memory const int oop_mask = Universe::verify_oop_mask(); const int oop_bits = Universe::verify_oop_bits(); __ mov(rdx, rax); __ andptr(rdx, oop_mask); __ cmpptr(rdx, oop_bits); __ jcc(Assembler::notZero, error); // make sure klass is 'reasonable', which is not zero. __ movptr(rax, Address(rax, oopDesc::klass_offset_in_bytes())); // get klass __ testptr(rax, rax); __ jcc(Assembler::zero, error); // if klass is NULL it is broken // return if everything seems ok __ bind(exit); __ movptr(rax, Address(rsp, 5 * wordSize)); // get saved rax, back __ pop(rdx); // restore rdx __ popf(); // restore EFLAGS __ ret(3 * wordSize); // pop arguments // handle errors __ bind(error); __ movptr(rax, Address(rsp, 5 * wordSize)); // get saved rax, back __ pop(rdx); // get saved rdx back __ popf(); // get saved EFLAGS off stack -- will be ignored __ pusha(); // push registers (eip = return address & msg are already pushed) BLOCK_COMMENT("call MacroAssembler::debug"); __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32))); __ hlt(); return start; } // Copy 64 bytes chunks // // Inputs: // from - source array address // to_from - destination array address - from // qword_count - 8-bytes element count, negative // void xmm_copy_forward(Register from, Register to_from, Register qword_count) { assert( UseSSE >= 2, "supported cpu only" ); Label L_copy_64_bytes_loop, L_copy_64_bytes, L_copy_8_bytes, L_exit; // Copy 64-byte chunks __ jmpb(L_copy_64_bytes); __ align(OptoLoopAlignment); __ BIND(L_copy_64_bytes_loop); if (UseUnalignedLoadStores) { if (UseAVX > 2) { __ evmovdqul(xmm0, Address(from, 0), Assembler::AVX_512bit); __ evmovdqul(Address(from, to_from, Address::times_1, 0), xmm0, Assembler::AVX_512bit) } else if (UseAVX == 2) { __ vmovdqu(xmm0, Address(from, 0)); __ vmovdqu(Address(from, to_from, Address::times_1, 0), xmm0); __ vmovdqu(xmm1, Address(from, 32)); __ vmovdqu(Address(from, to_from, Address::times_1, 32), xmm1); } else { __ movdqu(xmm0, Address(from, 0)); __ movdqu(Address(from, to_from, Address::times_1, 0), xmm0); __ movdqu(xmm1, Address(from, 16)); __ movdqu(Address(from, to_from, Address::times_1, 16), xmm1); __ movdqu(xmm2, Address(from, 32)); __ movdqu(Address(from, to_from, Address::times_1, 32), xmm2); __ movdqu(xmm3, Address(from, 48)); __ movdqu(Address(from, to_from, Address::times_1, 48), xmm3); } } else { __ movq(xmm0, Address(from, 0)); __ movq(Address(from, to_from, Address::times_1, 0), xmm0); __ movq(xmm1, Address(from, 8)); __ movq(Address(from, to_from, Address::times_1, 8), xmm1); __ movq(xmm2, Address(from, 16)); __ movq(Address(from, to_from, Address::times_1, 16), xmm2); __ movq(xmm3, Address(from, 24)); __ movq(Address(from, to_from, Address::times_1, 24), xmm3); __ movq(xmm4, Address(from, 32)); __ movq(Address(from, to_from, Address::times_1, 32), xmm4); __ movq(xmm5, Address(from, 40)); __ movq(Address(from, to_from, Address::times_1, 40), xmm5); __ movq(xmm6, Address(from, 48)); __ movq(Address(from, to_from, Address::times_1, 48), xmm6); __ movq(xmm7, Address(from, 56)); __ movq(Address(from, to_from, Address::times_1, 56), xmm7); } __ addl(from, 64); __ BIND(L_copy_64_bytes); __ subl(qword_count, 8); __ jcc(Assembler::greaterEqual, L_copy_64_bytes_loop); if (UseUnalignedLoadStores && (UseAVX == 2)) { // clean upper bits of YMM registers __ vpxor(xmm0, xmm0); __ vpxor(xmm1, xmm1); } __ addl(qword_count, 8); __ jccb(Assembler::zero, L_exit); // // length is too short, just copy qwords // __ BIND(L_copy_8_bytes); __ movq(xmm0, Address(from, 0)); __ movq(Address(from, to_from, Address::times_1), xmm0); __ addl(from, 8); __ decrement(qword_count); __ jcc(Assembler::greater, L_copy_8_bytes); __ BIND(L_exit); } address generate_disjoint_copy(BasicType t, bool aligned, Address::ScaleFactor sf, address* entry, const char *name, bool dest_uninitialized = false) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); Label L_0_count, L_exit, L_skip_align1, L_skip_align2, L_copy_byte; Label L_copy_2_bytes, L_copy_4_bytes, L_copy_64_bytes; int shift = Address::times_ptr - sf; const Register from = rsi; // source array address const Register to = rdi; // destination array address const Register count = rcx; // elements count const Register to_from = to; // (to - from) const Register saved_to = rdx; // saved destination array address __ enter(); // required for proper stackwalking of RuntimeStub frame __ push(rsi); __ push(rdi); __ movptr(from , Address(rsp, 12+ 4)); __ movptr(to , Address(rsp, 12+ 8)); __ movl(count, Address(rsp, 12+ 12)); if (entry != NULL) { *entry = __ pc(); // Entry point from conjoint arraycopy stub. BLOCK_COMMENT("Entry:"); } if (t == T_OBJECT) { __ testl(count, count); __ jcc(Assembler::zero, L_0_count); } DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT; if (dest_uninitialized) { decorators |= IS_DEST_UNINITIALIZED; } if (aligned) { decorators |= ARRAYCOPY_ALIGNED; } BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); bs->arraycopy_prologue(_masm, decorators, t, from, to, count); { bool add_entry = (t != T_OBJECT && (!aligned || t == T_INT)); // UnsafeCopyMemory page error: continue after ucm UnsafeCopyMemoryMark ucmm(this, add_entry, true); __ subptr(to, from); // to --> to_from __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) { // align source address at 4 bytes address boundary if (t == T_BYTE) { // One byte misalignment happens only for byte arrays __ testl(from, 1); __ jccb(Assembler::zero, L_skip_align1); __ movb(rax, Address(from, 0)); __ movb(Address(from, to_from, Address::times_1, 0), rax); __ increment(from); __ decrement(count); __ BIND(L_skip_align1); } // Two bytes misalignment happens only for byte and short (char) arrays __ testl(from, 2); __ jccb(Assembler::zero, L_skip_align2); __ movw(rax, Address(from, 0)); __ movw(Address(from, to_from, Address::times_1, 0), rax); __ addptr(from, 2); __ subl(count, 1<<(shift-1)); __ BIND(L_skip_align2); } if (!UseXMMForArrayCopy) { __ mov(rax, count); // save 'count' __ shrl(count, shift); // bytes count __ addptr(to_from, from);// restore 'to' __ rep_mov(); __ subptr(to_from, from);// restore 'to_from' __ mov(count, rax); // restore 'count' __ jmpb(L_copy_2_bytes); // all dwords were copied } else { if (!UseUnalignedLoadStores) { // align to 8 bytes, we know we are 4 byte aligned to start __ testptr(from, 4); __ jccb(Assembler::zero, L_copy_64_bytes); __ movl(rax, Address(from, 0)); __ movl(Address(from, to_from, Address::times_1, 0), rax); __ addptr(from, 4); __ subl(count, 1<<shift); } __ BIND(L_copy_64_bytes); __ mov(rax, count); __ shrl(rax, shift+1); // 8 bytes chunk count // // Copy 8-byte chunks through XMM registers, 8 per iteration of the loop // xmm_copy_forward(from, to_from, rax); } // copy tailing dword __ BIND(L_copy_4_bytes); __ testl(count, 1<<shift); __ jccb(Assembler::zero, L_copy_2_bytes); __ movl(rax, Address(from, 0)); __ movl(Address(from, to_from, Address::times_1, 0), rax); if (t == T_BYTE || t == T_SHORT) { __ addptr(from, 4); __ BIND(L_copy_2_bytes); // copy tailing word __ testl(count, 1<<(shift-1)); __ jccb(Assembler::zero, L_copy_byte); __ movw(rax, Address(from, 0)); __ movw(Address(from, to_from, Address::times_1, 0), rax); if (t == T_BYTE) { __ addptr(from, 2); __ BIND(L_copy_byte); // copy tailing byte __ testl(count, 1); __ jccb(Assembler::zero, L_exit); __ movb(rax, Address(from, 0)); __ movb(Address(from, to_from, Address::times_1, 0), rax); __ BIND(L_exit); } else { __ BIND(L_copy_byte); } } else { __ BIND(L_copy_2_bytes); } } __ movl(count, Address(rsp, 12+12)); // reread 'count' bs->arraycopy_epilogue(_masm, decorators, t, from, to, count); if (t == T_OBJECT) { __ BIND(L_0_count); } inc_copy_counter_np(t); __ pop(rdi); __ pop(rsi); __ leave(); // required for proper stackwalking of RuntimeStub frame __ vzeroupper(); __ xorptr(rax, rax); // return 0 __ ret(0); return start; } address generate_fill(BasicType t, bool aligned, const char *name) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); BLOCK_COMMENT("Entry:"); const Register to = rdi; // source array address const Register value = rdx; // value const Register count = rsi; // elements count __ enter(); // required for proper stackwalking of RuntimeStub frame __ push(rsi); __ push(rdi); __ movptr(to , Address(rsp, 12+ 4)); __ movl(value, Address(rsp, 12+ 8)); __ movl(count, Address(rsp, 12+ 12)); __ generate_fill(t, aligned, to, value, count, rax, xmm0); __ pop(rdi); __ pop(rsi); __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); return start; } address generate_conjoint_copy(BasicType t, bool aligned, Address::ScaleFactor sf, address nooverlap_target, address* entry, const char *name, bool dest_uninitialized = false) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); Label L_0_count, L_exit, L_skip_align1, L_skip_align2, L_copy_byte; Label L_copy_2_bytes, L_copy_4_bytes, L_copy_8_bytes, L_copy_8_bytes_loop; int shift = Address::times_ptr - sf; const Register src = rax; // source array address const Register dst = rdx; // destination array address const Register from = rsi; // source array address const Register to = rdi; // destination array address const Register count = rcx; // elements count const Register end = rax; // array end address __ enter(); // required for proper stackwalking of RuntimeStub frame __ push(rsi); __ push(rdi); __ movptr(src , Address(rsp, 12+ 4)); // from __ movptr(dst , Address(rsp, 12+ 8)); // to __ movl2ptr(count, Address(rsp, 12+12)); // count if (entry != NULL) { *entry = __ pc(); // Entry point from generic arraycopy stub. BLOCK_COMMENT("Entry:"); } // nooverlap_target expects arguments in rsi and rdi. __ mov(from, src); __ mov(to , dst); // arrays overlap test: dispatch to disjoint stub if necessary. RuntimeAddress nooverlap(nooverlap_target); __ cmpptr(dst, src); __ lea(end, Address(src, count, sf, 0)); // src + count * elem_size __ jump_cc(Assembler::belowEqual, nooverlap); __ cmpptr(dst, end); __ jump_cc(Assembler::aboveEqual, nooverlap); if (t == T_OBJECT) { __ testl(count, count); __ jcc(Assembler::zero, L_0_count); } DecoratorSet decorators = IN_HEAP | IS_ARRAY; if (dest_uninitialized) { decorators |= IS_DEST_UNINITIALIZED; } if (aligned) { decorators |= ARRAYCOPY_ALIGNED; } BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); bs->arraycopy_prologue(_masm, decorators, t, from, to, count); { bool add_entry = (t != T_OBJECT && (!aligned || t == T_INT)); // UnsafeCopyMemory page error: continue after ucm UnsafeCopyMemoryMark ucmm(this, add_entry, true); // copy from high to low __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp if (t == T_BYTE || t == T_SHORT) { // Align the end of destination array at 4 bytes address boundary __ lea(end, Address(dst, count, sf, 0)); if (t == T_BYTE) { // One byte misalignment happens only for byte arrays __ testl(end, 1); __ jccb(Assembler::zero, L_skip_align1); __ decrement(count); __ movb(rdx, Address(from, count, sf, 0)); __ movb(Address(to, count, sf, 0), rdx); __ BIND(L_skip_align1); } // Two bytes misalignment happens only for byte and short (char) arrays __ testl(end, 2); __ jccb(Assembler::zero, L_skip_align2); __ subptr(count, 1<<(shift-1)); __ movw(rdx, Address(from, count, sf, 0)); __ movw(Address(to, count, sf, 0), rdx); __ BIND(L_skip_align2); __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element __ jcc(Assembler::below, L_copy_4_bytes); } if (!UseXMMForArrayCopy) { __ std(); __ mov(rax, count); // Save 'count' __ mov(rdx, to); // Save 'to' __ lea(rsi, Address(from, count, sf, -4)); __ lea(rdi, Address(to , count, sf, -4)); __ shrptr(count, shift); // bytes count __ rep_mov(); __ cld(); __ mov(count, rax); // restore 'count' __ andl(count, (1<<shift)-1); // mask the number of rest elements __ movptr(from, Address(rsp, 12+4)); // reread 'from' __ mov(to, rdx); // restore 'to' __ jmpb(L_copy_2_bytes); // all dword were copied } else { // Align to 8 bytes the end of array. It is aligned to 4 bytes already. __ testptr(end, 4); __ jccb(Assembler::zero, L_copy_8_bytes); __ subl(count, 1<<shift); __ movl(rdx, Address(from, count, sf, 0)); __ movl(Address(to, count, sf, 0), rdx); __ jmpb(L_copy_8_bytes); __ align(OptoLoopAlignment); // Move 8 bytes __ BIND(L_copy_8_bytes_loop); __ movq(xmm0, Address(from, count, sf, 0)); __ movq(Address(to, count, sf, 0), xmm0); __ BIND(L_copy_8_bytes); __ subl(count, 2<<shift); __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop); __ addl(count, 2<<shift); } __ BIND(L_copy_4_bytes); // copy prefix qword __ testl(count, 1<<shift); __ jccb(Assembler::zero, L_copy_2_bytes); __ movl(rdx, Address(from, count, sf, -4)); __ movl(Address(to, count, sf, -4), rdx); if (t == T_BYTE || t == T_SHORT) { __ subl(count, (1<<shift)); __ BIND(L_copy_2_bytes); // copy prefix dword __ testl(count, 1<<(shift-1)); __ jccb(Assembler::zero, L_copy_byte); __ movw(rdx, Address(from, count, sf, -2)); __ movw(Address(to, count, sf, -2), rdx); if (t == T_BYTE) { __ subl(count, 1<<(shift-1)); __ BIND(L_copy_byte); // copy prefix byte __ testl(count, 1); __ jccb(Assembler::zero, L_exit); __ movb(rdx, Address(from, 0)); __ movb(Address(to, 0), rdx); __ BIND(L_exit); } else { __ BIND(L_copy_byte); } } else { __ BIND(L_copy_2_bytes); } } __ movl2ptr(count, Address(rsp, 12+12)); // reread count bs->arraycopy_epilogue(_masm, decorators, t, from, to, count); if (t == T_OBJECT) { __ BIND(L_0_count); } inc_copy_counter_np(t); __ pop(rdi); __ pop(rsi); __ leave(); // required for proper stackwalking of RuntimeStub frame __ xorptr(rax, rax); // return 0 __ ret(0); return start; } address generate_disjoint_long_copy(address* entry, const char *name) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); Label L_copy_8_bytes, L_copy_8_bytes_loop; const Register from = rax; // source array address const Register to = rdx; // destination array address const Register count = rcx; // elements count const Register to_from = rdx; // (to - from) __ enter(); // required for proper stackwalking of RuntimeStub frame __ movptr(from , Address(rsp, 8+0)); // from __ movptr(to , Address(rsp, 8+4)); // to __ movl2ptr(count, Address(rsp, 8+8)); // count *entry = __ pc(); // Entry point from conjoint arraycopy stub. BLOCK_COMMENT("Entry:"); { // UnsafeCopyMemory page error: continue after ucm UnsafeCopyMemoryMark ucmm(this, true, true); __ subptr(to, from); // to --> to_from if (UseXMMForArrayCopy) { xmm_copy_forward(from, to_from, count); } else { __ jmpb(L_copy_8_bytes); __ align(OptoLoopAlignment); __ BIND(L_copy_8_bytes_loop); __ fild_d(Address(from, 0)); __ fistp_d(Address(from, to_from, Address::times_1)); __ addptr(from, 8); __ BIND(L_copy_8_bytes); __ decrement(count); __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop); } } inc_copy_counter_np(T_LONG); __ leave(); // required for proper stackwalking of RuntimeStub frame __ vzeroupper(); __ xorptr(rax, rax); // return 0 __ ret(0); return start; } address generate_conjoint_long_copy(address nooverlap_target, address* entry, const char *name) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); Label L_copy_8_bytes, L_copy_8_bytes_loop; const Register from = rax; // source array address const Register to = rdx; // destination array address const Register count = rcx; // elements count const Register end_from = rax; // source array end address __ enter(); // required for proper stackwalking of RuntimeStub frame __ movptr(from , Address(rsp, 8+0)); // from __ movptr(to , Address(rsp, 8+4)); // to __ movl2ptr(count, Address(rsp, 8+8)); // count *entry = __ pc(); // Entry point from generic arraycopy stub. BLOCK_COMMENT("Entry:"); // arrays overlap test __ cmpptr(to, from); RuntimeAddress nooverlap(nooverlap_target); __ jump_cc(Assembler::belowEqual, nooverlap); __ lea(end_from, Address(from, count, Address::times_8, 0)); __ cmpptr(to, end_from); __ movptr(from, Address(rsp, 8)); // from __ jump_cc(Assembler::aboveEqual, nooverlap); { // UnsafeCopyMemory page error: continue after ucm UnsafeCopyMemoryMark ucmm(this, true, true); __ jmpb(L_copy_8_bytes); __ align(OptoLoopAlignment); __ BIND(L_copy_8_bytes_loop); if (UseXMMForArrayCopy) { __ movq(xmm0, Address(from, count, Address::times_8)); __ movq(Address(to, count, Address::times_8), xmm0); } else { __ fild_d(Address(from, count, Address::times_8)); __ fistp_d(Address(to, count, Address::times_8)); } __ BIND(L_copy_8_bytes); __ decrement(count); __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop); } inc_copy_counter_np(T_LONG); __ leave(); // required for proper stackwalking of RuntimeStub frame __ xorptr(rax, rax); // return 0 __ ret(0); return start; } // Helper for generating a dynamic type check. // The sub_klass must be one of {rbx, rdx, rsi}. // The temp is killed. void generate_type_check(Register sub_klass, Address& super_check_offset_addr, Address& super_klass_addr, Register temp, Label* L_success, Label* L_failure) { BLOCK_COMMENT("type_check:"); Label L_fallthrough; #define LOCAL_JCC(assembler_con, label_ptr) \ if (label_ptr != NULL) __ jcc(assembler_con, *(label_ptr)); \ else __ jcc(assembler_con, L_fallthrough) /*omit semi*/ // The following is a strange variation of the fast path which requires // one less register, because needed values are on the argument stack. // __ check_klass_subtype_fast_path(sub_klass, *super_klass*, temp, // L_success, L_failure, NULL); assert_different_registers(sub_klass, temp); int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); // if the pointers are equal, we are done (e.g., String[] elements) __ cmpptr(sub_klass, super_klass_addr); LOCAL_JCC(Assembler::equal, L_success); // check the supertype display: __ movl2ptr(temp, super_check_offset_addr); Address super_check_addr(sub_klass, temp, Address::times_1, 0); __ movptr(temp, super_check_addr); // load displayed supertype __ cmpptr(temp, super_klass_addr); // test the super type LOCAL_JCC(Assembler::equal, L_success); // if it was a primary super, we can just fail immediately __ cmpl(super_check_offset_addr, sc_offset); LOCAL_JCC(Assembler::notEqual, L_failure); // The repne_scan instruction uses fixed registers, which will get spilled. // We happen to know this works best when super_klass is in rax. Register super_klass = temp; __ movptr(super_klass, super_klass_addr); __ check_klass_subtype_slow_path(sub_klass, super_klass, noreg, noreg, L_success, L_failure); __ bind(L_fallthrough); if (L_success == NULL) { BLOCK_COMMENT("L_success:"); } if (L_failure == NULL) { BLOCK_COMMENT("L_failure:"); } #undef LOCAL_JCC } // // Generate checkcasting array copy stub // // Input: // 4(rsp) - source array address // 8(rsp) - destination array address // 12(rsp) - element count, can be zero // 16(rsp) - size_t ckoff (super_check_offset) // 20(rsp) - oop ckval (super_klass) // // Output: // rax, == 0 - success // rax, == -1^K - failure, where K is partial transfer count // address generate_checkcast_copy(const char *name, address* entry, bool dest_uninitializ __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); Label L_load_element, L_store_element, L_do_card_marks, L_done; // register use: // rax, rdx, rcx -- loop control (end_from, end_to, count) // rdi, rsi -- element access (oop, klass) // rbx, -- temp const Register from = rax; // source array address const Register to = rdx; // destination array address const Register length = rcx; // elements count const Register elem = rdi; // each oop copied const Register elem_klass = rsi; // each elem._klass (sub_klass) const Register temp = rbx; // lone remaining temp __ enter(); // required for proper stackwalking of RuntimeStub frame __ push(rsi); __ push(rdi); __ push(rbx); Address from_arg(rsp, 16+ 4); // from Address to_arg(rsp, 16+ 8); // to Address length_arg(rsp, 16+12); // elements count Address ckoff_arg(rsp, 16+16); // super_check_offset Address ckval_arg(rsp, 16+20); // super_klass // Load up: __ movptr(from, from_arg); __ movptr(to, to_arg); __ movl2ptr(length, length_arg); if (entry != NULL) { *entry = __ pc(); // Entry point from generic arraycopy stub. BLOCK_COMMENT("Entry:"); } //--------------------------------------------------------------- // Assembler stub will be used for this call to arraycopy // if the two arrays are subtypes of Object[] but the // destination array type is not equal to or a supertype // of the source type. Each element must be separately // checked. // Loop-invariant addresses. They are exclusive end pointers. Address end_from_addr(from, length, Address::times_ptr, 0); Address end_to_addr(to, length, Address::times_ptr, 0); Register end_from = from; // re-use Register end_to = to; // re-use Register count = length; // re-use // Loop-variant addresses. They assume post-incremented count < 0. Address from_element_addr(end_from, count, Address::times_ptr, 0); Address to_element_addr(end_to, count, Address::times_ptr, 0); Address elem_klass_addr(elem, oopDesc::klass_offset_in_bytes()); DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_CHECKCAST; if (dest_uninitialized) { decorators |= IS_DEST_UNINITIALIZED; } BasicType type = T_OBJECT; BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); bs->arraycopy_prologue(_masm, decorators, type, from, to, count); // Copy from low to high addresses, indexed from the end of each array. __ lea(end_from, end_from_addr); __ lea(end_to, end_to_addr); assert(length == count, ""); // else fix next line: __ negptr(count); // negate and test the length __ jccb(Assembler::notZero, L_load_element); // Empty array: Nothing to do. __ xorptr(rax, rax); // return 0 on (trivial) success __ jmp(L_done); // ======== begin loop ======== // (Loop is rotated; its entry is L_load_element.) // Loop control: // for (count = -count; count != 0; count++) // Base pointers src, dst are biased by 8*count,to last element. __ align(OptoLoopAlignment); __ BIND(L_store_element); __ movptr(to_element_addr, elem); // store the oop __ increment(count); // increment the count toward zero __ jccb(Assembler::zero, L_do_card_marks); // ======== loop entry is here ======== __ BIND(L_load_element); __ movptr(elem, from_element_addr); // load the oop __ testptr(elem, elem); __ jccb(Assembler::zero, L_store_element); // (Could do a trick here: Remember last successful non-null // element stored and make a quick oop equality check on it.) __ movptr(elem_klass, elem_klass_addr); // query the object klass generate_type_check(elem_klass, ckoff_arg, ckval_arg, temp, &L_store_element, NULL); // (On fall-through, we have failed the element type check.) // ======== end loop ======== // It was a real error; we must depend on the caller to finish the job. // Register "count" = -1 * number of *remaining* oops, length_arg = *total* oops. // Emit GC store barriers for the oops we have copied (length_arg + count), // and report their number to the caller. assert_different_registers(to, count, rax); Label L_post_barrier; __ addl(count, length_arg); // transfers = (length - remaining) __ movl2ptr(rax, count); // save the value __ notptr(rax); // report (-1^K) to caller (does not affect flags) __ jccb(Assembler::notZero, L_post_barrier); __ jmp(L_done); // K == 0, nothing was copied, skip post barrier // Come here on success only. __ BIND(L_do_card_marks); __ xorptr(rax, rax); // return 0 on success __ movl2ptr(count, length_arg); __ BIND(L_post_barrier); __ movptr(to, to_arg); // reload bs->arraycopy_epilogue(_masm, decorators, type, from, to, count); // Common exit point (success or failure). __ BIND(L_done); __ pop(rbx); __ pop(rdi); __ pop(rsi); inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr); __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); return start; } // // Generate 'unsafe' array copy stub // Though just as safe as the other stubs, it takes an unscaled // size_t argument instead of an element count. // // Input: // 4(rsp) - source array address // 8(rsp) - destination array address // 12(rsp) - byte count, can be zero // // Output: // rax, == 0 - success // rax, == -1 - need to call System.arraycopy // // Examines the alignment of the operands and dispatches // to a long, int, short, or byte copy loop. // address generate_unsafe_copy(const char *name, --> -------------------- --> maximum size reached --> -------------------- ¤ Dauer der Verarbeitung: 0.64 Sekunden (vorverarbeitet) ¤*© Formatika GbR, Deutschland |
|
2026-03-28