diff --git a/src/hotspot/cpu/aarch64/foreign_globals_aarch64.hpp b/src/hotspot/cpu/aarch64/foreign_globals_aarch64.hpp index f7441641a894d..f3748e20b1476 100644 --- a/src/hotspot/cpu/aarch64/foreign_globals_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/foreign_globals_aarch64.hpp @@ -28,22 +28,7 @@ #include "asm/macroAssembler.hpp" #include "utilities/growableArray.hpp" -#define __ _masm-> - -struct VectorRegister { - static const size_t VECTOR_MAX_WIDTH_BITS = 128; - static const size_t VECTOR_MAX_WIDTH_BYTES = VECTOR_MAX_WIDTH_BITS / 8; - static const size_t VECTOR_MAX_WIDTH_U64S = VECTOR_MAX_WIDTH_BITS / 64; - static const size_t VECTOR_MAX_WIDTH_FLOATS = VECTOR_MAX_WIDTH_BITS / 32; - static const size_t VECTOR_MAX_WIDTH_DOUBLES = VECTOR_MAX_WIDTH_BITS / 64; - - union { - uint8_t bits[VECTOR_MAX_WIDTH_BYTES]; - uint64_t u64[VECTOR_MAX_WIDTH_U64S]; - float f[VECTOR_MAX_WIDTH_FLOATS]; - double d[VECTOR_MAX_WIDTH_DOUBLES]; - }; -}; +constexpr size_t float_reg_size = 16; // bytes struct ABIDescriptor { GrowableArray _integer_argument_registers; diff --git a/src/hotspot/cpu/aarch64/universalNativeInvoker_aarch64.cpp b/src/hotspot/cpu/aarch64/universalNativeInvoker_aarch64.cpp index 288f5dadc2578..8139b700cf746 100644 --- a/src/hotspot/cpu/aarch64/universalNativeInvoker_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/universalNativeInvoker_aarch64.cpp @@ -28,6 +28,8 @@ #include "memory/resourceArea.hpp" #include "prims/universalNativeInvoker.hpp" +#define __ _masm-> + void ProgrammableInvoker::Generator::generate() { __ enter(); @@ -73,7 +75,7 @@ void ProgrammableInvoker::Generator::generate() { __ bind(Ldone); for (int i = 0; i < _abi->_vector_argument_registers.length(); i++) { - ssize_t offs = _layout->arguments_vector + i * sizeof(VectorRegister); + ssize_t offs = _layout->arguments_vector + i * float_reg_size; __ ldrq(_abi->_vector_argument_registers.at(i), Address(Rctx, offs)); } @@ -99,7 +101,7 @@ void ProgrammableInvoker::Generator::generate() { } for (int i = 0; i < _abi->_vector_return_registers.length(); i++) { - ssize_t offs = _layout->returns_vector + i * sizeof(VectorRegister); + ssize_t offs = _layout->returns_vector + i * float_reg_size; __ strq(_abi->_vector_return_registers.at(i), Address(Rctx, offs)); } diff --git a/src/hotspot/cpu/aarch64/universalUpcallHandler_aarch64.cpp b/src/hotspot/cpu/aarch64/universalUpcallHandler_aarch64.cpp index 3cb524ca9a592..598364d2ec888 100644 --- a/src/hotspot/cpu/aarch64/universalUpcallHandler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/universalUpcallHandler_aarch64.cpp @@ -27,6 +27,8 @@ #include "memory/resourceArea.hpp" #include "prims/universalUpcallHandler.hpp" +#define __ _masm-> + // 1. Create buffer according to layout // 2. Load registers & stack args into buffer // 3. Call upcall helper with upcall handler instance & buffer pointer (C++ ABI) @@ -63,7 +65,7 @@ address ProgrammableUpcallHandler::generate_upcall_stub(jobject rec, jobject jab for (int i = 0; i < abi._vector_argument_registers.length(); i++) { FloatRegister reg = abi._vector_argument_registers.at(i); - ssize_t offset = layout.arguments_vector + i * sizeof(VectorRegister); + ssize_t offset = layout.arguments_vector + i * float_reg_size; __ strq(reg, Address(sp, offset)); } @@ -84,7 +86,7 @@ address ProgrammableUpcallHandler::generate_upcall_stub(jobject rec, jobject jab for (int i = 0; i < abi._vector_return_registers.length(); i++) { FloatRegister reg = abi._vector_return_registers.at(i); - ssize_t offs = layout.returns_vector + i * sizeof(VectorRegister); + ssize_t offs = layout.returns_vector + i * float_reg_size; __ ldrq(reg, Address(sp, offs)); } diff --git a/src/hotspot/cpu/aarch64/vmreg_aarch64.cpp b/src/hotspot/cpu/aarch64/vmreg_aarch64.cpp index 3e95168c8df66..1cf7bdb553a17 100644 --- a/src/hotspot/cpu/aarch64/vmreg_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/vmreg_aarch64.cpp @@ -54,7 +54,6 @@ void VMRegImpl::set_regName() { #define INTEGER_TYPE 0 #define VECTOR_TYPE 1 -#define X87_TYPE 2 #define STACK_TYPE 3 VMReg VMRegImpl::vmStorageToVMReg(int type, int index) { diff --git a/src/hotspot/cpu/x86/foreign_globals_x86.hpp b/src/hotspot/cpu/x86/foreign_globals_x86.hpp index ffc53f25a6add..aa25c454c42f4 100644 --- a/src/hotspot/cpu/x86/foreign_globals_x86.hpp +++ b/src/hotspot/cpu/x86/foreign_globals_x86.hpp @@ -27,22 +27,7 @@ #include "asm/macroAssembler.hpp" #include "utilities/growableArray.hpp" -#define __ _masm-> - -struct VectorRegister { - static const size_t VECTOR_MAX_WIDTH_BITS = 512; // AVX-512 (64-byte) vector types - static const size_t VECTOR_MAX_WIDTH_BYTES = VECTOR_MAX_WIDTH_BITS / 8; - static const size_t VECTOR_MAX_WIDTH_U64S = VECTOR_MAX_WIDTH_BITS / 64; - static const size_t VECTOR_MAX_WIDTH_FLOATS = VECTOR_MAX_WIDTH_BITS / 32; - static const size_t VECTOR_MAX_WIDTH_DOUBLES = VECTOR_MAX_WIDTH_BITS / 64; - - union { - uint8_t bits[VECTOR_MAX_WIDTH_BYTES]; - uint64_t u64[VECTOR_MAX_WIDTH_U64S]; - float f[VECTOR_MAX_WIDTH_FLOATS]; - double d[VECTOR_MAX_WIDTH_DOUBLES]; - }; -}; +constexpr size_t xmm_reg_size = 16; // size of XMM reg struct ABIDescriptor { GrowableArray _integer_argument_registers; diff --git a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp index 0b81aaa0b0bb5..0de0e5eb98398 100644 --- a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp +++ b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp @@ -3462,10 +3462,10 @@ bool target_uses_register(VMReg reg) { #endif }; -address SharedRuntime::make_native_invoker(address call_target, - int shadow_space_bytes, - const GrowableArray& input_registers, - const GrowableArray& output_registers) { +BufferBlob* SharedRuntime::make_native_invoker(address call_target, + int shadow_space_bytes, + const GrowableArray& input_registers, + const GrowableArray& output_registers) { BufferBlob* _invoke_native_blob = BufferBlob::create("nep_invoker_blob", native_invoker_code_size); if (_invoke_native_blob == NULL) return NULL; // allocation failure @@ -3475,7 +3475,7 @@ address SharedRuntime::make_native_invoker(address call_target, g.generate(); code.log_section_sizes("nep_invoker_blob"); - return _invoke_native_blob->code_begin(); + return _invoke_native_blob; } void NativeInvokerGenerator::generate() { diff --git a/src/hotspot/cpu/x86/universalNativeInvoker_x86.cpp b/src/hotspot/cpu/x86/universalNativeInvoker_x86.cpp index d3a45dc3d2811..ec1f1575dcb95 100644 --- a/src/hotspot/cpu/x86/universalNativeInvoker_x86.cpp +++ b/src/hotspot/cpu/x86/universalNativeInvoker_x86.cpp @@ -27,6 +27,8 @@ #include "memory/resourceArea.hpp" #include "prims/universalNativeInvoker.hpp" +#define __ _masm-> + void ProgrammableInvoker::Generator::generate() { __ enter(); @@ -73,14 +75,8 @@ void ProgrammableInvoker::Generator::generate() { // [8] -> 512 bit -> zmm XMMRegister reg = _abi->_vector_argument_registers.at(i); - size_t offs = _layout->arguments_vector + i * sizeof(VectorRegister); - if (UseAVX >= 3) { - __ evmovdqul(reg, Address(ctxt_reg, (int)offs), Assembler::AVX_512bit); - } else if (UseAVX >= 1) { - __ vmovdqu(reg, Address(ctxt_reg, (int)offs)); - } else { - __ movdqu(reg, Address(ctxt_reg, (int)offs)); - } + size_t offs = _layout->arguments_vector + i * xmm_reg_size; + __ movdqu(reg, Address(ctxt_reg, (int)offs)); } for (int i = 0; i < _abi->_integer_argument_registers.length(); i++) { @@ -115,14 +111,8 @@ void ProgrammableInvoker::Generator::generate() { // [8] -> 512 bit -> zmm (AVX-512, aka AVX3) XMMRegister reg = _abi->_vector_return_registers.at(i); - size_t offs = _layout->returns_vector + i * sizeof(VectorRegister); - if (UseAVX >= 3) { - __ evmovdqul(Address(ctxt_reg, (int)offs), reg, Assembler::AVX_512bit); - } else if (UseAVX >= 1) { - __ vmovdqu(Address(ctxt_reg, (int)offs), reg); - } else { - __ movdqu(Address(ctxt_reg, (int)offs), reg); - } + size_t offs = _layout->returns_vector + i * xmm_reg_size; + __ movdqu(Address(ctxt_reg, (int)offs), reg); } for (size_t i = 0; i < _abi->_X87_return_registers_noof; i++) { diff --git a/src/hotspot/cpu/x86/universalUpcallHandler_x86.cpp b/src/hotspot/cpu/x86/universalUpcallHandler_x86.cpp index b26871dc57f2d..0cbf716d32785 100644 --- a/src/hotspot/cpu/x86/universalUpcallHandler_x86.cpp +++ b/src/hotspot/cpu/x86/universalUpcallHandler_x86.cpp @@ -26,6 +26,8 @@ #include "memory/resourceArea.hpp" #include "prims/universalUpcallHandler.hpp" +#define __ _masm-> + // 1. Create buffer according to layout // 2. Load registers & stack args into buffer // 3. Call upcall helper with upcall handler instance & buffer pointer (C++ ABI) @@ -41,7 +43,7 @@ address ProgrammableUpcallHandler::generate_upcall_stub(jobject rec, jobject jab MacroAssembler* _masm = new MacroAssembler(&buffer); int stack_alignment_C = 16; // bytes int register_size = sizeof(uintptr_t); - int buffer_alignment = sizeof(VectorRegister); + int buffer_alignment = xmm_reg_size; // stub code __ enter(); @@ -76,14 +78,8 @@ address ProgrammableUpcallHandler::generate_upcall_stub(jobject rec, jobject jab for (int i = 0; i < abi._vector_argument_registers.length(); i++) { XMMRegister reg = abi._vector_argument_registers.at(i); - size_t offs = buffer_offset + layout.arguments_vector + i * sizeof(VectorRegister); - if (UseAVX >= 3) { - __ evmovdqul(Address(rsp, (int)offs), reg, Assembler::AVX_512bit); - } else if (UseAVX >= 1) { - __ vmovdqu(Address(rsp, (int)offs), reg); - } else { - __ movdqu(Address(rsp, (int)offs), reg); - } + size_t offs = buffer_offset + layout.arguments_vector + i * xmm_reg_size; + __ movdqu(Address(rsp, (int)offs), reg); } // Capture prev stack pointer (stack arguments base) @@ -121,14 +117,8 @@ address ProgrammableUpcallHandler::generate_upcall_stub(jobject rec, jobject jab for (int i = 0; i < abi._vector_return_registers.length(); i++) { XMMRegister reg = abi._vector_return_registers.at(i); - size_t offs = buffer_offset + layout.returns_vector + i * sizeof(VectorRegister); - if (UseAVX >= 3) { - __ evmovdqul(reg, Address(rsp, (int)offs), Assembler::AVX_512bit); - } else if (UseAVX >= 1) { - __ vmovdqu(reg, Address(rsp, (int)offs)); - } else { - __ movdqu(reg, Address(rsp, (int)offs)); - } + size_t offs = buffer_offset + layout.returns_vector + i * xmm_reg_size; + __ movdqu(reg, Address(rsp, (int)offs)); } for (size_t i = abi._X87_return_registers_noof; i > 0 ; i--) { diff --git a/src/hotspot/share/ci/ciEnv.cpp b/src/hotspot/share/ci/ciEnv.cpp index 948020a7d119d..1467967969333 100644 --- a/src/hotspot/share/ci/ciEnv.cpp +++ b/src/hotspot/share/ci/ciEnv.cpp @@ -970,8 +970,7 @@ void ciEnv::register_method(ciMethod* target, bool has_unsafe_access, bool has_wide_vectors, RTMState rtm_state, - address* native_stubs, - int num_stubs) { + const GrowableArrayView& native_invokers) { VM_ENTRY_MARK; nmethod* nm = NULL; { @@ -1061,7 +1060,7 @@ void ciEnv::register_method(ciMethod* target, frame_words, oop_map_set, handler_table, inc_table, compiler, task()->comp_level(), - native_stubs, num_stubs); + native_invokers); // Free codeBlobs code_buffer->free_blob(); diff --git a/src/hotspot/share/ci/ciEnv.hpp b/src/hotspot/share/ci/ciEnv.hpp index a435d1bc76028..041c23fdb2e89 100644 --- a/src/hotspot/share/ci/ciEnv.hpp +++ b/src/hotspot/share/ci/ciEnv.hpp @@ -381,8 +381,7 @@ class ciEnv : StackObj { bool has_unsafe_access, bool has_wide_vectors, RTMState rtm_state = NoRTM, - address* native_stubs = NULL, - int num_stubs = 0); + const GrowableArrayView& native_invokers = GrowableArrayView::EMPTY); // Access to certain well known ciObjects. diff --git a/src/hotspot/share/code/codeCache.cpp b/src/hotspot/share/code/codeCache.cpp index e2af7f6007ef9..a0c42c6dd8395 100644 --- a/src/hotspot/share/code/codeCache.cpp +++ b/src/hotspot/share/code/codeCache.cpp @@ -564,7 +564,7 @@ void CodeCache::free(CodeBlob* cb) { if (ptr->has_dependencies()) { _number_of_nmethods_with_dependencies--; } - ptr->free_native_stubs(); + ptr->free_native_invokers(); } if (cb->is_adapter_blob()) { heap->set_adapter_count(heap->adapter_count() - 1); diff --git a/src/hotspot/share/code/nmethod.cpp b/src/hotspot/share/code/nmethod.cpp index 96201c7890d1f..561e82d194328 100644 --- a/src/hotspot/share/code/nmethod.cpp +++ b/src/hotspot/share/code/nmethod.cpp @@ -65,6 +65,7 @@ #include "runtime/sweeper.hpp" #include "runtime/vmThread.hpp" #include "utilities/align.hpp" +#include "utilities/copy.hpp" #include "utilities/dtrace.hpp" #include "utilities/events.hpp" #include "utilities/resourceHash.hpp" @@ -497,8 +498,7 @@ nmethod* nmethod::new_nmethod(const methodHandle& method, ImplicitExceptionTable* nul_chk_table, AbstractCompiler* compiler, int comp_level, - address* native_stubs, - int num_stubs + const GrowableArrayView& native_invokers #if INCLUDE_JVMCI , char* speculations, int speculations_len, @@ -520,6 +520,7 @@ nmethod* nmethod::new_nmethod(const methodHandle& method, CodeBlob::allocation_size(code_buffer, sizeof(nmethod)) + adjust_pcs_size(debug_info->pcs_size()) + align_up((int)dependencies->size_in_bytes(), oopSize) + + align_up(native_invokers.data_size_in_bytes() , oopSize) + align_up(handler_table->size_in_bytes() , oopSize) + align_up(nul_chk_table->size_in_bytes() , oopSize) #if INCLUDE_JVMCI @@ -536,8 +537,7 @@ nmethod* nmethod::new_nmethod(const methodHandle& method, nul_chk_table, compiler, comp_level, - native_stubs, - num_stubs + native_invokers #if INCLUDE_JVMCI , speculations, speculations_len, @@ -601,8 +601,7 @@ nmethod::nmethod( : CompiledMethod(method, "native nmethod", type, nmethod_size, sizeof(nmethod), code_buffer, offsets->value(CodeOffsets::Frame_Complete), frame_size, oop_maps, false), _is_unloading_state(0), _native_receiver_sp_offset(basic_lock_owner_sp_offset), - _native_basic_lock_sp_offset(basic_lock_sp_offset), - _native_stubs(NULL), _num_stubs(0) + _native_basic_lock_sp_offset(basic_lock_sp_offset) { { int scopes_data_offset = 0; @@ -626,7 +625,8 @@ nmethod::nmethod( scopes_data_offset = _metadata_offset + align_up(code_buffer->total_metadata_size(), wordSize); _scopes_pcs_offset = scopes_data_offset; _dependencies_offset = _scopes_pcs_offset; - _handler_table_offset = _dependencies_offset; + _native_invokers_offset = _dependencies_offset; + _handler_table_offset = _native_invokers_offset; _nul_chk_table_offset = _handler_table_offset; #if INCLUDE_JVMCI _speculations_offset = _nul_chk_table_offset; @@ -723,8 +723,7 @@ nmethod::nmethod( ImplicitExceptionTable* nul_chk_table, AbstractCompiler* compiler, int comp_level, - address* native_stubs, - int num_stubs + const GrowableArrayView& native_invokers #if INCLUDE_JVMCI , char* speculations, int speculations_len, @@ -734,8 +733,7 @@ nmethod::nmethod( : CompiledMethod(method, "nmethod", type, nmethod_size, sizeof(nmethod), code_buffer, offsets->value(CodeOffsets::Frame_Complete), frame_size, oop_maps, false), _is_unloading_state(0), _native_receiver_sp_offset(in_ByteSize(-1)), - _native_basic_lock_sp_offset(in_ByteSize(-1)), - _native_stubs(native_stubs), _num_stubs(num_stubs) + _native_basic_lock_sp_offset(in_ByteSize(-1)) { assert(debug_info->oop_recorder() == code_buffer->oop_recorder(), "shared OR"); { @@ -802,7 +800,8 @@ nmethod::nmethod( _scopes_pcs_offset = scopes_data_offset + align_up(debug_info->data_size (), oopSize); _dependencies_offset = _scopes_pcs_offset + adjust_pcs_size(debug_info->pcs_size()); - _handler_table_offset = _dependencies_offset + align_up((int)dependencies->size_in_bytes (), oopSize); + _native_invokers_offset = _dependencies_offset + align_up((int)dependencies->size_in_bytes(), oopSize); + _handler_table_offset = _native_invokers_offset + align_up(native_invokers.data_size_in_bytes(), oopSize); _nul_chk_table_offset = _handler_table_offset + align_up(handler_table->size_in_bytes(), oopSize); #if INCLUDE_JVMCI _speculations_offset = _nul_chk_table_offset + align_up(nul_chk_table->size_in_bytes(), oopSize); @@ -824,6 +823,10 @@ nmethod::nmethod( code_buffer->copy_values_to(this); debug_info->copy_to(this); dependencies->copy_to(this); + if (native_invokers.is_nonempty()) { // can not get address of zero-length array + // Copy native stubs + memcpy(native_invokers_begin(), native_invokers.adr_at(0), native_invokers.data_size_in_bytes()); + } clear_unloading_state(); Universe::heap()->register_nmethod(this); @@ -986,6 +989,10 @@ void nmethod::print_nmethod(bool printmethod) { print_dependencies(); tty->print_cr("- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - "); } + if (printmethod && native_invokers_begin() < native_invokers_end()) { + print_native_invokers(); + tty->print_cr("- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - "); + } if (printmethod || PrintExceptionHandlers) { print_handler_table(); tty->print_cr("- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - "); @@ -1046,14 +1053,9 @@ void nmethod::copy_values(GrowableArray* array) { } } -void nmethod::free_native_stubs() { - if (_native_stubs != NULL) { - for (int i = 0; i < _num_stubs; i++) { - CodeBlob* cb = CodeCache::find_blob((char*) _native_stubs[i]); - assert(cb != NULL, "Expected to find blob"); - CodeCache::free(cb); - } - FREE_C_HEAP_ARRAY(address, _native_stubs); +void nmethod::free_native_invokers() { + for (BufferBlob** it = native_invokers_begin(); it < native_invokers_end(); it++) { + CodeCache::free(*it); } } @@ -2689,6 +2691,14 @@ void nmethod::print_pcs_on(outputStream* st) { } } +void nmethod::print_native_invokers() { + ResourceMark m; // in case methods get printed via debugger + tty->print_cr("Native invokers:"); + for (BufferBlob** itt = native_invokers_begin(); itt < native_invokers_end(); itt++) { + (*itt)->print_on(tty); + } +} + void nmethod::print_handler_table() { ExceptionHandlerTable(this).print(); } diff --git a/src/hotspot/share/code/nmethod.hpp b/src/hotspot/share/code/nmethod.hpp index 62be8b8e94cd1..592fa8f7017fa 100644 --- a/src/hotspot/share/code/nmethod.hpp +++ b/src/hotspot/share/code/nmethod.hpp @@ -207,6 +207,7 @@ class nmethod : public CompiledMethod { int _scopes_data_offset; int _scopes_pcs_offset; int _dependencies_offset; + int _native_invokers_offset; int _handler_table_offset; int _nul_chk_table_offset; #if INCLUDE_JVMCI @@ -282,9 +283,6 @@ class nmethod : public CompiledMethod { ByteSize _native_receiver_sp_offset; ByteSize _native_basic_lock_sp_offset; - address* _native_stubs; - int _num_stubs; - friend class nmethodLocker; // For native wrappers @@ -316,8 +314,7 @@ class nmethod : public CompiledMethod { ImplicitExceptionTable* nul_chk_table, AbstractCompiler* compiler, int comp_level, - address* native_stubs, - int num_stubs + const GrowableArrayView& native_invokers #if INCLUDE_JVMCI , char* speculations, int speculations_len, @@ -366,8 +363,7 @@ class nmethod : public CompiledMethod { ImplicitExceptionTable* nul_chk_table, AbstractCompiler* compiler, int comp_level, - address* native_stubs = NULL, - int num_stubs = 0 + const GrowableArrayView& native_invokers = GrowableArrayView::EMPTY #if INCLUDE_JVMCI , char* speculations = NULL, int speculations_len = 0, @@ -416,7 +412,9 @@ class nmethod : public CompiledMethod { PcDesc* scopes_pcs_begin () const { return (PcDesc*)(header_begin() + _scopes_pcs_offset ); } PcDesc* scopes_pcs_end () const { return (PcDesc*)(header_begin() + _dependencies_offset) ; } address dependencies_begin () const { return header_begin() + _dependencies_offset ; } - address dependencies_end () const { return header_begin() + _handler_table_offset ; } + address dependencies_end () const { return header_begin() + _native_invokers_offset ; } + BufferBlob** native_invokers_begin() const { return (BufferBlob**)(header_begin() + _native_invokers_offset) ; } + BufferBlob** native_invokers_end () const { return (BufferBlob**)(header_begin() + _handler_table_offset); } address handler_table_begin () const { return header_begin() + _handler_table_offset ; } address handler_table_end () const { return header_begin() + _nul_chk_table_offset ; } address nul_chk_table_begin () const { return header_begin() + _nul_chk_table_offset ; } @@ -532,7 +530,7 @@ class nmethod : public CompiledMethod { void copy_values(GrowableArray* oops); void copy_values(GrowableArray* metadata); - void free_native_stubs(); + void free_native_invokers(); // Relocation support private: @@ -670,6 +668,7 @@ class nmethod : public CompiledMethod { void print_scopes() { print_scopes_on(tty); } void print_scopes_on(outputStream* st) PRODUCT_RETURN; void print_value_on(outputStream* st) const; + void print_native_invokers(); void print_handler_table(); void print_nul_chk_table(); void print_recorded_oops(); diff --git a/src/hotspot/share/jvmci/jvmciRuntime.cpp b/src/hotspot/share/jvmci/jvmciRuntime.cpp index 8e597e6a1eaab..03518c2e9cad0 100644 --- a/src/hotspot/share/jvmci/jvmciRuntime.cpp +++ b/src/hotspot/share/jvmci/jvmciRuntime.cpp @@ -1627,7 +1627,7 @@ JVMCI::CodeInstallResult JVMCIRuntime::register_method(JVMCIEnv* JVMCIENV, debug_info, dependencies, code_buffer, frame_words, oop_map_set, handler_table, implicit_exception_table, - compiler, comp_level, NULL, 0, + compiler, comp_level, GrowableArrayView::EMPTY, speculations, speculations_len, nmethod_mirror_index, nmethod_mirror_name, failed_speculations); diff --git a/src/hotspot/share/opto/callnode.cpp b/src/hotspot/share/opto/callnode.cpp index cf53ae81c754c..4fdc5e2958efc 100644 --- a/src/hotspot/share/opto/callnode.cpp +++ b/src/hotspot/share/opto/callnode.cpp @@ -1138,15 +1138,9 @@ Node* CallNativeNode::match(const ProjNode *proj, const Matcher *matcher) { case TypeFunc::ReturnAdr: case TypeFunc::FramePtr: ShouldNotReachHere(); - case TypeFunc::Parms: - default: { - if(tf()->range()->field_at(proj->_con) == Type::HALF) { - assert(_ret_regs.at(proj->_con - TypeFunc::Parms) == VMRegImpl::Bad(), "Unexpected register for Type::HALF"); - // 2nd half of doubles and longs - return new MachProjNode(this,proj->_con, RegMask::Empty, (uint)OptoReg::Bad); - } - - const BasicType bt = tf()->range()->field_at(proj->_con)->basic_type(); + case TypeFunc::Parms: { + const Type* field_at_con = tf()->range()->field_at(proj->_con); + const BasicType bt = field_at_con->basic_type(); OptoReg::Name optoreg = OptoReg::as_OptoReg(_ret_regs.at(proj->_con - TypeFunc::Parms)); OptoRegPair regs; if (bt == T_DOUBLE || bt == T_LONG) { @@ -1155,21 +1149,40 @@ Node* CallNativeNode::match(const ProjNode *proj, const Matcher *matcher) { regs.set1(optoreg); } RegMask rm = RegMask(regs.first()); - if( OptoReg::is_valid(regs.second()) ) - rm.Insert( regs.second() ); - return new MachProjNode(this,proj->_con,rm,tf()->range()->field_at(proj->_con)->ideal_reg()); + if(OptoReg::is_valid(regs.second())) + rm.Insert(regs.second()); + return new MachProjNode(this, proj->_con, rm, field_at_con->ideal_reg()); } + case TypeFunc::Parms + 1: { + assert(tf()->range()->field_at(proj->_con) == Type::HALF, "Expected HALF"); + assert(_ret_regs.at(proj->_con - TypeFunc::Parms) == VMRegImpl::Bad(), "Unexpected register for Type::HALF"); + // 2nd half of doubles and longs + return new MachProjNode(this, proj->_con, RegMask::Empty, (uint) OptoReg::Bad); + } + default: + ShouldNotReachHere(); } return NULL; } #ifndef PRODUCT +void CallNativeNode::print_regs(const GrowableArray& regs, outputStream* st) { + st->print("{ "); + for (int i = 0; i < regs.length(); i++) { + regs.at(i)->print_on(st); + if (i < regs.length() - 1) { + st->print(", "); + } + } + st->print(" } "); +} + void CallNativeNode::dump_spec(outputStream *st) const { st->print("# "); st->print("%s ", _name); st->print("_arg_regs: "); - _arg_regs.print_on(st); + print_regs(_arg_regs, st); st->print("_ret_regs: "); - _ret_regs.print_on(st); + print_regs(_ret_regs, st); CallNode::dump_spec(st); } #endif @@ -1181,7 +1194,7 @@ void CallRuntimeNode::calling_convention( BasicType* sig_bt, VMRegPair *parm_reg void CallNativeNode::calling_convention( BasicType* sig_bt, VMRegPair *parm_regs, uint argcnt ) const { assert((tf()->domain()->cnt() - TypeFunc::Parms) == argcnt, "arg counts must match!"); -#ifndef PRODUCT +#ifdef ASSERT for (uint i = 0; i < argcnt; i++) { assert(tf()->domain()->field_at(TypeFunc::Parms + i)->basic_type() == sig_bt[i], "types must match!"); } diff --git a/src/hotspot/share/opto/callnode.hpp b/src/hotspot/share/opto/callnode.hpp index 5f3f44a2a7915..0ca117e21c959 100644 --- a/src/hotspot/share/opto/callnode.hpp +++ b/src/hotspot/share/opto/callnode.hpp @@ -812,8 +812,10 @@ class CallLeafNode : public CallRuntimeNode { // Make a direct call into a foreign function with an arbitrary ABI // safepoints class CallNativeNode : public CallNode { + friend class MachCallNativeNode; virtual bool cmp( const Node &n ) const; virtual uint size_of() const; + static void print_regs(const GrowableArray& regs, outputStream* st); public: GrowableArray _arg_regs; GrowableArray _ret_regs; diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index 9082d68c58244..251ac7a4b34fa 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -536,7 +536,7 @@ Compile::Compile( ciEnv* ci_env, ciMethod* target, int osr_bci, _vector_reboxing_late_inlines(comp_arena(), 2, 0, NULL), _late_inlines_pos(0), _number_of_mh_late_inlines(0), - _native_stubs(comp_arena(), 1, 0, NULL), + _native_invokers(comp_arena(), 1, 0, NULL), _print_inlining_stream(NULL), _print_inlining_list(NULL), _print_inlining_idx(0), @@ -833,7 +833,7 @@ Compile::Compile( ciEnv* ci_env, _for_igvn(NULL), _warm_calls(NULL), _number_of_mh_late_inlines(0), - _native_stubs(), + _native_invokers(), _print_inlining_stream(NULL), _print_inlining_list(NULL), _print_inlining_idx(0), @@ -4699,6 +4699,6 @@ void Compile::igv_print_method_to_network(const char* phase_name) { } #endif -void Compile::add_native_stub(address stubAddress) { - _native_stubs.append(stubAddress); +void Compile::add_native_invoker(BufferBlob* stub) { + _native_invokers.append(stub); } diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp index 603ada28b49b0..583ebdbfafefd 100644 --- a/src/hotspot/share/opto/compile.hpp +++ b/src/hotspot/share/opto/compile.hpp @@ -384,7 +384,7 @@ class Compile : public Phase { int _late_inlines_pos; // Where in the queue should the next late inlining candidate go (emulate depth first inlining) uint _number_of_mh_late_inlines; // number of method handle late inlining still pending - GrowableArray
_native_stubs; + GrowableArray _native_invokers; // Inlining may not happen in parse order which would make // PrintInlining output confusing. Keep track of PrintInlining @@ -936,9 +936,9 @@ class Compile : public Phase { _vector_reboxing_late_inlines.push(cg); } - void add_native_stub(address stub); + void add_native_invoker(BufferBlob* stub); - const GrowableArray
& native_stubs() const { return _native_stubs; } + const GrowableArray& native_invokers() const { return _native_invokers; } void remove_useless_late_inlines(GrowableArray* inlines, Unique_Node_List &useful); void remove_useless_nodes (GrowableArray& node_list, Unique_Node_List &useful); diff --git a/src/hotspot/share/opto/graphKit.cpp b/src/hotspot/share/opto/graphKit.cpp index 7a3ba50896e7e..a6c0aee48eebe 100644 --- a/src/hotspot/share/opto/graphKit.cpp +++ b/src/hotspot/share/opto/graphKit.cpp @@ -2581,7 +2581,7 @@ Node* GraphKit::make_native_call(const TypeFunc* call_type, uint nargs, ciNative uint n_filtered_args = nargs - 2; // -fallback, -nep; ResourceMark rm; Node** argument_nodes = NEW_RESOURCE_ARRAY(Node*, n_filtered_args); - const Type** arg_types = NEW_RESOURCE_ARRAY(const Type*, n_filtered_args); + const Type** arg_types = TypeTuple::fields(n_filtered_args); GrowableArray arg_regs(C->comp_arena(), n_filtered_args, n_filtered_args, VMRegImpl::Bad()); VMReg* argRegs = nep->argMoves(); @@ -2596,14 +2596,14 @@ Node* GraphKit::make_native_call(const TypeFunc* call_type, uint nargs, ciNative : argRegs[java_arg_read_pos++]; argument_nodes[vm_arg_pos] = node; - arg_types[vm_arg_pos] = type; + arg_types[TypeFunc::Parms + vm_arg_pos] = type; arg_regs.at_put(vm_arg_pos, reg); } } uint n_returns = call_type->range()->cnt() - TypeFunc::Parms; GrowableArray ret_regs(C->comp_arena(), n_returns, n_returns, VMRegImpl::Bad()); - const Type** ret_types = NEW_RESOURCE_ARRAY(const Type*, n_returns); + const Type** ret_types = TypeTuple::fields(n_returns); VMReg* retRegs = nep->returnMoves(); { @@ -2615,25 +2615,26 @@ Node* GraphKit::make_native_call(const TypeFunc* call_type, uint nargs, ciNative : retRegs[java_ret_read_pos++]; ret_regs.at_put(vm_ret_pos, reg); - ret_types[vm_ret_pos] = type; + ret_types[TypeFunc::Parms + vm_ret_pos] = type; } } const TypeFunc* new_call_type = TypeFunc::make( - TypeTuple::make_func(n_filtered_args, arg_types), - TypeTuple::make_func(n_returns, ret_types) + TypeTuple::make(TypeFunc::Parms + n_filtered_args, arg_types), + TypeTuple::make(TypeFunc::Parms + n_returns, ret_types) ); address call_addr = nep->entry_point(); if (nep->need_transition()) { - call_addr = SharedRuntime::make_native_invoker(call_addr, - nep->shadow_space(), - arg_regs, ret_regs); + BufferBlob* invoker = SharedRuntime::make_native_invoker(call_addr, + nep->shadow_space(), + arg_regs, ret_regs); if (call_addr == NULL) { C->record_failure("native invoker not implemented on this platform"); return NULL; } - C->add_native_stub(call_addr); + C->add_native_invoker(invoker); + call_addr = invoker->code_begin(); } assert(call_addr != NULL, "sanity"); @@ -2662,33 +2663,22 @@ Node* GraphKit::make_native_call(const TypeFunc* call_type, uint nargs, ciNative if (method() == NULL || method()->return_type()->basic_type() == T_VOID) { ret = top(); } else { - Node* current_value = NULL; - for (uint vm_ret_pos = 0; vm_ret_pos < n_returns; vm_ret_pos++) { - if (new_call_type->range()->field_at(TypeFunc::Parms + vm_ret_pos) == Type::HALF) { - // FIXME is this needed? - gvn().transform(new ProjNode(call, TypeFunc::Parms + vm_ret_pos)); - } else { - assert(current_value == NULL, "Must not overwrite"); - current_value = gvn().transform(new ProjNode(call, TypeFunc::Parms + vm_ret_pos)); - } - } - assert(current_value != NULL, "Should not be null"); + ret = gvn().transform(new ProjNode(call, TypeFunc::Parms)); // Unpack native results if needed // Need this method type since it's unerased switch (nep->method_type()->rtype()->basic_type()) { case T_CHAR: - current_value = _gvn.transform(new AndINode(current_value, _gvn.intcon(0xFFFF))); + ret = _gvn.transform(new AndINode(ret, _gvn.intcon(0xFFFF))); break; case T_BYTE: - current_value = sign_extend_byte(current_value); + ret = sign_extend_byte(ret); break; case T_SHORT: - current_value = sign_extend_short(current_value); + ret = sign_extend_short(ret); break; default: // do nothing break; } - ret = current_value; } push_node(method()->return_type()->basic_type(), ret); diff --git a/src/hotspot/share/opto/lcm.cpp b/src/hotspot/share/opto/lcm.cpp index 8b8a3a5fa861e..2b554df8ac427 100644 --- a/src/hotspot/share/opto/lcm.cpp +++ b/src/hotspot/share/opto/lcm.cpp @@ -863,7 +863,9 @@ uint PhaseCFG::sched_call(Block* block, uint node_cnt, Node_List& worklist, Grow save_policy = _matcher._register_save_policy; break; case Op_CallNative: - // FIXME compute actual save policy based on nep->abi + // We use the c reg save policy here since Panama + // only supports the C ABI currently. + // TODO compute actual save policy based on nep->abi save_policy = _matcher._c_reg_save_policy; break; diff --git a/src/hotspot/share/opto/machnode.cpp b/src/hotspot/share/opto/machnode.cpp index 5f7645138e996..09e4cfde42923 100644 --- a/src/hotspot/share/opto/machnode.cpp +++ b/src/hotspot/share/opto/machnode.cpp @@ -828,9 +828,9 @@ bool MachCallNativeNode::cmp( const Node &n ) const { void MachCallNativeNode::dump_spec(outputStream *st) const { st->print("%s ",_name); st->print("_arg_regs: "); - _arg_regs.print_on(st); + CallNativeNode::print_regs(_arg_regs, st); st->print("_ret_regs: "); - _ret_regs.print_on(st); + CallNativeNode::print_regs(_ret_regs, st); MachCallNode::dump_spec(st); } #endif diff --git a/src/hotspot/share/opto/machnode.hpp b/src/hotspot/share/opto/machnode.hpp index 1e317f55b34f6..72d1284a178d4 100644 --- a/src/hotspot/share/opto/machnode.hpp +++ b/src/hotspot/share/opto/machnode.hpp @@ -1013,6 +1013,7 @@ class MachCallLeafNode: public MachCallRuntimeNode { class MachCallNativeNode: public MachCallNode { virtual bool cmp( const Node &n ) const; virtual uint size_of() const; + void print_regs(const GrowableArray& regs, outputStream* st) const; public: const char *_name; GrowableArray _arg_regs; diff --git a/src/hotspot/share/opto/output.cpp b/src/hotspot/share/opto/output.cpp index 3d11173a982ca..681cd20504497 100644 --- a/src/hotspot/share/opto/output.cpp +++ b/src/hotspot/share/opto/output.cpp @@ -1003,6 +1003,7 @@ void PhaseOutput::Process_OopMap_Node(MachNode *mach, int current_offset) { int safepoint_pc_offset = current_offset; bool is_method_handle_invoke = false; + bool is_opt_native = false; bool return_oop = false; bool has_ea_local_in_scope = sfn->_has_ea_local_in_scope; bool arg_escape = false; @@ -1021,6 +1022,8 @@ void PhaseOutput::Process_OopMap_Node(MachNode *mach, int current_offset) { is_method_handle_invoke = true; } arg_escape = mcall->as_MachCallJava()->_arg_escape; + } else if (mcall->is_MachCallNative()) { + is_opt_native = true; } // Check if a call returns an object. @@ -1141,7 +1144,6 @@ void PhaseOutput::Process_OopMap_Node(MachNode *mach, int current_offset) { // Now we can describe the scope. methodHandle null_mh; bool rethrow_exception = false; - bool is_opt_native = mach->is_MachCallNative(); C->debug_info()->describe_scope( safepoint_pc_offset, null_mh, @@ -3391,16 +3393,6 @@ void PhaseOutput::install_code(ciMethod* target, _code_offsets.set_value(CodeOffsets::OSR_Entry, 0); } - address* native_stubs = NULL; - int num_stubs = 0; - if (!C->native_stubs().is_empty()) { - num_stubs = C->native_stubs().length(); - native_stubs = NEW_C_HEAP_ARRAY(address, num_stubs, mtInternal); - for (int i = 0; i < num_stubs; i++) { - native_stubs[i] = C->native_stubs().at(i); - } - } - C->env()->register_method(target, entry_bci, &_code_offsets, @@ -3414,8 +3406,7 @@ void PhaseOutput::install_code(ciMethod* target, has_unsafe_access, SharedRuntime::is_wide_vector(C->max_vector_size()), C->rtm_state(), - native_stubs, - num_stubs); + C->native_invokers()); if (C->log() != NULL) { // Print code cache state into compiler log C->log()->code_cache_state(); diff --git a/src/hotspot/share/opto/type.cpp b/src/hotspot/share/opto/type.cpp index a3e1c952e2e7a..ff26780f7866c 100644 --- a/src/hotspot/share/opto/type.cpp +++ b/src/hotspot/share/opto/type.cpp @@ -1993,14 +1993,6 @@ const TypeTuple *TypeTuple::make( uint cnt, const Type **fields ) { return (TypeTuple*)(new TypeTuple(cnt,fields))->hashcons(); } -const TypeTuple *TypeTuple::make_func( uint arg_cnt, const Type **arg_fields ) { - const Type** field_array = fields(arg_cnt); - for (uint i = 0; i < arg_cnt; i++) { - field_array[i + TypeFunc::Parms] = arg_fields[i]; - } - return make(arg_cnt + TypeFunc::Parms, field_array); -} - //------------------------------fields----------------------------------------- // Subroutine call type with space allocated for argument types // Memory for Control, I_O, Memory, FramePtr, and ReturnAdr is allocated implicitly diff --git a/src/hotspot/share/opto/type.hpp b/src/hotspot/share/opto/type.hpp index 9d78e9918d52b..51a48c04e4b78 100644 --- a/src/hotspot/share/opto/type.hpp +++ b/src/hotspot/share/opto/type.hpp @@ -675,7 +675,6 @@ class TypeTuple : public Type { static const TypeTuple *make( uint cnt, const Type **fields ); static const TypeTuple *make_range(ciSignature *sig); static const TypeTuple *make_domain(ciInstanceKlass* recv, ciSignature *sig); - static const TypeTuple *make_func(uint arg_cnt, const Type **arg_fields); // Subroutine call type with space allocated for argument types // Memory for Control, I_O, Memory, FramePtr, and ReturnAdr is allocated implicitly diff --git a/src/hotspot/share/prims/universalUpcallHandler.cpp b/src/hotspot/share/prims/universalUpcallHandler.cpp index 85a30a435b0de..33ba631e61603 100644 --- a/src/hotspot/share/prims/universalUpcallHandler.cpp +++ b/src/hotspot/share/prims/universalUpcallHandler.cpp @@ -22,12 +22,12 @@ */ #include "precompiled.hpp" +#include "classfile/symbolTable.hpp" #include "memory/resourceArea.hpp" #include "prims/universalUpcallHandler.hpp" -#include "runtime/jniHandles.inline.hpp" #include "runtime/interfaceSupport.inline.hpp" #include "runtime/javaCalls.hpp" -#include "classfile/symbolTable.hpp" +#include "runtime/jniHandles.inline.hpp" #define FOREIGN_ABI "jdk/internal/foreign/abi/" @@ -74,8 +74,9 @@ const ProgrammableUpcallHandler& ProgrammableUpcallHandler::instance() { } ProgrammableUpcallHandler::ProgrammableUpcallHandler() { - Symbol* sym = SymbolTable::new_symbol(FOREIGN_ABI "ProgrammableUpcallHandler"); Thread* THREAD = Thread::current(); + ResourceMark rm(THREAD); + Symbol* sym = SymbolTable::new_symbol(FOREIGN_ABI "ProgrammableUpcallHandler"); Klass* k = SystemDictionary::resolve_or_null(sym, Handle(), Handle(), CATCH); k->initialize(CATCH); diff --git a/src/hotspot/share/runtime/sharedRuntime.hpp b/src/hotspot/share/runtime/sharedRuntime.hpp index e62ff4b4f6d53..8ef36761a9f8f 100644 --- a/src/hotspot/share/runtime/sharedRuntime.hpp +++ b/src/hotspot/share/runtime/sharedRuntime.hpp @@ -513,10 +513,10 @@ class SharedRuntime: AllStatic { static address handle_unsafe_access(JavaThread* thread, address next_pc); - static address make_native_invoker(address call_target, - int shadow_space_bytes, - const GrowableArray& input_registers, - const GrowableArray& output_registers); + static BufferBlob* make_native_invoker(address call_target, + int shadow_space_bytes, + const GrowableArray& input_registers, + const GrowableArray& output_registers); #ifndef PRODUCT diff --git a/src/hotspot/share/utilities/growableArray.hpp b/src/hotspot/share/utilities/growableArray.hpp index 2a86a30d4319c..edee4c8a431a3 100644 --- a/src/hotspot/share/utilities/growableArray.hpp +++ b/src/hotspot/share/utilities/growableArray.hpp @@ -124,6 +124,8 @@ class GrowableArrayView : public GrowableArrayBase { ~GrowableArrayView() {} public: + const static GrowableArrayView EMPTY; + bool operator==(const GrowableArrayView& rhs) const { if (_len != rhs._len) return false; @@ -310,7 +312,11 @@ class GrowableArrayView : public GrowableArrayBase { return min; } - void print() { + size_t data_size_in_bytes() const { + return _len * sizeof(E); + } + + void print() const { tty->print("Growable Array " INTPTR_FORMAT, p2i(this)); tty->print(": length %d (_max %d) { ", _len, _max); for (int i = 0; i < _len; i++) { @@ -320,6 +326,9 @@ class GrowableArrayView : public GrowableArrayBase { } }; +template +const GrowableArrayView GrowableArrayView::EMPTY(nullptr, 0, 0); + // GrowableArrayWithAllocator extends the "view" with // the capability to grow and deallocate the data array. // diff --git a/src/jdk.incubator.foreign/share/classes/jdk/internal/foreign/abi/x64/X86_64Architecture.java b/src/jdk.incubator.foreign/share/classes/jdk/internal/foreign/abi/x64/X86_64Architecture.java index 98c777add1b88..5793eb7729f8d 100644 --- a/src/jdk.incubator.foreign/share/classes/jdk/internal/foreign/abi/x64/X86_64Architecture.java +++ b/src/jdk.incubator.foreign/share/classes/jdk/internal/foreign/abi/x64/X86_64Architecture.java @@ -33,7 +33,7 @@ public class X86_64Architecture implements Architecture { public static final Architecture INSTANCE = new X86_64Architecture(); private static final int INTEGER_REG_SIZE = 8; // bytes - private static final int VECTOR_REG_SIZE = 64; // sizeof(VectorRegister) + private static final int VECTOR_REG_SIZE = 16; // size of XMM register private static final int X87_REG_SIZE = 16; private static final int STACK_SLOT_SIZE = 8;