/* * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as * published by the Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA * or visit www.oracle.com if you need additional information or have any * questions. */ #include "asm/macroAssembler.inline.hpp" #include "code/codeBlob.hpp" #include "code/vmreg.inline.hpp" #include "gc/z/zAddress.hpp" #include "gc/z/zBarrier.inline.hpp" #include "gc/z/zBarrierSet.hpp" #include "gc/z/zBarrierSetAssembler.hpp" #include "gc/z/zBarrierSetRuntime.hpp" #include "gc/z/zThreadLocalData.hpp" #include "memory/resourceArea.hpp" #include "nativeInst_aarch64.hpp" #include "runtime/icache.hpp" #include "runtime/jniHandles.hpp" #include "runtime/sharedRuntime.hpp" #include "utilities/macros.hpp" #ifdef COMPILER1 #include "c1/c1_LIRAssembler.hpp" #include "c1/c1_MacroAssembler.hpp" #include "gc/z/c1/zBarrierSetC1.hpp" #endif // COMPILER1 #ifdef COMPILER2 #include "gc/z/c2/zBarrierSetC2.hpp" #include "opto/output.hpp" #endif // COMPILER2 #ifdef PRODUCT #define BLOCK_COMMENT(str) /* nothing */ #else #define BLOCK_COMMENT(str) __ block_comment(str) #endif #undef __ #define __ masm-> // Helper for saving and restoring registers across a runtime call that does // not have any live vector registers. class ZRuntimeCallSpill { private: MacroAssembler* _masm; Register _result; void save() { MacroAssembler* masm = _masm; __ enter(true /* strip_ret_addr */); if (_result != noreg) { __ push_call_clobbered_registers_except(RegSet::of(_result)); } else { __ push_call_clobbered_registers(); } } void restore() { MacroAssembler* masm = _masm; if (_result != noreg) { // Make sure _result has the return value. if (_result != r0) { __ mov(_result, r0); } __ pop_call_clobbered_registers_except(RegSet::of(_result)); } else { __ pop_call_clobbered_registers(); } __ leave(); } public: ZRuntimeCallSpill(MacroAssembler* masm, Register result) : _masm(masm), _result(result) { save(); } ~ZRuntimeCallSpill() { restore(); } }; void ZBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp2) { if (!ZBarrierSet::barrier_needed(decorators, type)) { // Barrier not needed BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2); return; } assert_different_registers(tmp1, tmp2, src.base(), noreg); assert_different_registers(tmp1, tmp2, src.index()); assert_different_registers(tmp1, tmp2, dst, noreg); assert_different_registers(tmp2, rscratch1); Label done; Label uncolor; // Load bad mask into scratch register. const bool on_non_strong = (decorators & ON_WEAK_OOP_REF) != 0 || (decorators & ON_PHANTOM_OOP_REF) != 0; if (on_non_strong) { __ ldr(tmp1, mark_bad_mask_from_thread(rthread)); } else { __ ldr(tmp1, load_bad_mask_from_thread(rthread)); } __ lea(tmp2, src); __ ldr(dst, tmp2); // Test reference against bad mask. If mask bad, then we need to fix it up. __ tst(dst, tmp1); __ br(Assembler::EQ, uncolor); { // Call VM ZRuntimeCallSpill rcs(masm, dst); if (c_rarg0 != dst) { __ mov(c_rarg0, dst); } __ mov(c_rarg1, tmp2); __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2); } // Slow-path has already uncolored __ b(done); __ bind(uncolor); // Remove the color bits __ lsr(dst, dst, ZPointerLoadShift); __ bind(done); } void ZBarrierSetAssembler::store_barrier_fast(MacroAssembler* masm, Address ref_addr, Register rnew_zaddress, Register rnew_zpointer, Register rtmp, bool in_nmethod, bool is_atomic, Label& medium_path, Label& medium_path_continuation) const { assert_different_registers(ref_addr.base(), rnew_zpointer, rtmp); assert_different_registers(ref_addr.index(), rnew_zpointer, rtmp); assert_different_registers(rnew_zaddress, rnew_zpointer, rtmp); if (in_nmethod) { if (is_atomic) { __ ldrh(rtmp, ref_addr); // Atomic operations must ensure that the contents of memory are store-good before // an atomic operation can execute. // A not relocatable object could have spurious raw null pointers in its fields after // getting promoted to the old generation. __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodBeforeMov); __ movzw(rnew_zpointer, barrier_Relocation::unpatched); __ cmpw(rtmp, rnew_zpointer); } else { __ ldr(rtmp, ref_addr); // Stores on relocatable objects never need to deal with raw null pointers in fields. // Raw null pointers may only exist in the young generation, as they get pruned when // the object is relocated to old. And no pre-write barrier needs to perform any action // in the young generation. __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreBadBeforeMov); __ movzw(rnew_zpointer, barrier_Relocation::unpatched); __ tst(rtmp, rnew_zpointer); } __ br(Assembler::NE, medium_path); __ bind(medium_path_continuation); assert_different_registers(rnew_zaddress, rnew_zpointer); __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodBeforeMov); __ movzw(rnew_zpointer, barrier_Relocation::unpatched); __ orr(rnew_zpointer, rnew_zpointer, rnew_zaddress, Assembler::LSL, ZPointerLoadShift); } else { assert(!is_atomic, "atomics outside of nmethods not supported"); __ lea(rtmp, ref_addr); __ ldr(rtmp, rtmp); __ ldr(rnew_zpointer, Address(rthread, ZThreadLocalData::store_bad_mask_offset())); __ tst(rtmp, rnew_zpointer); __ br(Assembler::NE, medium_path); __ bind(medium_path_continuation); if (rnew_zaddress == noreg) { __ eor(rnew_zpointer, rnew_zpointer, rnew_zpointer); } else { __ mov(rnew_zpointer, rnew_zaddress); } // Load the current good shift, and add the color bits __ lsl(rnew_zpointer, rnew_zpointer, ZPointerLoadShift); __ ldr(rtmp, Address(rthread, ZThreadLocalData::store_good_mask_offset())); __ orr(rnew_zpointer, rnew_zpointer, rtmp); } } static void store_barrier_buffer_add(MacroAssembler* masm, Address ref_addr, Register tmp1, Register tmp2, Label& slow_path) { Address buffer(rthread, ZThreadLocalData::store_barrier_buffer_offset()); assert_different_registers(ref_addr.base(), ref_addr.index(), tmp1, tmp2); __ ldr(tmp1, buffer); // Combined pointer bump and check if the buffer is disabled or full __ ldr(tmp2, Address(tmp1, ZStoreBarrierBuffer::current_offset())); __ cmp(tmp2, (uint8_t)0); __ br(Assembler::EQ, slow_path); // Bump the pointer __ sub(tmp2, tmp2, sizeof(ZStoreBarrierEntry)); __ str(tmp2, Address(tmp1, ZStoreBarrierBuffer::current_offset())); // Compute the buffer entry address __ lea(tmp2, Address(tmp2, ZStoreBarrierBuffer::buffer_offset())); __ add(tmp2, tmp2, tmp1); // Compute and log the store address __ lea(tmp1, ref_addr); __ str(tmp1, Address(tmp2, in_bytes(ZStoreBarrierEntry::p_offset()))); // Load and log the prev value __ ldr(tmp1, tmp1); __ str(tmp1, Address(tmp2, in_bytes(ZStoreBarrierEntry::prev_offset()))); } void ZBarrierSetAssembler::store_barrier_medium(MacroAssembler* masm, Address ref_addr, Register rtmp1, Register rtmp2, Register rtmp3, bool is_native, bool is_atomic, Label& medium_path_continuation, Label& slow_path, Label& slow_path_continuation) const { assert_different_registers(ref_addr.base(), ref_addr.index(), rtmp1, rtmp2); // The reason to end up in the medium path is that the pre-value was not 'good'. if (is_native) { __ b(slow_path); __ bind(slow_path_continuation); __ b(medium_path_continuation); } else if (is_atomic) { // Atomic accesses can get to the medium fast path because the value was a // raw null value. If it was not null, then there is no doubt we need to take a slow path. __ lea(rtmp2, ref_addr); __ ldr(rtmp1, rtmp2); __ cbnz(rtmp1, slow_path); // If we get this far, we know there is a young raw null value in the field. __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodBeforeMov); __ movzw(rtmp1, barrier_Relocation::unpatched); __ cmpxchg(rtmp2, zr, rtmp1, Assembler::xword, false /* acquire */, false /* release */, true /* weak */, rtmp3); __ br(Assembler::NE, slow_path); __ bind(slow_path_continuation); __ b(medium_path_continuation); } else { // A non-atomic relocatable object won't get to the medium fast path due to a // raw null in the young generation. We only get here because the field is bad. // In this path we don't need any self healing, so we can avoid a runtime call // most of the time by buffering the store barrier to be applied lazily. store_barrier_buffer_add(masm, ref_addr, rtmp1, rtmp2, slow_path); __ bind(slow_path_continuation); __ b(medium_path_continuation); } } void ZBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) { if (!ZBarrierSet::barrier_needed(decorators, type)) { BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3); return; } bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; assert_different_registers(val, tmp1, dst.base(), dst.index()); if (dest_uninitialized) { if (val == noreg) { __ eor(tmp1, tmp1, tmp1); } else { __ mov(tmp1, val); } // Add the color bits __ lsl(tmp1, tmp1, ZPointerLoadShift); __ ldr(tmp2, Address(rthread, ZThreadLocalData::store_good_mask_offset())); __ orr(tmp1, tmp2, tmp1); } else { Label done; Label medium; Label medium_continuation; Label slow; Label slow_continuation; store_barrier_fast(masm, dst, val, tmp1, tmp2, false, false, medium, medium_continuation); __ b(done); __ bind(medium); store_barrier_medium(masm, dst, tmp1, tmp2, noreg /* tmp3 */, false /* is_native */, false /* is_atomic */, medium_continuation, slow, slow_continuation); __ bind(slow); { // Call VM ZRuntimeCallSpill rcs(masm, noreg); __ lea(c_rarg0, dst); __ MacroAssembler::call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr(), 1); } __ b(slow_continuation); __ bind(done); } // Store value BarrierSetAssembler::store_at(masm, decorators, type, dst, tmp1, tmp2, tmp3, noreg); } static FloatRegister z_copy_load_bad_vreg = v17; static FloatRegister z_copy_store_good_vreg = v18; static FloatRegister z_copy_store_bad_vreg = v19; static void load_wide_arraycopy_masks(MacroAssembler* masm) { __ lea(rscratch1, ExternalAddress((address)&ZPointerVectorLoadBadMask)); __ ldrq(z_copy_load_bad_vreg, Address(rscratch1, 0)); __ lea(rscratch1, ExternalAddress((address)&ZPointerVectorStoreBadMask)); __ ldrq(z_copy_store_bad_vreg, Address(rscratch1, 0)); __ lea(rscratch1, ExternalAddress((address)&ZPointerVectorStoreGoodMask)); __ ldrq(z_copy_store_good_vreg, Address(rscratch1, 0)); } class ZCopyRuntimeCallSpill { private: MacroAssembler* _masm; Register _result; void save() { MacroAssembler* masm = _masm; __ enter(true /* strip_ret_addr */); if (_result != noreg) { __ push(__ call_clobbered_gp_registers() - RegSet::of(_result), sp); } else { __ push(__ call_clobbered_gp_registers(), sp); } int neonSize = wordSize * 2; __ sub(sp, sp, 4 * neonSize); __ st1(v0, v1, v2, v3, Assembler::T16B, Address(sp, 0)); __ sub(sp, sp, 4 * neonSize); __ st1(v4, v5, v6, v7, Assembler::T16B, Address(sp, 0)); __ sub(sp, sp, 4 * neonSize); __ st1(v16, v17, v18, v19, Assembler::T16B, Address(sp, 0)); } void restore() { MacroAssembler* masm = _masm; int neonSize = wordSize * 2; __ ld1(v16, v17, v18, v19, Assembler::T16B, Address(sp, 0)); __ add(sp, sp, 4 * neonSize); __ ld1(v4, v5, v6, v7, Assembler::T16B, Address(sp, 0)); __ add(sp, sp, 4 * neonSize); __ ld1(v0, v1, v2, v3, Assembler::T16B, Address(sp, 0)); __ add(sp, sp, 4 * neonSize); if (_result != noreg) { if (_result != r0) { __ mov(_result, r0); } __ pop(__ call_clobbered_gp_registers() - RegSet::of(_result), sp); } else { __ pop(__ call_clobbered_gp_registers(), sp); } __ leave(); } public: ZCopyRuntimeCallSpill(MacroAssembler* masm, Register result) : _masm(masm), _result(result) { save(); } ~ZCopyRuntimeCallSpill() { restore(); } }; void ZBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, Register src, Register dst, Register count, RegSet saved_regs) { if (!is_oop) { // Barrier not needed return; } BLOCK_COMMENT("ZBarrierSetAssembler::arraycopy_prologue {"); load_wide_arraycopy_masks(masm); BLOCK_COMMENT("} ZBarrierSetAssembler::arraycopy_prologue"); } static void copy_load_barrier(MacroAssembler* masm, Register ref, Address src, Register tmp) { Label done; __ ldr(tmp, Address(rthread, ZThreadLocalData::load_bad_mask_offset())); // Test reference against bad mask. If mask bad, then we need to fix it up. __ tst(ref, tmp); __ br(Assembler::EQ, done); { // Call VM ZCopyRuntimeCallSpill rcs(masm, ref); __ lea(c_rarg1, src); if (c_rarg0 != ref) { __ mov(c_rarg0, ref); } __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(IN_HEAP | ON_STRONG_OOP_REF), 2); } // Slow-path has uncolored; revert __ lsl(ref, ref, ZPointerLoadShift); __ bind(done); } static void copy_load_barrier(MacroAssembler* masm, FloatRegister ref, Address src, Register tmp1, Register tmp2, FloatRegister vec_tmp) { Label done; // Test reference against bad mask. If mask bad, then we need to fix it up. __ andr(vec_tmp, Assembler::T16B, ref, z_copy_load_bad_vreg); __ umaxv(vec_tmp, Assembler::T16B, vec_tmp); __ fcmpd(vec_tmp, 0.0); __ br(Assembler::EQ, done); __ umov(tmp2, ref, Assembler::D, 0); copy_load_barrier(masm, tmp2, Address(src.base(), src.offset() + 0), tmp1); __ mov(ref, __ D, 0, tmp2); __ umov(tmp2, ref, Assembler::D, 1); copy_load_barrier(masm, tmp2, Address(src.base(), src.offset() + 8), tmp1); __ mov(ref, __ D, 1, tmp2); __ bind(done); } static void copy_store_barrier(MacroAssembler* masm, Register pre_ref, Register new_ref, Address src, Register tmp1, Register tmp2) { Label done; Label slow; // Test reference against bad mask. If mask bad, then we need to fix it up. __ ldr(tmp1, Address(rthread, ZThreadLocalData::store_bad_mask_offset())); __ tst(pre_ref, tmp1); __ br(Assembler::EQ, done); store_barrier_buffer_add(masm, src, tmp1, tmp2, slow); __ b(done); __ bind(slow); { // Call VM ZCopyRuntimeCallSpill rcs(masm, noreg); __ lea(c_rarg0, src); __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr(), 1); } __ bind(done); if (new_ref != noreg) { // Set store-good color, replacing whatever color was there before __ ldr(tmp1, Address(rthread, ZThreadLocalData::store_good_mask_offset())); __ bfi(new_ref, tmp1, 0, 16); } } static void copy_store_barrier(MacroAssembler* masm, FloatRegister pre_ref, FloatRegister new_ref, Address src, Register tmp1, Register tmp2, Register tmp3, FloatRegister vec_tmp) { Label done; // Test reference against bad mask. If mask bad, then we need to fix it up. __ andr(vec_tmp, Assembler::T16B, pre_ref, z_copy_store_bad_vreg); __ umaxv(vec_tmp, Assembler::T16B, vec_tmp); __ fcmpd(vec_tmp, 0.0); __ br(Assembler::EQ, done); // Extract the 2 oops from the pre_ref vector register __ umov(tmp2, pre_ref, Assembler::D, 0); copy_store_barrier(masm, tmp2, noreg, Address(src.base(), src.offset() + 0), tmp1, tmp3); __ umov(tmp2, pre_ref, Assembler::D, 1); copy_store_barrier(masm, tmp2, noreg, Address(src.base(), src.offset() + 8), tmp1, tmp3); __ bind(done); // Remove any bad colors __ bic(new_ref, Assembler::T16B, new_ref, z_copy_store_bad_vreg); // Add good colors __ orr(new_ref, Assembler::T16B, new_ref, z_copy_store_good_vreg); } class ZAdjustAddress { private: MacroAssembler* _masm; Address _addr; int _pre_adjustment; int _post_adjustment; void pre() { if (_pre_adjustment != 0) { _masm->add(_addr.base(), _addr.base(), _addr.offset()); } } void post() { if (_post_adjustment != 0) { _masm->add(_addr.base(), _addr.base(), _addr.offset()); } } public: ZAdjustAddress(MacroAssembler* masm, Address addr) : _masm(masm), _addr(addr), _pre_adjustment(addr.getMode() == Address::pre ? addr.offset() : 0), _post_adjustment(addr.getMode() == Address::post ? addr.offset() : 0) { pre(); } ~ZAdjustAddress() { post(); } Address address() { if (_pre_adjustment != 0 || _post_adjustment != 0) { return Address(_addr.base(), 0); } else { return Address(_addr.base(), _addr.offset()); } } }; void ZBarrierSetAssembler::copy_load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, size_t bytes, Register dst1, Register dst2, Address src, Register tmp) { if (!is_reference_type(type)) { BarrierSetAssembler::copy_load_at(masm, decorators, type, bytes, dst1, dst2, src, noreg); return; } ZAdjustAddress adjust(masm, src); src = adjust.address(); BarrierSetAssembler::copy_load_at(masm, decorators, type, bytes, dst1, dst2, src, noreg); if (bytes == 8) { copy_load_barrier(masm, dst1, src, tmp); } else if (bytes == 16) { copy_load_barrier(masm, dst1, Address(src.base(), src.offset() + 0), tmp); copy_load_barrier(masm, dst2, Address(src.base(), src.offset() + 8), tmp); } else { ShouldNotReachHere(); } if ((decorators & ARRAYCOPY_CHECKCAST) != 0) { __ lsr(dst1, dst1, ZPointerLoadShift); } } void ZBarrierSetAssembler::copy_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, size_t bytes, Address dst, Register src1, Register src2, Register tmp1, Register tmp2, Register tmp3) { if (!is_reference_type(type)) { BarrierSetAssembler::copy_store_at(masm, decorators, type, bytes, dst, src1, src2, noreg, noreg, noreg); return; } ZAdjustAddress adjust(masm, dst); dst = adjust.address(); if ((decorators & ARRAYCOPY_CHECKCAST) != 0) { __ lsl(src1, src1, ZPointerLoadShift); } bool is_dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; if (is_dest_uninitialized) { __ ldr(tmp1, Address(rthread, ZThreadLocalData::store_good_mask_offset())); if (bytes == 8) { __ bfi(src1, tmp1, 0, 16); } else if (bytes == 16) { __ bfi(src1, tmp1, 0, 16); __ bfi(src2, tmp1, 0, 16); } else { ShouldNotReachHere(); } } else { // Store barrier pre values and color new values if (bytes == 8) { __ ldr(tmp1, dst); copy_store_barrier(masm, tmp1, src1, dst, tmp2, tmp3); } else if (bytes == 16) { Address dst1(dst.base(), dst.offset() + 0); Address dst2(dst.base(), dst.offset() + 8); __ ldr(tmp1, dst1); copy_store_barrier(masm, tmp1, src1, dst1, tmp2, tmp3); __ ldr(tmp1, dst2); copy_store_barrier(masm, tmp1, src2, dst2, tmp2, tmp3); } else { ShouldNotReachHere(); } } // Store new values BarrierSetAssembler::copy_store_at(masm, decorators, type, bytes, dst, src1, src2, noreg, noreg, noreg); } void ZBarrierSetAssembler::copy_load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, size_t bytes, FloatRegister dst1, FloatRegister dst2, Address src, Register tmp1, Register tmp2, FloatRegister vec_tmp) { if (!is_reference_type(type)) { BarrierSetAssembler::copy_load_at(masm, decorators, type, bytes, dst1, dst2, src, noreg, noreg, fnoreg); return; } ZAdjustAddress adjust(masm, src); src = adjust.address(); BarrierSetAssembler::copy_load_at(masm, decorators, type, bytes, dst1, dst2, src, noreg, noreg, fnoreg); if (bytes == 32) { copy_load_barrier(masm, dst1, Address(src.base(), src.offset() + 0), tmp1, tmp2, vec_tmp); copy_load_barrier(masm, dst2, Address(src.base(), src.offset() + 16), tmp1, tmp2, vec_tmp); } else { ShouldNotReachHere(); } } void ZBarrierSetAssembler::copy_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, size_t bytes, Address dst, FloatRegister src1, FloatRegister src2, Register tmp1, Register tmp2, Register tmp3, FloatRegister vec_tmp1, FloatRegister vec_tmp2, FloatRegister vec_tmp3) { if (!is_reference_type(type)) { BarrierSetAssembler::copy_store_at(masm, decorators, type, bytes, dst, src1, src2, noreg, noreg, noreg, fnoreg, fnoreg, fnoreg); return; } bool is_dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; ZAdjustAddress adjust(masm, dst); dst = adjust.address(); if (is_dest_uninitialized) { if (bytes == 32) { __ bic(src1, Assembler::T16B, src1, z_copy_store_bad_vreg); __ orr(src1, Assembler::T16B, src1, z_copy_store_good_vreg); __ bic(src2, Assembler::T16B, src2, z_copy_store_bad_vreg); __ orr(src2, Assembler::T16B, src2, z_copy_store_good_vreg); } else { ShouldNotReachHere(); } } else { // Load pre values BarrierSetAssembler::copy_load_at(masm, decorators, type, bytes, vec_tmp1, vec_tmp2, dst, noreg, noreg, fnoreg); // Store barrier pre values and color new values if (bytes == 32) { copy_store_barrier(masm, vec_tmp1, src1, Address(dst.base(), dst.offset() + 0), tmp1, tmp2, tmp3, vec_tmp3); copy_store_barrier(masm, vec_tmp2, src2, Address(dst.base(), dst.offset() + 16), tmp1, tmp2, tmp3, vec_tmp3); } else { ShouldNotReachHere(); } } // Store new values BarrierSetAssembler::copy_store_at(masm, decorators, type, bytes, dst, src1, src2, noreg, noreg, noreg, fnoreg, fnoreg, fnoreg); } void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env, Register robj, Register tmp, Label& slowpath) { BLOCK_COMMENT("ZBarrierSetAssembler::try_resolve_jobject_in_native {"); Label done, tagged, weak_tagged, uncolor; // Test for tag __ tst(robj, JNIHandles::tag_mask); __ br(Assembler::NE, tagged); // Resolve local handle __ ldr(robj, robj); __ b(done); __ bind(tagged); // Test for weak tag __ tst(robj, JNIHandles::TypeTag::weak_global); __ br(Assembler::NE, weak_tagged); // Resolve global handle __ ldr(robj, Address(robj, -JNIHandles::TypeTag::global)); __ lea(tmp, load_bad_mask_from_jni_env(jni_env)); __ ldr(tmp, tmp); __ tst(robj, tmp); __ br(Assembler::NE, slowpath); __ b(uncolor); __ bind(weak_tagged); // Resolve weak handle __ ldr(robj, Address(robj, -JNIHandles::TypeTag::weak_global)); __ lea(tmp, mark_bad_mask_from_jni_env(jni_env)); __ ldr(tmp, tmp); __ tst(robj, tmp); __ br(Assembler::NE, slowpath); __ bind(uncolor); // Uncolor __ lsr(robj, robj, ZPointerLoadShift); __ bind(done); BLOCK_COMMENT("} ZBarrierSetAssembler::try_resolve_jobject_in_native"); } static uint16_t patch_barrier_relocation_value(int format) { switch (format) { case ZBarrierRelocationFormatLoadGoodBeforeTbX: return (uint16_t)exact_log2(ZPointerRemapped); case ZBarrierRelocationFormatMarkBadBeforeMov: return (uint16_t)ZPointerMarkBadMask; case ZBarrierRelocationFormatStoreGoodBeforeMov: return (uint16_t)ZPointerStoreGoodMask; case ZBarrierRelocationFormatStoreBadBeforeMov: return (uint16_t)ZPointerStoreBadMask; default: ShouldNotReachHere(); return 0; } } static void change_immediate(uint32_t& instr, uint32_t imm, uint32_t start, uint32_t end) { uint32_t imm_mask = ((1u << start) - 1u) ^ ((1u << (end + 1)) - 1u); instr &= ~imm_mask; instr |= imm << start; } void ZBarrierSetAssembler::patch_barrier_relocation(address addr, int format) { const uint16_t value = patch_barrier_relocation_value(format); uint32_t* const patch_addr = (uint32_t*)addr; switch (format) { case ZBarrierRelocationFormatLoadGoodBeforeTbX: change_immediate(*patch_addr, value, 19, 23); break; case ZBarrierRelocationFormatStoreGoodBeforeMov: case ZBarrierRelocationFormatMarkBadBeforeMov: case ZBarrierRelocationFormatStoreBadBeforeMov: change_immediate(*patch_addr, value, 5, 20); break; default: ShouldNotReachHere(); } OrderAccess::fence(); ICache::invalidate_word((address)patch_addr); } #ifdef COMPILER1 #undef __ #define __ ce->masm()-> static void z_uncolor(LIR_Assembler* ce, LIR_Opr ref) { __ lsr(ref->as_register(), ref->as_register(), ZPointerLoadShift); } static void z_color(LIR_Assembler* ce, LIR_Opr ref) { __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodBeforeMov); __ movzw(rscratch2, barrier_Relocation::unpatched); __ orr(ref->as_register(), rscratch2, ref->as_register(), Assembler::LSL, ZPointerLoadShift); } void ZBarrierSetAssembler::generate_c1_uncolor(LIR_Assembler* ce, LIR_Opr ref) const { z_uncolor(ce, ref); } void ZBarrierSetAssembler::generate_c1_color(LIR_Assembler* ce, LIR_Opr ref) const { z_color(ce, ref); } void ZBarrierSetAssembler::generate_c1_load_barrier(LIR_Assembler* ce, LIR_Opr ref, ZLoadBarrierStubC1* stub, bool on_non_strong) const { if (on_non_strong) { // Test against MarkBad mask assert_different_registers(rscratch1, rthread, ref->as_register()); __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatMarkBadBeforeMov); __ movzw(rscratch1, barrier_Relocation::unpatched); __ tst(ref->as_register(), rscratch1); __ br(Assembler::NE, *stub->entry()); z_uncolor(ce, ref); } else { Label good; __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatLoadGoodBeforeTbX); __ tbz(ref->as_register(), barrier_Relocation::unpatched, good); __ b(*stub->entry()); __ bind(good); z_uncolor(ce, ref); } __ bind(*stub->continuation()); } void ZBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce, ZLoadBarrierStubC1* stub) const { // Stub entry __ bind(*stub->entry()); Register ref = stub->ref()->as_register(); Register ref_addr = noreg; Register tmp = noreg; if (stub->tmp()->is_valid()) { // Load address into tmp register ce->leal(stub->ref_addr(), stub->tmp()); ref_addr = tmp = stub->tmp()->as_pointer_register(); } else { // Address already in register ref_addr = stub->ref_addr()->as_address_ptr()->base()->as_pointer_register(); } assert_different_registers(ref, ref_addr, noreg); // Save r0 unless it is the result or tmp register // Set up SP to accommodate parameters and maybe r0.. if (ref != r0 && tmp != r0) { __ sub(sp, sp, 32); __ str(r0, Address(sp, 16)); } else { __ sub(sp, sp, 16); } // Setup arguments and call runtime stub ce->store_parameter(ref_addr, 1); ce->store_parameter(ref, 0); __ far_call(stub->runtime_stub()); // Verify result __ verify_oop(r0); // Move result into place if (ref != r0) { __ mov(ref, r0); } // Restore r0 unless it is the result or tmp register if (ref != r0 && tmp != r0) { __ ldr(r0, Address(sp, 16)); __ add(sp, sp, 32); } else { __ add(sp, sp, 16); } // Stub exit __ b(*stub->continuation()); } void ZBarrierSetAssembler::generate_c1_store_barrier(LIR_Assembler* ce, LIR_Address* addr, LIR_Opr new_zaddress, LIR_Opr new_zpointer, ZStoreBarrierStubC1* stub) const { Register rnew_zaddress = new_zaddress->as_register(); Register rnew_zpointer = new_zpointer->as_register(); store_barrier_fast(ce->masm(), ce->as_Address(addr), rnew_zaddress, rnew_zpointer, rscratch2, true, stub->is_atomic(), *stub->entry(), *stub->continuation()); } void ZBarrierSetAssembler::generate_c1_store_barrier_stub(LIR_Assembler* ce, ZStoreBarrierStubC1* stub) const { // Stub entry __ bind(*stub->entry()); Label slow; Label slow_continuation; store_barrier_medium(ce->masm(), ce->as_Address(stub->ref_addr()->as_address_ptr()), rscratch2, stub->new_zpointer()->as_register(), rscratch1, false /* is_native */, stub->is_atomic(), *stub->continuation(), slow, slow_continuation); __ bind(slow); __ lea(stub->new_zpointer()->as_register(), ce->as_Address(stub->ref_addr()->as_address_ptr())); __ sub(sp, sp, 16); // Setup arguments and call runtime stub assert(stub->new_zpointer()->is_valid(), "invariant"); ce->store_parameter(stub->new_zpointer()->as_register(), 0); __ far_call(stub->runtime_stub()); __ add(sp, sp, 16); // Stub exit __ b(slow_continuation); } #undef __ #define __ sasm-> void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* sasm, DecoratorSet decorators) const { __ prologue("zgc_load_barrier stub", false); __ push_call_clobbered_registers_except(RegSet::of(r0)); // Setup arguments __ load_parameter(0, c_rarg0); __ load_parameter(1, c_rarg1); __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2); __ pop_call_clobbered_registers_except(RegSet::of(r0)); __ epilogue(); } void ZBarrierSetAssembler::generate_c1_store_barrier_runtime_stub(StubAssembler* sasm, bool self_healing) const { __ prologue("zgc_store_barrier stub", false); __ push_call_clobbered_registers(); // Setup arguments __ load_parameter(0, c_rarg0); if (self_healing) { __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_with_healing_addr(), 1); } else { __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr(), 1); } __ pop_call_clobbered_registers(); __ epilogue(); } #endif // COMPILER1 #ifdef COMPILER2 #undef __ #define __ _masm-> class ZSetupArguments { private: MacroAssembler* const _masm; const Register _ref; const Address _ref_addr; public: ZSetupArguments(MacroAssembler* masm, ZLoadBarrierStubC2* stub) : _masm(masm), _ref(stub->ref()), _ref_addr(stub->ref_addr()) { // Setup arguments if (_ref_addr.base() == noreg) { // No self healing if (_ref != c_rarg0) { __ mov(c_rarg0, _ref); } __ mov(c_rarg1, 0); } else { // Self healing if (_ref == c_rarg0) { // _ref is already at correct place __ lea(c_rarg1, _ref_addr); } else if (_ref != c_rarg1) { // _ref is in wrong place, but not in c_rarg1, so fix it first __ lea(c_rarg1, _ref_addr); __ mov(c_rarg0, _ref); } else if (_ref_addr.base() != c_rarg0 && _ref_addr.index() != c_rarg0) { assert(_ref == c_rarg1, "Mov ref first, vacating c_rarg0"); __ mov(c_rarg0, _ref); __ lea(c_rarg1, _ref_addr); } else { assert(_ref == c_rarg1, "Need to vacate c_rarg1 and _ref_addr is using c_rarg0"); if (_ref_addr.base() == c_rarg0 || _ref_addr.index() == c_rarg0) { __ mov(rscratch2, c_rarg1); __ lea(c_rarg1, _ref_addr); __ mov(c_rarg0, rscratch2); } else { ShouldNotReachHere(); } } } } ~ZSetupArguments() { // Transfer result if (_ref != r0) { __ mov(_ref, r0); } } }; #undef __ #define __ masm-> void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, ZLoadBarrierStubC2* stub) const { Assembler::InlineSkippedInstructionsCounter skipped_counter(masm); BLOCK_COMMENT("ZLoadBarrierStubC2"); // Stub entry if (!Compile::current()->output()->in_scratch_emit_size()) { __ bind(*stub->entry()); } { SaveLiveRegisters save_live_registers(masm, stub); ZSetupArguments setup_arguments(masm, stub); __ mov(rscratch1, stub->slow_path()); __ blr(rscratch1); } // Stub exit __ b(*stub->continuation()); } void ZBarrierSetAssembler::generate_c2_store_barrier_stub(MacroAssembler* masm, ZStoreBarrierStubC2* stub) const { Assembler::InlineSkippedInstructionsCounter skipped_counter(masm); BLOCK_COMMENT("ZStoreBarrierStubC2"); // Stub entry __ bind(*stub->entry()); Label slow; Label slow_continuation; store_barrier_medium(masm, stub->ref_addr(), stub->new_zpointer(), rscratch1, rscratch2, stub->is_native(), stub->is_atomic(), *stub->continuation(), slow, slow_continuation); __ bind(slow); { SaveLiveRegisters save_live_registers(masm, stub); __ lea(c_rarg0, stub->ref_addr()); if (stub->is_native()) { __ lea(rscratch1, RuntimeAddress(ZBarrierSetRuntime::store_barrier_on_native_oop_field_without_healing_addr())); } else if (stub->is_atomic()) { __ lea(rscratch1, RuntimeAddress(ZBarrierSetRuntime::store_barrier_on_oop_field_with_healing_addr())); } else if (stub->is_nokeepalive()) { __ lea(rscratch1, RuntimeAddress(ZBarrierSetRuntime::no_keepalive_store_barrier_on_oop_field_without_healing_addr())); } else { __ lea(rscratch1, RuntimeAddress(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr())); } __ blr(rscratch1); } // Stub exit __ b(slow_continuation); } // Only handles forward branch jumps, target_offset >= branch_offset static bool aarch64_test_and_branch_reachable(int branch_offset, int target_offset) { assert(branch_offset >= 0, "branch to stub offsets must be positive"); assert(target_offset >= 0, "offset in stubs section must be positive"); assert(target_offset >= branch_offset, "forward branches only, branch_offset -> target_offset"); const int test_and_branch_delta_limit = 32 * K; const int test_and_branch_to_trampoline_delta = target_offset - branch_offset; return test_and_branch_to_trampoline_delta < test_and_branch_delta_limit; } ZLoadBarrierStubC2Aarch64::ZLoadBarrierStubC2Aarch64(const MachNode* node, Address ref_addr, Register ref) : ZLoadBarrierStubC2(node, ref_addr, ref), _test_and_branch_reachable_entry(), _offset(), _deferred_emit(false), _test_and_branch_reachable(false) {} ZLoadBarrierStubC2Aarch64::ZLoadBarrierStubC2Aarch64(const MachNode* node, Address ref_addr, Register ref, int offset) : ZLoadBarrierStubC2(node, ref_addr, ref), _test_and_branch_reachable_entry(), _offset(offset), _deferred_emit(false), _test_and_branch_reachable(false) { PhaseOutput* const output = Compile::current()->output(); if (output->in_scratch_emit_size()) { return; } const int code_size = output->buffer_sizing_data()->_code; const int offset_code = _offset; // Assumption that the stub can always be reached from a branch immediate. (128 M Product, 2 M Debug) // Same assumption is made in z_aarch64.ad const int trampoline_offset = trampoline_stubs_count() * NativeInstruction::instruction_size; _test_and_branch_reachable = aarch64_test_and_branch_reachable(offset_code, code_size + trampoline_offset); if (_test_and_branch_reachable) { inc_trampoline_stubs_count(); } } int ZLoadBarrierStubC2Aarch64::get_stub_size() { PhaseOutput* const output = Compile::current()->output(); assert(!output->in_scratch_emit_size(), "only used when emitting stubs"); BufferBlob* const blob = output->scratch_buffer_blob(); CodeBuffer cb(blob->content_begin(), (address)output->scratch_locs_memory() - blob->content_begin()); MacroAssembler masm(&cb); output->set_in_scratch_emit_size(true); ZLoadBarrierStubC2::emit_code(masm); output->set_in_scratch_emit_size(false); return cb.insts_size(); } ZLoadBarrierStubC2Aarch64* ZLoadBarrierStubC2Aarch64::create(const MachNode* node, Address ref_addr, Register ref) { ZLoadBarrierStubC2Aarch64* const stub = new (Compile::current()->comp_arena()) ZLoadBarrierStubC2Aarch64(node, ref_addr, ref); register_stub(stub); return stub; } ZLoadBarrierStubC2Aarch64* ZLoadBarrierStubC2Aarch64::create(const MachNode* node, Address ref_addr, Register ref, int offset) { ZLoadBarrierStubC2Aarch64* const stub = new (Compile::current()->comp_arena()) ZLoadBarrierStubC2Aarch64(node, ref_addr, ref, offset); register_stub(stub); return stub; } #undef __ #define __ masm. void ZLoadBarrierStubC2Aarch64::emit_code(MacroAssembler& masm) { PhaseOutput* const output = Compile::current()->output(); const int branch_offset = _offset; const int target_offset = __ offset(); // Deferred emission, emit actual stub if (_deferred_emit) { ZLoadBarrierStubC2::emit_code(masm); return; } _deferred_emit = true; // No trampoline used, defer emission to after trampolines if (!_test_and_branch_reachable) { register_stub(this); return; } // Current assumption is that the barrier stubs are the first stubs emitted after the actual code assert(stubs_start_offset() <= output->buffer_sizing_data()->_code, "stubs are assumed to be emitted directly after code and code_size is a hard limit on where it can start"); __ bind(_test_and_branch_reachable_entry); // Next branch's offset is unknown, but is > branch_offset const int next_branch_offset = branch_offset + NativeInstruction::instruction_size; // If emitting the stub directly does not interfere with emission of the next trampoline then do it to avoid a double jump. if (aarch64_test_and_branch_reachable(next_branch_offset, target_offset + get_stub_size())) { // The next potential trampoline will still be reachable even if we emit the whole stub ZLoadBarrierStubC2::emit_code(masm); } else { // Emit trampoline and defer actual stub to the end assert(aarch64_test_and_branch_reachable(branch_offset, target_offset), "trampoline should be reachable"); __ b(*ZLoadBarrierStubC2::entry()); register_stub(this); } } bool ZLoadBarrierStubC2Aarch64::is_test_and_branch_reachable() { return _test_and_branch_reachable; } Label* ZLoadBarrierStubC2Aarch64::entry() { if (_test_and_branch_reachable) { return &_test_and_branch_reachable_entry; } return ZBarrierStubC2::entry(); } ZStoreBarrierStubC2Aarch64::ZStoreBarrierStubC2Aarch64(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic, bool is_nokeepalive) : ZStoreBarrierStubC2(node, ref_addr, new_zaddress, new_zpointer, is_native, is_atomic, is_nokeepalive), _deferred_emit(false) {} ZStoreBarrierStubC2Aarch64* ZStoreBarrierStubC2Aarch64::create(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic, bool is_nokeepalive) { ZStoreBarrierStubC2Aarch64* const stub = new (Compile::current()->comp_arena()) ZStoreBarrierStubC2Aarch64(node, ref_addr, new_zaddress, new_zpointer, is_native, is_atomic, is_nokeepalive); register_stub(stub); return stub; } void ZStoreBarrierStubC2Aarch64::emit_code(MacroAssembler& masm) { if (_deferred_emit) { ZStoreBarrierStubC2::emit_code(masm); return; } // Defer emission of store barriers so that trampolines are emitted first _deferred_emit = true; register_stub(this); } #undef __ #endif // COMPILER2 #undef __ #define __ masm-> void ZBarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) { // C1 calls verfy_oop in the middle of barriers, before they have been uncolored // and after being colored. Therefore, we must deal with colored oops as well. Label done; Label check_oop; Label check_zaddress; int color_bits = ZPointerRemappedShift + ZPointerRemappedBits; uintptr_t shifted_base_start_mask = (UCONST64(1) << (ZAddressHeapBaseShift + color_bits + 1)) - 1; uintptr_t shifted_base_end_mask = (UCONST64(1) << (ZAddressHeapBaseShift + 1)) - 1; uintptr_t shifted_base_mask = shifted_base_start_mask ^ shifted_base_end_mask; uintptr_t shifted_address_end_mask = (UCONST64(1) << (color_bits + 1)) - 1; uintptr_t shifted_address_mask = shifted_address_end_mask ^ (uintptr_t)CONST64(-1); __ get_nzcv(tmp2); // Check colored null __ mov(tmp1, shifted_address_mask); __ tst(tmp1, obj); __ br(Assembler::EQ, done); // Check for zpointer __ mov(tmp1, shifted_base_mask); __ tst(tmp1, obj); __ br(Assembler::EQ, check_oop); // Uncolor presumed zpointer __ lsr(obj, obj, ZPointerLoadShift); __ b(check_zaddress); __ bind(check_oop); // make sure klass is 'reasonable', which is not zero. __ load_klass(tmp1, obj); // get klass __ tst(tmp1, tmp1); __ br(Assembler::EQ, error); // if klass is null it is broken __ bind(check_zaddress); // Check if the oop is in the right area of memory __ mov(tmp1, (intptr_t) Universe::verify_oop_mask()); __ andr(tmp1, tmp1, obj); __ mov(obj, (intptr_t) Universe::verify_oop_bits()); __ cmp(tmp1, obj); __ br(Assembler::NE, error); __ bind(done); __ set_nzcv(tmp2); } #undef __