diff --git a/erts/configure b/erts/configure index 798868db8631..94295978886f 100755 --- a/erts/configure +++ b/erts/configure @@ -658,6 +658,7 @@ EMU_CC ERTS_BUILD_FALLBACK_POLL FLAVORS PRIMARY_FLAVOR +JIT_ARCH_BITS JIT_ARCH JIT_ENABLED M4 @@ -25459,6 +25460,7 @@ fi JIT_ARCH= +JIT_ARCH_BITS= if test ${enable_jit} != no then : @@ -25499,6 +25501,7 @@ printf "%s\n" "$as_me: WARNING: JIT disabled due to annoying popus on x86 Macs w # instruction cache, and don't require us to manually issue # instruction barriers on all threads. JIT_ARCH=arm + JIT_ARCH_BITS=64 ;; *) # We need to use `DC CVAU`, `IC IVAU`, and `ISB SY` to clear @@ -25507,6 +25510,7 @@ printf "%s\n" "$as_me: WARNING: JIT disabled due to annoying popus on x86 Macs w if test "$ethr_arm_isb_sy_instr_val$ethr_arm_dc_cvau_instr_val$ethr_arm_ic_ivau_instr_val" = "111"; then JIT_ARCH=arm + JIT_ARCH_BITS=64 else enable_jit=no { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: JIT disabled due to lack of cache-clearing instructions" >&5 @@ -25515,6 +25519,22 @@ printf "%s\n" "$as_me: WARNING: JIT disabled due to lack of cache-clearing instr ;; esac ;; + arm) + case "$OPSYS" in + linux) + # These platforms have dedicated system calls for clearing + # instruction cache, and don't require us to manually issue + # instruction barriers on all threads. + JIT_ARCH=arm + JIT_ARCH_BITS=32 + ;; + *) + enable_jit=no + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: JIT disabled due to lack of support on $ARCH-$OPSYS" >&5 +printf "%s\n" "$as_me: WARNING: JIT disabled due to lack of support on $ARCH-$OPSYS" >&2;} + ;; + esac + ;; *) if test ${enable_jit} = yes; then as_fn_error $? "JIT only works on x86 64-bit and ARM 64-bit" "$LINENO" 5 @@ -25816,6 +25836,7 @@ fi + # # Check for working poll(). # diff --git a/erts/emulator/beam/jit/arm/32/beam_asm.hpp b/erts/emulator/beam/jit/arm/32/beam_asm.hpp index 75090914ae32..799f64e4ffd1 100644 --- a/erts/emulator/beam/jit/arm/32/beam_asm.hpp +++ b/erts/emulator/beam/jit/arm/32/beam_asm.hpp @@ -165,7 +165,7 @@ struct BeamAssembler : public BeamAssemblerCommon { const int margin_bytes = margin_words * sizeof(Eterm); Label next = a.newLabel(); - a.sub(TMP, E, imm(margin_bytes)); + sub(TMP, E, margin_bytes); a.cmp(HTOP, TMP); a.b_ls(next); @@ -330,23 +330,23 @@ struct BeamAssembler : public BeamAssemblerCommon { ERTS_CT_ASSERT((Spec & (Update::eReductions | Update::eStack | Update::eHeap)) == Spec); if (Spec & Update::eStack) { - a.str(E, arm::Mem(c_p, offsetof(Process, stop))); + this->safe_str(E, arm::Mem(c_p, offsetof(Process, stop))); } else { #ifdef DEBUG /* Store some garbage in the process structure to catch missing * updates. */ - a.str(active_code_ix, arm::Mem(c_p, offsetof(Process, stop))); + this->safe_str(active_code_ix, arm::Mem(c_p, offsetof(Process, stop))); #endif } if (Spec & Update::eHeap) { - a.str(HTOP, arm::Mem(c_p, offsetof(Process, htop))); + this->safe_str(HTOP, arm::Mem(c_p, offsetof(Process, htop))); } else { #ifdef DEBUG - a.str(active_code_ix, arm::Mem(c_p, offsetof(Process, htop))); + this->safe_str(active_code_ix, arm::Mem(c_p, offsetof(Process, htop))); #endif } if (Spec & Update::eReductions) { - a.str(FCALLS, arm::Mem(c_p, offsetof(Process, fcalls))); + this->safe_str(FCALLS, arm::Mem(c_p, offsetof(Process, fcalls))); } // We do not have any X register cached in machine registers // so nothing else needs to be saved. @@ -358,13 +358,13 @@ struct BeamAssembler : public BeamAssemblerCommon { (Spec & (Update::eReductions | Update::eStack | Update::eHeap | Update::eCodeIndex)) == Spec); if (Spec & Update::eStack) { - a.ldr(E, arm::Mem(c_p, offsetof(Process, stop))); + this->safe_ldr(E, arm::Mem(c_p, offsetof(Process, stop))); } if (Spec & Update::eHeap) { - a.ldr(HTOP, arm::Mem(c_p, offsetof(Process, htop))); + this->safe_ldr(HTOP, arm::Mem(c_p, offsetof(Process, htop))); } if (Spec & Update::eReductions) { - a.ldr(FCALLS, arm::Mem(c_p, offsetof(Process, fcalls))); + this->safe_ldr(FCALLS, arm::Mem(c_p, offsetof(Process, fcalls))); } if (Spec & Update::eCodeIndex) { @@ -444,7 +444,7 @@ struct BeamAssembler : public BeamAssemblerCommon { } void emit_branch_if_eq(a32::Gp reg, Uint value, Label lbl) { - if (value <= 255) { + if (isAArch32Immediate(value)) { a.cmp(reg, imm(value)); } else { mov_imm(TMP, value); @@ -454,7 +454,7 @@ struct BeamAssembler : public BeamAssemblerCommon { } void emit_branch_if_ne(a32::Gp reg, Uint value, Label lbl) { - if (value <= 255) { + if (isAArch32Immediate(value)) { a.cmp(reg, imm(value)); } else { mov_imm(TMP, value); @@ -512,12 +512,17 @@ struct BeamAssembler : public BeamAssemblerCommon { mov_imm(to, 0); } + static bool isAArch32Immediate(uint64_t value) { + uint32_t encoded; + return arm::Utils::encodeAArch32Imm(value, &encoded); + } + void sub(a32::Gp to, a32::Gp src, int64_t val) { if (val < 0) { add(to, src, -val); } else if (val == 0 && to != src) { a.mov(to, src); - } else if (val <= 255) { + } else if (isAArch32Immediate(val)) { a.sub(to, src, imm(val)); } else { ASSERT(src != TMP); @@ -531,7 +536,7 @@ struct BeamAssembler : public BeamAssemblerCommon { sub(to, src, -val); } else if (val == 0 && to != src) { a.mov(to, src); - } else if (val <= 255) { + } else if (isAArch32Immediate(val)) { a.add(to, src, imm(val)); } else { ASSERT(src != TMP); @@ -541,9 +546,9 @@ struct BeamAssembler : public BeamAssemblerCommon { } void subs(a32::Gp to, a32::Gp src, int64_t val) { - if (val >= 0 && val <= 255) { + if (val >= 0 && isAArch32Immediate(val)) { a.subs(to, src, imm(val)); - } else if (val < 0 && -val <= 255) { + } else if (val < 0 && isAArch32Immediate(-val)) { a.adds(to, src, imm(-val)); } else { ASSERT(src != TMP); @@ -552,6 +557,78 @@ struct BeamAssembler : public BeamAssemblerCommon { } } + void safe_str(a32::Gp gp, arm::Mem mem) { + size_t abs_offset = std::abs(mem.offset()); + auto offset = mem.offset(); + constexpr size_t max_disp = 4095; + + ASSERT(mem.hasBaseReg() && !mem.hasIndex()); + ASSERT(gp.isGp()); + + if (abs_offset <= max_disp) { + a.str(gp, mem); + } else { + a32::Gp addr = (gp != TMP) ? TMP : VAR; + ASSERT(addr != gp); + add(addr, a32::Gp(mem.baseId()), offset); + a.str(gp, arm::Mem(addr)); + } + } + + void safe_ldr(a32::Gp gp, arm::Mem mem) { + size_t abs_offset = std::abs(mem.offset()); + auto offset = mem.offset(); + constexpr size_t max_disp = 4095; + + ASSERT(mem.hasBaseReg() && !mem.hasIndex()); + ASSERT(gp.isGp()); + + if (abs_offset <= max_disp) { + a.ldr(gp, mem); + } else { + a32::Gp addr = (gp != TMP) ? TMP : VAR; + ASSERT(addr != gp); + add(addr, a32::Gp(mem.baseId()), offset); + a.ldr(gp, arm::Mem(addr)); + } + } + + void safe_strb(a32::Gp gp, arm::Mem mem) { + size_t abs_offset = std::abs(mem.offset()); + auto offset = mem.offset(); + constexpr size_t max_disp = 4095; + + ASSERT(mem.hasBaseReg() && !mem.hasIndex()); + ASSERT(gp.isGp()); + + if (abs_offset <= max_disp) { + a.strb(gp, mem); + } else { + a32::Gp addr = (gp != TMP) ? TMP : VAR; + ASSERT(addr != gp); + add(addr, a32::Gp(mem.baseId()), offset); + a.strb(gp, arm::Mem(addr)); + } + } + + void safe_ldrb(a32::Gp gp, arm::Mem mem) { + size_t abs_offset = std::abs(mem.offset()); + auto offset = mem.offset(); + constexpr size_t max_disp = 4095; + + ASSERT(mem.hasBaseReg() && !mem.hasIndex()); + ASSERT(gp.isGp()); + + if (abs_offset <= max_disp) { + a.ldrb(gp, mem); + } else { + a32::Gp addr = (gp != TMP) ? TMP : VAR; + ASSERT(addr != gp); + add(addr, a32::Gp(mem.baseId()), offset); + a.ldrb(gp, arm::Mem(addr)); + } + } + void ldur(a32::Gp reg, arm::Mem mem) { // TODO ASSERT(false); @@ -1484,6 +1561,42 @@ class BeamModuleAssembler : public BeamAssembler, } } + void safe_ldrb(a32::Gp gp, arm::Mem mem) { + size_t abs_offset = std::abs(mem.offset()); + auto offset = mem.offset(); + + ASSERT(mem.hasBaseReg() && !mem.hasIndex()); + ASSERT(gp.isGp()); + + if (abs_offset <= disp4KB) { + preserve_cache( + [&]() { + a.ldrb(gp, mem); + }, + gp); + } else { + add(TMP, a32::Gp(mem.baseId()), offset); + a.ldrb(gp, arm::Mem(TMP)); + } + } + + void safe_strb(a32::Gp gp, arm::Mem mem) { + size_t abs_offset = std::abs(mem.offset()); + auto offset = mem.offset(); + + ASSERT(mem.hasBaseReg() && !mem.hasIndex()); + ASSERT(gp.isGp()); + + if (abs_offset <= disp4KB) { + a.strb(gp, mem); + } else { + a32::Gp addr = (gp != TMP) ? TMP : VAR; + ASSERT(addr != gp); + add(addr, a32::Gp(mem.baseId()), offset); + a.strb(gp, arm::Mem(addr)); + } + } + void safe_stmia(a32::Gp gp1, a32::Gp gp2, diff --git a/erts/emulator/beam/jit/arm/32/beam_asm_global.cpp b/erts/emulator/beam/jit/arm/32/beam_asm_global.cpp index 9331a014c817..c7fdca3c0078 100644 --- a/erts/emulator/beam/jit/arm/32/beam_asm_global.cpp +++ b/erts/emulator/beam/jit/arm/32/beam_asm_global.cpp @@ -103,11 +103,11 @@ void BeamGlobalAssembler::emit_garbage_collect() { * twice. */ a.sub(ARG2, ARG3, HTOP); a.lsr(ARG2, ARG2, imm(2)); - a.sub(ARG2, ARG2, imm(S_RESERVED)); + sub(ARG2, ARG2, S_RESERVED); /* Save our return address in c_p->i so we can tell where we crashed if we * did so during GC. */ - a.str(a32::lr, arm::Mem(c_p, offsetof(Process, i))); + safe_str(a32::lr, arm::Mem(c_p, offsetof(Process, i))); emit_enter_runtime(); @@ -125,7 +125,7 @@ void BeamGlobalAssembler::emit_garbage_collect() { emit_leave_runtime(); emit_leave_runtime_frame(); - a.ldr(TMP, arm::Mem(c_p, offsetof(Process, state.value))); + safe_ldr(TMP, arm::Mem(c_p, offsetof(Process, state.value))); a.tst(TMP, imm(ERTS_PSFLG_EXITING)); a.b_ne(labels[do_schedule]); @@ -142,7 +142,7 @@ void BeamGlobalAssembler::emit_garbage_collect() { * * Assumes that c_p->current points into the MFA of an export entry. */ void BeamGlobalAssembler::emit_bif_export_trap() { - a.ldr(ARG1, arm::Mem(c_p, offsetof(Process, current))); + safe_ldr(ARG1, arm::Mem(c_p, offsetof(Process, current))); sub(ARG1, ARG1, offsetof(Export, info.mfa)); emit_leave_erlang_frame(); @@ -187,7 +187,7 @@ void BeamGlobalAssembler::emit_export_trampoline() { ssize_t func_offset = offsetof(Export, trampoline.bif.address); lea(ARG2, arm::Mem(ARG1, offsetof(Export, info.mfa))); - a.ldr(ARG3, arm::Mem(c_p, offsetof(Process, i))); + safe_ldr(ARG3, arm::Mem(c_p, offsetof(Process, i))); a.ldr(ARG4, arm::Mem(ARG1, func_offset)); /* `call_bif_shared` assumes that the return address has been pushed to diff --git a/erts/emulator/beam/jit/arm/32/beam_asm_module.cpp b/erts/emulator/beam/jit/arm/32/beam_asm_module.cpp index 36698cb9599c..573c0dde4ddd 100644 --- a/erts/emulator/beam/jit/arm/32/beam_asm_module.cpp +++ b/erts/emulator/beam/jit/arm/32/beam_asm_module.cpp @@ -571,7 +571,10 @@ arm::Mem BeamModuleAssembler::embed_constant(const ArgVal &value, } auto it = _constants.emplace(value, - Constant{.latestOffset = maxOffset, + Constant{.latestOffset = std::max( + currOffset, + maxOffset - + STUB_CHECK_INTERVAL), .anchor = a.newLabel(), .value = value}); const Constant &constant = it->second; @@ -723,7 +726,7 @@ void BeamModuleAssembler::emit_veneer(const Veneer &veneer) { a.align(AlignMode::kCode, 4); a.bind(pointer); - a.embedLabel(veneer.target); + a.embedLabel(veneer.target, 4); } } @@ -742,7 +745,7 @@ void BeamModuleAssembler::emit_constant(const Constant &constant) { } else if (value.isWord()) { a.embedUInt32(value.as().get()); } else if (value.isLabel()) { - a.embedLabel(rawLabels.at(value.as().get())); + a.embedLabel(rawLabels.at(value.as().get()), 4); } else { switch (value.getType()) { case ArgVal::BytePtr: diff --git a/erts/emulator/beam/jit/arm/32/instr_arith.cpp b/erts/emulator/beam/jit/arm/32/instr_arith.cpp index 40b0d9b40688..f8566df27173 100644 --- a/erts/emulator/beam/jit/arm/32/instr_arith.cpp +++ b/erts/emulator/beam/jit/arm/32/instr_arith.cpp @@ -26,6 +26,11 @@ extern "C" #include "big.h" } +static bool canEncodeAArch32Imm(Uint value) { + uint32_t encoded; + return arm::Utils::encodeAArch32Imm(value, &encoded); +} + void BeamModuleAssembler::emit_add_sub_types(bool is_small_result, const ArgSource &LHS, const a32::Gp lhs_reg, @@ -148,8 +153,10 @@ void BeamModuleAssembler::emit_i_plus(const ArgLabel &Fail, const ArgSource &LHS, const ArgSource &RHS, const ArgRegister &Dst) { - bool rhs_is_arm_literal = - RHS.isSmall() && Support::isUInt12(RHS.as().get()); + bool rhs_is_arm_literal = RHS.isSmall() && + canEncodeAArch32Imm( + RHS.as().get() & + ~_TAG_IMMED1_MASK); bool is_small_result = is_sum_small_if_args_are_small(LHS, RHS); if (always_small(LHS) && always_small(RHS) && is_small_result) { @@ -341,8 +348,10 @@ void BeamModuleAssembler::emit_i_minus(const ArgLabel &Fail, const ArgSource &LHS, const ArgSource &RHS, const ArgRegister &Dst) { - bool rhs_is_arm_literal = - RHS.isSmall() && Support::isUInt12(RHS.as().get()); + bool rhs_is_arm_literal = RHS.isSmall() && + canEncodeAArch32Imm( + RHS.as().get() & + ~_TAG_IMMED1_MASK); bool is_small_result = is_diff_small_if_args_are_small(LHS, RHS); if (always_small(LHS) && always_small(RHS) && is_small_result) { diff --git a/erts/emulator/beam/jit/arm/32/instr_bif.cpp b/erts/emulator/beam/jit/arm/32/instr_bif.cpp index cb7d8bf34ab7..c347baa35254 100644 --- a/erts/emulator/beam/jit/arm/32/instr_bif.cpp +++ b/erts/emulator/beam/jit/arm/32/instr_bif.cpp @@ -263,7 +263,7 @@ void BeamGlobalAssembler::emit_i_length_common(Label fail, int state_size) { { a.ldr(ARG2, TMP_MEM1q); a.ldr(ARG3, TMP_MEM2q); - a.ldr(TMP, arm::Mem(c_p, offsetof(Process, freason))); + safe_ldr(TMP, arm::Mem(c_p, offsetof(Process, freason))); a.cmp(TMP, imm(TRAP)); a.b_ne(fail); @@ -272,11 +272,11 @@ void BeamGlobalAssembler::emit_i_length_common(Label fail, int state_size) { /* The trap state is stored in the registers above the current live * ones, so we add the state size (in words) to keep it alive. */ - a.add(ARG2, ARG2, imm(state_size)); + add(ARG2, ARG2, state_size); mov_imm(TMP, 0); - a.str(TMP, arm::Mem(c_p, offsetof(Process, current))); - a.strb(ARG2, arm::Mem(c_p, offsetof(Process, arity))); + safe_str(TMP, arm::Mem(c_p, offsetof(Process, current))); + safe_strb(ARG2, arm::Mem(c_p, offsetof(Process, arity))); /* We'll find our way back through the entry address (ARG3). */ a.b(labels[context_switch_simplified]); @@ -390,7 +390,7 @@ void BeamGlobalAssembler::emit_call_light_bif_shared() { Label trace = a.newLabel(), yield = a.newLabel(), skip_trace = a.newLabel(); /* Spill everything we may need on the error and GC paths. */ - a.ldr(TMP, arm::Mem(c_p, offsetof(Process, mbuf))); + safe_ldr(TMP, arm::Mem(c_p, offsetof(Process, mbuf))); a.str(TMP, mbuf_mem); lea(TMP, TMP_MEM1q); a.stmia(arm::Mem(TMP), a32::GpList({ARG3, ARG4})); @@ -456,13 +456,13 @@ void BeamGlobalAssembler::emit_call_light_bif_shared() { Label check_fragments = a.newLabel(); /* Test whether GC is forced. */ - a.ldr(TMP, arm::Mem(c_p, offsetof(Process, flags))); + safe_ldr(TMP, arm::Mem(c_p, offsetof(Process, flags))); a.tst(TMP, imm(F_FORCE_GC | F_DISABLE_GC)); a.b_ne(gc_after_bif_call); /* Test if binary heap size should trigger GC. */ - a.ldr(TMP, arm::Mem(c_p, offsetof(Process, bin_vheap_sz))); - a.ldr(VAR, arm::Mem(c_p, offsetof(Process, off_heap.overhead))); + safe_ldr(TMP, arm::Mem(c_p, offsetof(Process, bin_vheap_sz))); + safe_ldr(VAR, arm::Mem(c_p, offsetof(Process, off_heap.overhead))); a.cmp(VAR, TMP); a.b_ls(check_fragments); a.b(gc_after_bif_call); @@ -470,7 +470,7 @@ void BeamGlobalAssembler::emit_call_light_bif_shared() { /* Test if heap fragment size is larger than remaining heap size. */ a.bind(check_fragments); a.sub(TMP, E, HTOP); - a.ldr(VAR, arm::Mem(c_p, offsetof(Process, mbuf_sz))); + safe_ldr(VAR, arm::Mem(c_p, offsetof(Process, mbuf_sz))); a.lsl(VAR, VAR, imm(3)); a.cmp(TMP, VAR); a.b_lo(gc_after_bif_call); @@ -487,13 +487,13 @@ void BeamGlobalAssembler::emit_call_light_bif_shared() { a.bind(trap); { - a.ldr(TMP, arm::Mem(c_p, offsetof(Process, freason))); + safe_ldr(TMP, arm::Mem(c_p, offsetof(Process, freason))); a.cmp(TMP, imm(TRAP)); a.b_ne(error); /* Trap out, preserving our continuation on the Erlang stack. */ emit_enter_erlang_frame(); - a.ldr(ARG3, arm::Mem(c_p, offsetof(Process, i))); + safe_ldr(ARG3, arm::Mem(c_p, offsetof(Process, i))); a.b(labels[context_switch_simplified]); } @@ -541,8 +541,8 @@ void BeamGlobalAssembler::emit_call_light_bif_shared() { { a.ldrb(ARG2, arm::Mem(ARG4, offsetof(Export, info.mfa.arity))); add(ARG4, ARG4, offsetof(Export, info.mfa)); - a.strb(ARG2, arm::Mem(c_p, offsetof(Process, arity))); - a.str(ARG4, arm::Mem(c_p, offsetof(Process, current))); + safe_strb(ARG2, arm::Mem(c_p, offsetof(Process, arity))); + safe_str(ARG4, arm::Mem(c_p, offsetof(Process, current))); /* We'll find our way back through ARG3 (entry address). */ a.b(labels[context_switch_simplified]); @@ -606,27 +606,27 @@ void BeamGlobalAssembler::emit_bif_nif_epilogue(void) { if (erts_alcu_enable_code_atags) { /* See emit_i_test_yield. */ - a.str(a32::lr, arm::Mem(c_p, offsetof(Process, i))); + safe_str(a32::lr, arm::Mem(c_p, offsetof(Process, i))); } a.bx(a32::lr); a.bind(check_trap); - a.ldr(TMP, arm::Mem(c_p, offsetof(Process, freason))); + safe_ldr(TMP, arm::Mem(c_p, offsetof(Process, freason))); a.cmp(TMP, imm(TRAP)); a.b_ne(error); { comment("yield"); comment("test trap to hibernate"); - a.ldr(TMP, arm::Mem(c_p, offsetof(Process, flags))); + safe_ldr(TMP, arm::Mem(c_p, offsetof(Process, flags))); a.tst(TMP, imm(F_HIBERNATE_SCHED)); a.b_eq(trap); comment("do hibernate trap"); mov_imm(VAR, ~F_HIBERNATE_SCHED); a.and_(TMP, TMP, VAR); - a.str(TMP, arm::Mem(c_p, offsetof(Process, flags))); + safe_str(TMP, arm::Mem(c_p, offsetof(Process, flags))); a.b(labels[do_schedule]); } @@ -636,7 +636,7 @@ void BeamGlobalAssembler::emit_bif_nif_epilogue(void) { /* The BIF_TRAP macros all set up c_p->arity and c_p->current, so we * can use a simplified context switch. */ - a.ldr(ARG3, arm::Mem(c_p, offsetof(Process, i))); + safe_ldr(ARG3, arm::Mem(c_p, offsetof(Process, i))); a.b(labels[context_switch_simplified]); } @@ -652,7 +652,7 @@ void BeamGlobalAssembler::emit_bif_nif_epilogue(void) { emit_leave_runtime(); a.mov(ARG2, ARG1); - a.ldr(ARG4, arm::Mem(c_p, offsetof(Process, current))); + safe_ldr(ARG4, arm::Mem(c_p, offsetof(Process, current))); a.b(labels[raise_exception_shared]); } } @@ -671,12 +671,12 @@ void BeamGlobalAssembler::emit_call_bif_shared(void) { * `c_p->arity`. */ emit_enter_runtime_frame(); - a.str(ARG2, arm::Mem(c_p, offsetof(Process, current))); + safe_str(ARG2, arm::Mem(c_p, offsetof(Process, current))); a.ldr(TMP, arm::Mem(ARG2, offsetof(ErtsCodeMFA, arity))); a.mov(VAR, TMP); - a.strb(VAR, arm::Mem(c_p, offsetof(Process, arity))); - a.str(ARG3, arm::Mem(c_p, offsetof(Process, i))); + safe_strb(VAR, arm::Mem(c_p, offsetof(Process, arity))); + safe_str(ARG3, arm::Mem(c_p, offsetof(Process, i))); /* The corresponding leave can be found in the epilogue. */ emit_enter_runtimei points into the trampoline of a ErtsNativeFunc, right after the * `info` structure. */ - a.ldr(ARG3, arm::Mem(c_p, offsetof(Process, i))); + safe_ldr(ARG3, arm::Mem(c_p, offsetof(Process, i))); ERTS_CT_ASSERT(offsetof(ErtsNativeFunc, trampoline.call_bif_nif) == @@ -719,7 +719,7 @@ void BeamGlobalAssembler::emit_dispatch_bif(void) { ssize_t mfa_offset = offsetof(ErtsNativeFunc, trampoline.call_bif_nif) - offsetof(ErtsNativeFunc, trampoline.info.mfa); - a.sub(ARG2, ARG3, imm(mfa_offset)); + sub(ARG2, ARG3, mfa_offset); ssize_t dfunc_offset = offsetof(ErtsNativeFunc, trampoline.dfunc) - offsetof(ErtsNativeFunc, trampoline.call_bif_nif); @@ -747,7 +747,7 @@ void BeamModuleAssembler::emit_call_bif_mfa(const ArgAtom &M, func = (BeamInstr)bif_table[e->bif_number].f; a.adr(ARG3, current_label); - a.sub(ARG2, ARG3, imm(sizeof(ErtsCodeMFA))); + sub(ARG2, ARG3, sizeof(ErtsCodeMFA)); comment("HBIF: %T:%T/%d", e->info.mfa.module, e->info.mfa.function, @@ -759,7 +759,7 @@ void BeamModuleAssembler::emit_call_bif_mfa(const ArgAtom &M, void BeamGlobalAssembler::emit_call_nif_early() { a.mov(ARG2, a32::lr); - a.sub(ARG2, ARG2, imm(BEAM_ASM_FUNC_PROLOGUE_SIZE + sizeof(ErtsCodeInfo))); + sub(ARG2, ARG2, BEAM_ASM_FUNC_PROLOGUE_SIZE + sizeof(ErtsCodeInfo)); emit_enter_runtime(); @@ -820,7 +820,7 @@ void BeamGlobalAssembler::emit_dispatch_nif(void) { * * ErtsNativeFunc follows the call_nif layout, so we don't need to do * anything beyond loading the address. */ - a.ldr(ARG3, arm::Mem(c_p, offsetof(Process, i))); + safe_ldr(ARG3, arm::Mem(c_p, offsetof(Process, i))); a.b(labels[call_nif_shared]); } @@ -829,7 +829,7 @@ void BeamGlobalAssembler::emit_call_nif_yield_helper() { if (erts_alcu_enable_code_atags) { /* See emit_i_test_yield. */ - a.str(ARG3, arm::Mem(c_p, offsetof(Process, i))); + safe_str(ARG3, arm::Mem(c_p, offsetof(Process, i))); } a.subs(FCALLS, FCALLS, imm(1)); @@ -842,14 +842,14 @@ void BeamGlobalAssembler::emit_call_nif_yield_helper() { int arity_offset = offsetof(ErtsCodeMFA, arity) - mfa_offset; a.ldrb(TMP, arm::Mem(ARG3, arity_offset)); - a.strb(TMP, arm::Mem(c_p, offsetof(Process, arity))); + safe_strb(TMP, arm::Mem(c_p, offsetof(Process, arity))); - a.sub(TMP, ARG3, imm(mfa_offset)); - a.str(TMP, arm::Mem(c_p, offsetof(Process, current))); + sub(TMP, ARG3, mfa_offset); + safe_str(TMP, arm::Mem(c_p, offsetof(Process, current))); /* Yield to `dispatch` rather than `entry` to avoid pushing too many * frames to the stack. See `emit_call_nif` for details. */ - a.add(ARG3, ARG3, imm(BEAM_ASM_NFUNC_SIZE + sizeof(UWord[3]))); + add(ARG3, ARG3, BEAM_ASM_NFUNC_SIZE + sizeof(UWord[3])); a.b(labels[context_switch_simplified]); } } diff --git a/erts/emulator/beam/jit/arm/32/instr_common.cpp b/erts/emulator/beam/jit/arm/32/instr_common.cpp index f90afca35d3b..d16b2c0d034f 100644 --- a/erts/emulator/beam/jit/arm/32/instr_common.cpp +++ b/erts/emulator/beam/jit/arm/32/instr_common.cpp @@ -72,15 +72,15 @@ using namespace asmjit; void BeamModuleAssembler::emit_error(int reason) { mov_imm(TMP, reason); - a.str(TMP, arm::Mem(c_p, offsetof(Process, freason))); + safe_str(TMP, arm::Mem(c_p, offsetof(Process, freason))); emit_raise_exception(); } void BeamModuleAssembler::emit_error(int reason, const ArgSource &Src) { mov_imm(TMP, reason); - a.str(TMP, arm::Mem(c_p, offsetof(Process, freason))); + safe_str(TMP, arm::Mem(c_p, offsetof(Process, freason))); auto src = load_source(Src, TMP); - a.str(src.reg, arm::Mem(c_p, offsetof(Process, fvalue))); + safe_str(src.reg, arm::Mem(c_p, offsetof(Process, fvalue))); emit_raise_exception(); } @@ -249,9 +249,9 @@ void BeamModuleAssembler::emit_normal_exit() { emit_proc_lc_unrequire(); mov_imm(TMP, EXC_NORMAL); - a.str(TMP, arm::Mem(c_p, offsetof(Process, freason))); + safe_str(TMP, arm::Mem(c_p, offsetof(Process, freason))); mov_imm(TMP, 0); - a.strb(TMP, arm::Mem(c_p, offsetof(Process, arity))); + safe_strb(TMP, arm::Mem(c_p, offsetof(Process, arity))); a.mov(ARG1, c_p); mov_imm(ARG2, am_normal); runtime_call<2>(erts_do_exit_process); @@ -945,7 +945,7 @@ void BeamModuleAssembler::emit_update_record_in_place( emit_is_not_boxed(update, value.reg); } - a.ldr(ARG4, arm::Mem(c_p, offsetof(Process, high_water))); + safe_ldr(ARG4, arm::Mem(c_p, offsetof(Process, high_water))); a.cmp(untagged_src, HTOP); a.b_hs(copy); a.cmp(untagged_src, ARG4); @@ -2369,9 +2369,9 @@ void BeamModuleAssembler::emit_badrecord(const ArgSource &Src) { void BeamModuleAssembler::emit_catch(const ArgYRegister &Y, const ArgCatch &Handler) { - a.ldr(TMP, arm::Mem(c_p, offsetof(Process, catches))); + safe_ldr(TMP, arm::Mem(c_p, offsetof(Process, catches))); a.add(TMP, TMP, imm(1)); - a.str(TMP, arm::Mem(c_p, offsetof(Process, catches))); + safe_str(TMP, arm::Mem(c_p, offsetof(Process, catches))); mov_arg(Y, Handler); } @@ -2456,17 +2456,17 @@ void BeamModuleAssembler::emit_catch_end(const ArgYRegister &CatchTag) { } void BeamModuleAssembler::emit_try_end(const ArgYRegister &CatchTag) { - a.ldr(TMP, arm::Mem(c_p, offsetof(Process, catches))); + safe_ldr(TMP, arm::Mem(c_p, offsetof(Process, catches))); a.sub(TMP, TMP, imm(1)); - a.str(TMP, arm::Mem(c_p, offsetof(Process, catches))); + safe_str(TMP, arm::Mem(c_p, offsetof(Process, catches))); mov_imm(TMP, NIL); a.str(TMP, getArgRef(CatchTag)); } void BeamModuleAssembler::emit_try_end_deallocate(const ArgWord &Deallocate) { - a.ldr(TMP, arm::Mem(c_p, offsetof(Process, catches))); + safe_ldr(TMP, arm::Mem(c_p, offsetof(Process, catches))); a.sub(TMP, TMP, imm(1)); - a.str(TMP, arm::Mem(c_p, offsetof(Process, catches))); + safe_str(TMP, arm::Mem(c_p, offsetof(Process, catches))); if (Deallocate.get() > 0) { add(E, E, Deallocate.get() * sizeof(Eterm)); } @@ -2476,9 +2476,9 @@ void BeamModuleAssembler::emit_try_end_move_deallocate( const ArgSource &Src, const ArgRegister &Dst, const ArgWord &Deallocate) { - a.ldr(TMP, arm::Mem(c_p, offsetof(Process, catches))); + safe_ldr(TMP, arm::Mem(c_p, offsetof(Process, catches))); a.sub(TMP, TMP, imm(1)); - a.str(TMP, arm::Mem(c_p, offsetof(Process, catches))); + safe_str(TMP, arm::Mem(c_p, offsetof(Process, catches))); mov_arg(Dst, Src); if (Deallocate.get() > 0) { add(E, E, Deallocate.get() * sizeof(Eterm)); @@ -2491,11 +2491,11 @@ void BeamModuleAssembler::emit_try_case(const ArgYRegister &CatchTag) { * x1 = error reason/thrown value * x2 = raw stacktrace * x3 = class */ - a.ldr(TMP, arm::Mem(c_p, offsetof(Process, catches))); + safe_ldr(TMP, arm::Mem(c_p, offsetof(Process, catches))); a.ldr(ARG1, getXRef(3)); a.str(ARG1, getXRef(0)); a.sub(TMP, TMP, imm(1)); - a.str(TMP, arm::Mem(c_p, offsetof(Process, catches))); + safe_str(TMP, arm::Mem(c_p, offsetof(Process, catches))); /* The try_tag in the Y slot in the stack frame has already been * cleared. */ @@ -2505,8 +2505,8 @@ void BeamModuleAssembler::emit_try_case(const ArgYRegister &CatchTag) { Label ok = a.newLabel(); Label bad = a.newLabel(); comment("Start of assertion code"); - a.ldr(ARG1, arm::Mem(c_p, offsetof(Process, fvalue))); - a.ldr(ARG2, arm::Mem(c_p, offsetof(Process, ftrace))); + safe_ldr(ARG1, arm::Mem(c_p, offsetof(Process, fvalue))); + safe_ldr(ARG2, arm::Mem(c_p, offsetof(Process, ftrace))); mov_imm(TMP, NIL); a.cmp(ARG1, TMP); @@ -2527,8 +2527,8 @@ void BeamModuleAssembler::emit_try_case_end(const ArgSource &Src) { } void BeamGlobalAssembler::emit_raise_shared() { - a.str(ARG1, arm::Mem(c_p, offsetof(Process, fvalue))); - a.str(ARG2, arm::Mem(c_p, offsetof(Process, ftrace))); + safe_str(ARG1, arm::Mem(c_p, offsetof(Process, fvalue))); + safe_str(ARG2, arm::Mem(c_p, offsetof(Process, ftrace))); emit_enter_runtime(); a.mov(ARG1, c_p); @@ -2595,12 +2595,12 @@ void BeamModuleAssembler::emit_raw_raise() { /* ARG3 = current_label */ void BeamGlobalAssembler::emit_i_test_yield_shared() { - a.sub(ARG2, ARG3, imm(sizeof(ErtsCodeMFA))); - a.add(ARG3, ARG3, imm(TEST_YIELD_RETURN_OFFSET)); + sub(ARG2, ARG3, sizeof(ErtsCodeMFA)); + add(ARG3, ARG3, TEST_YIELD_RETURN_OFFSET); - a.str(ARG2, arm::Mem(c_p, offsetof(Process, current))); + safe_str(ARG2, arm::Mem(c_p, offsetof(Process, current))); a.ldr(ARG2, arm::Mem(ARG2, offsetof(ErtsCodeMFA, arity))); - a.strb(ARG2, arm::Mem(c_p, offsetof(Process, arity))); + safe_strb(ARG2, arm::Mem(c_p, offsetof(Process, arity))); a.b(labels[context_switch_simplified]); } @@ -2619,7 +2619,7 @@ void BeamModuleAssembler::emit_i_test_yield() { * impact on performance but there's little point in doing this unless * the user has requested it -- it's an undocumented feature for * now. */ - a.str(ARG3, arm::Mem(c_p, offsetof(Process, i))); + safe_str(ARG3, arm::Mem(c_p, offsetof(Process, i))); } a.subs(FCALLS, FCALLS, imm(1)); diff --git a/erts/emulator/beam/jit/arm/32/instr_fun.cpp b/erts/emulator/beam/jit/arm/32/instr_fun.cpp index cb048f96cff8..8ef66364efd4 100644 --- a/erts/emulator/beam/jit/arm/32/instr_fun.cpp +++ b/erts/emulator/beam/jit/arm/32/instr_fun.cpp @@ -187,7 +187,7 @@ void BeamModuleAssembler::emit_i_lambda_trampoline(const ArgLambda &Lambda, ssize_t i; emit_ptr_val(ARG4, ARG4); - a.add(ARG4, ARG4, imm(env_offset)); + add(ARG4, ARG4, env_offset); for (i = fun_arity; i < total_arity - 1; i += 2) { auto first = init_destination(ArgXRegister(i), VAR); diff --git a/erts/emulator/beam/jit/arm/32/instr_msg.cpp b/erts/emulator/beam/jit/arm/32/instr_msg.cpp index e4dcd246a576..f00c3ceed61c 100644 --- a/erts/emulator/beam/jit/arm/32/instr_msg.cpp +++ b/erts/emulator/beam/jit/arm/32/instr_msg.cpp @@ -112,7 +112,7 @@ void BeamGlobalAssembler::emit_i_loop_rec_shared() { a.ldr(TMP, flags); a.orr(TMP, TMP, imm(F_DELAY_GC)); a.str(TMP, flags); - a.str(ARG1, arm::Mem(c_p, offsetof(Process, i))); + safe_str(ARG1, arm::Mem(c_p, offsetof(Process, i))); a.str(ARG2, await_addr); a.bind(restart); @@ -126,7 +126,7 @@ void BeamGlobalAssembler::emit_i_loop_rec_shared() { comment("Peek next message"); a.bind(peek_message); { - a.ldr(TMP, arm::Mem(c_p, offsetof(Process, sig_qs.save))); + safe_ldr(TMP, arm::Mem(c_p, offsetof(Process, sig_qs.save))); a.ldr(ARG1, arm::Mem(TMP)); emit_branch_if_ne(ARG1, 0, check_is_distributed); comment("Inner queue empty, fetch more from outer/middle queues"); @@ -188,8 +188,8 @@ void BeamGlobalAssembler::emit_i_loop_rec_shared() { a.bic(TMP, TMP, imm(F_DELAY_GC)); a.str(TMP, flags); mov_imm(TMP, 0); - a.strb(TMP, arm::Mem(c_p, offsetof(Process, arity))); - a.str(TMP, arm::Mem(c_p, offsetof(Process, current))); + safe_strb(TMP, arm::Mem(c_p, offsetof(Process, arity))); + safe_str(TMP, arm::Mem(c_p, offsetof(Process, current))); a.b(labels[do_schedule]); } diff --git a/erts/emulator/beam/jit/arm/32/instr_select.cpp b/erts/emulator/beam/jit/arm/32/instr_select.cpp index 38f203ed32a6..630b482d9638 100644 --- a/erts/emulator/beam/jit/arm/32/instr_select.cpp +++ b/erts/emulator/beam/jit/arm/32/instr_select.cpp @@ -26,11 +26,17 @@ using namespace asmjit; template -static constexpr bool isInt13(T value) { +static bool canEncodeAArch32Imm(T value) { + uint32_t encoded; + return arm::Utils::encodeAArch32Imm(value, &encoded); +} + +template +static bool isInt13(T value) { typedef typename std::make_unsigned::type U; typedef typename std::make_signed::type S; - return Support::isUInt12(U(value)) || Support::isUInt12(-S(value)); + return canEncodeAArch32Imm(U(value)) || canEncodeAArch32Imm(-S(value)); } /* The `cmp`/`cmn` instructions in AArch64 only accept 12-bit unsigned immediate @@ -424,7 +430,7 @@ void BeamModuleAssembler::emit_i_jump_on_val(const ArgSource &Src, a.asr(TMP, TMP, imm(_TAG_IMMED1_SIZE)); if (Base.get() != 0) { - if (Support::isUInt12((Sint)Base.get())) { + if (canEncodeAArch32Imm(Base.get())) { a.sub(TMP, TMP, imm(Base.get())); } else { mov_imm(VAR, Base.get()); diff --git a/erts/emulator/beam/jit/arm/32/instr_trace.cpp b/erts/emulator/beam/jit/arm/32/instr_trace.cpp index 18c31616ecc4..7518573bdd40 100644 --- a/erts/emulator/beam/jit/arm/32/instr_trace.cpp +++ b/erts/emulator/beam/jit/arm/32/instr_trace.cpp @@ -68,7 +68,7 @@ void BeamGlobalAssembler::emit_generic_bp_local() { /* Our actual return address is valid (and word-aligned), but it points * just after the trampoline word so we'll need to skip that to find our * ErtsCodeInfo. */ - a.sub(ARG2, ARG2, imm(BEAM_ASM_FUNC_PROLOGUE_SIZE + sizeof(ErtsCodeInfo))); + sub(ARG2, ARG2, BEAM_ASM_FUNC_PROLOGUE_SIZE + sizeof(ErtsCodeInfo)); emit_enter_runtime_frame(); emit_enter_runtime(); @@ -97,7 +97,7 @@ void BeamGlobalAssembler::emit_debug_bp() { /* Read and adjust the return address we saved in generic_bp_local. */ a.ldr(ARG2, TMP_MEM1q); - a.sub(ARG2, ARG2, imm(BEAM_ASM_FUNC_PROLOGUE_SIZE + sizeof(ErtsCodeMFA))); + sub(ARG2, ARG2, BEAM_ASM_FUNC_PROLOGUE_SIZE + sizeof(ErtsCodeMFA)); emit_enter_runtime(); diff --git a/erts/emulator/beam/jit/arm/32/process_main.cpp b/erts/emulator/beam/jit/arm/32/process_main.cpp index 58157b166925..0e25a540004c 100644 --- a/erts/emulator/beam/jit/arm/32/process_main.cpp +++ b/erts/emulator/beam/jit/arm/32/process_main.cpp @@ -80,7 +80,7 @@ void BeamGlobalAssembler::emit_process_main() { const arm::Mem start_time = arm::Mem(ARG4, relative_start_t_offset); auto setup_start_time_base = [&]() { a.mov(ARG4, scheduler_registers); - a.add(ARG4, ARG4, imm(start_t_i_offset)); + add(ARG4, ARG4, start_t_i_offset); }; auto load_start_time = [&](const a32::Gp &dst) { setup_start_time_base(); @@ -125,7 +125,7 @@ void BeamGlobalAssembler::emit_process_main() { a.bind(do_schedule_local); { /* Figure out reds_used. def_arg_reg[5] = REDS_IN */ - a.ldr(TMP, arm::Mem(c_p, offsetof(Process, def_arg_reg[5]))); + safe_ldr(TMP, arm::Mem(c_p, offsetof(Process, def_arg_reg[5]))); a.sub(ARG3, TMP, FCALLS); a.b(schedule_next); } @@ -155,8 +155,8 @@ void BeamGlobalAssembler::emit_process_main() { a.bind(check_i); #endif - a.str(ARG3, arm::Mem(c_p, offsetof(Process, i))); - a.ldr(TMP, arm::Mem(c_p, offsetof(Process, state.value))); + safe_str(ARG3, arm::Mem(c_p, offsetof(Process, i))); + safe_ldr(TMP, arm::Mem(c_p, offsetof(Process, state.value))); a.tst(TMP, imm(ERTS_PSFLG_EXITING)); a.b_eq(not_exiting); @@ -164,17 +164,17 @@ void BeamGlobalAssembler::emit_process_main() { comment("Process exiting"); a.adr(TMP, labels[process_exit]); - a.str(TMP, arm::Mem(c_p, offsetof(Process, i))); + safe_str(TMP, arm::Mem(c_p, offsetof(Process, i))); mov_imm(TMP, 0); - a.strb(TMP, arm::Mem(c_p, offsetof(Process, arity))); - a.str(TMP, arm::Mem(c_p, offsetof(Process, current))); + safe_strb(TMP, arm::Mem(c_p, offsetof(Process, arity))); + safe_str(TMP, arm::Mem(c_p, offsetof(Process, current))); a.b(do_schedule_local); } a.bind(not_exiting); /* Figure out reds_used. def_arg_reg[5] = REDS_IN */ - a.ldr(TMP, arm::Mem(c_p, offsetof(Process, def_arg_reg[5]))); + safe_ldr(TMP, arm::Mem(c_p, offsetof(Process, def_arg_reg[5]))); a.sub(FCALLS, TMP, FCALLS); comment("Copy out X registers"); @@ -242,7 +242,7 @@ void BeamGlobalAssembler::emit_process_main() { /* Enable long schedule test */ runtime_call<0>(erts_timestamp_millis); store_start_time(ARG1); - a.ldr(TMP, arm::Mem(c_p, offsetof(Process, i))); + safe_ldr(TMP, arm::Mem(c_p, offsetof(Process, i))); store_start_time_i(TMP); } @@ -255,8 +255,8 @@ void BeamGlobalAssembler::emit_process_main() { runtime_call<2>(copy_in_registers); /* Setup reduction counting */ - a.ldr(FCALLS, arm::Mem(c_p, offsetof(Process, fcalls))); - a.str(FCALLS, arm::Mem(c_p, offsetof(Process, def_arg_reg[5]))); + safe_ldr(FCALLS, arm::Mem(c_p, offsetof(Process, fcalls))); + safe_str(FCALLS, arm::Mem(c_p, offsetof(Process, def_arg_reg[5]))); #ifdef DEBUG a.str(FCALLS, a32::Mem(c_p, offsetof(Process, debug_reds_in))); @@ -284,7 +284,7 @@ void BeamGlobalAssembler::emit_process_main() { * * This relies on `op_call_nif_WWW` / `op_call_bif_W` being encoded as * UDF(opcode) followed by UDF(0), which we will never emit. */ - a.ldr(ARG1, arm::Mem(c_p, offsetof(Process, i))); + safe_ldr(ARG1, arm::Mem(c_p, offsetof(Process, i))); a.ldr(TMP, arm::Mem(ARG1)); ERTS_CT_ASSERT((op_call_nif_WWW & 0xFFFF0000) == 0);