From 28b5d223c576e007913c0cbecee16a65ad915a0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= Date: Tue, 16 Jun 2026 06:56:17 +0200 Subject: [PATCH] Make bignum arithmetic count reductions This will make scheduling fairer when doing many arithmetic operations with large integers. --- erts/emulator/beam/emu/arith_instrs.tab | 9 ++ erts/emulator/beam/erl_arith.c | 61 ++++++++-- erts/emulator/beam/global.h | 5 + erts/emulator/beam/jit/arm/instr_arith.cpp | 90 +++++++-------- erts/emulator/beam/jit/x86/instr_arith.cpp | 58 +++++----- erts/emulator/test/big_SUITE.erl | 127 ++++++++++++++++++++- 6 files changed, 263 insertions(+), 87 deletions(-) diff --git a/erts/emulator/beam/emu/arith_instrs.tab b/erts/emulator/beam/emu/arith_instrs.tab index 595f86dc213c..236a0dd65cda 100644 --- a/erts/emulator/beam/emu/arith_instrs.tab +++ b/erts/emulator/beam/emu/arith_instrs.tab @@ -28,7 +28,9 @@ OUTLINED_ARITH_1(Fail, Name, BIF, Op1,Dst) { Eterm* orig_stop = E; #endif DEBUG_SWAPOUT; + c_p->fcalls = FCALLS; result = erts_$Name (c_p, $Op1); + FCALLS = c_p->fcalls; DEBUG_SWAPIN; ASSERT(orig_htop == HTOP && orig_stop == E); ERTS_HOLE_CHECK(c_p); @@ -46,7 +48,9 @@ OUTLINED_ARITH_2(Fail, Name, BIF, Op1, Op2, Dst) { Eterm* orig_stop = E; #endif DEBUG_SWAPOUT; + c_p->fcalls = FCALLS; result = erts_$Name (c_p, $Op1, $Op2); + FCALLS = c_p->fcalls; DEBUG_SWAPIN; ASSERT(orig_htop == HTOP && orig_stop == E); ERTS_HOLE_CHECK(c_p); @@ -487,6 +491,9 @@ shift.execute(Fail, Dst) { } ERTS_HOLE_CHECK(c_p); $Dst = Op1; + c_p->fcalls = FCALLS; + BUMP_REDS(c_p, big_need_size / ERTS_ARITH_WORDS_PER_REDUCTION + 1); + FCALLS = c_p->fcalls; $NEXT0(); } } @@ -514,7 +521,9 @@ i_int_bnot(Fail, Src, Dst) { if (ERTS_LIKELY(is_small(bnot_val))) { result = make_small(~signed_val(bnot_val)); } else { + c_p->fcalls = FCALLS; result = erts_bnot(c_p, bnot_val); + FCALLS = c_p->fcalls; ERTS_HOLE_CHECK(c_p); if (ERTS_UNLIKELY(is_non_value(result))) { $BIF_ERROR_ARITY_1($Fail, BIF_bnot_1, bnot_val); diff --git a/erts/emulator/beam/erl_arith.c b/erts/emulator/beam/erl_arith.c index dcc4f06171b2..33c65f34796c 100644 --- a/erts/emulator/beam/erl_arith.c +++ b/erts/emulator/beam/erl_arith.c @@ -227,12 +227,13 @@ erts_shift(Process* p, Eterm arg1, Eterm arg2, int right) maybe_shrink(p, bigp, arg1, need); if (is_nil(arg1)) { /* - * This result must have been only slight larger + * This result must have been only slightly larger * than allowed since it wasn't caught by the * previous test. */ BIF_ERROR(p, SYSTEM_LIMIT); } + BUMP_REDS(p, need / ERTS_ARITH_WORDS_PER_REDUCTION + 1); BIF_RET(arg1); } else if (is_big(arg1)) { if (i == 0) { @@ -298,6 +299,7 @@ BIF_RETTYPE bnot_1(BIF_ALIST_1) if (is_nil(ret)) { BIF_ERROR(BIF_P, SYSTEM_LIMIT); } + BUMP_REDS(BIF_P, need / ERTS_ARITH_WORDS_PER_REDUCTION + 1); } else { BIF_ERROR(BIF_P, BADARITH); } @@ -399,6 +401,7 @@ erts_mixed_plus(Process* p, Eterm arg1, Eterm arg2) p->freason = SYSTEM_LIMIT; return THE_NON_VALUE; } + BUMP_REDS(p, need_heap / ERTS_ARITH_WORDS_PER_REDUCTION + 1); return res; case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE): if (big_to_double(arg1, &f1.fd) < 0) { @@ -502,6 +505,7 @@ erts_unary_minus(Process* p, Eterm arg) res = big_minus(zero, arg, hp); maybe_shrink(p, hp, res, need_heap); ASSERT(is_not_nil(res)); + BUMP_REDS(p, need_heap / ERTS_ARITH_WORDS_PER_REDUCTION + 1); return res; } case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE): @@ -599,6 +603,7 @@ erts_mixed_minus(Process* p, Eterm arg1, Eterm arg2) p->freason = SYSTEM_LIMIT; return THE_NON_VALUE; } + BUMP_REDS(p, sz); return res; default: goto badarith; @@ -804,10 +809,15 @@ erts_mixed_times(Process* p, Eterm arg1, Eterm arg2) maybe_shrink(p, hp, res, need_heap); if (is_nil(res)) { - p->freason = SYSTEM_LIMIT; - return THE_NON_VALUE; - } - return res; + p->freason = SYSTEM_LIMIT; + return THE_NON_VALUE; + } + + /* Conservative estimate of effort, assuming + * schoolbook multiplication. */ + BUMP_REDS(p, big_size(arg1) * big_size(arg2) / + ERTS_ARITH_WORDS_PER_REDUCTION + 1); + return res; case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE): if (big_to_double(arg1, &f1.fd) < 0) { goto badarith; @@ -938,6 +948,10 @@ erts_mul_add(Process* p, Eterm arg1, Eterm arg2, Eterm arg3, Eterm* pp) p->freason = SYSTEM_LIMIT; return THE_NON_VALUE; } + + /* Conservative estimate of effort, + * assuming schoolbook multiplication. */ + BUMP_REDS(p, sz1 * sz2 / ERTS_ARITH_WORDS_PER_REDUCTION + 1); return res; } } @@ -1160,7 +1174,7 @@ int erts_int_div_rem(Process* p, Eterm arg1, Eterm arg2, Eterm *q, Eterm *r) SMALL_ONE : SMALL_MINUS_ONE; remainder = SMALL_ZERO; } else { - int lhs_size, rhs_size; + unsigned lhs_size, rhs_size; Uint q_need, r_need; Eterm *q_hp, *r_hp; @@ -1184,6 +1198,12 @@ int erts_int_div_rem(Process* p, Eterm arg1, Eterm arg2, Eterm *q, Eterm *r) ASSERT(q_need + r_need >= size_object(quotient) + size_object(remainder)); + /* Conservative estimate of effort, assuming shoolbook long + * division. */ + ASSERT(lhs_size >= rhs_size); + BUMP_REDS(p, (lhs_size - rhs_size + 1) * rhs_size / + ERTS_ARITH_WORDS_PER_REDUCTION + 1); + div_rem_shrink(p, q_hp, quotient, r_hp, remainder); } @@ -1227,11 +1247,12 @@ erts_int_div(Process* p, Eterm arg1, Eterm arg2) SMALL_ONE : SMALL_MINUS_ONE; } else { Eterm* hp; - int i = big_size(arg1); + unsigned lhs_size = big_size(arg1); + unsigned rhs_size = big_size(arg2); Uint need; - ires = big_size(arg2); - need = BIG_NEED_SIZE(i-ires+1) + BIG_NEED_SIZE(i); + need = BIG_NEED_SIZE(lhs_size-rhs_size+1) + + BIG_NEED_SIZE(lhs_size); hp = HeapFragOnlyAlloc(p, need); arg1 = big_div(arg1, arg2, hp); maybe_shrink(p, hp, arg1, need); @@ -1239,6 +1260,12 @@ erts_int_div(Process* p, Eterm arg1, Eterm arg2) p->freason = SYSTEM_LIMIT; return THE_NON_VALUE; } + + /* Conservative estimate of effort, assuming shoolbook + * long division. */ + ASSERT(lhs_size >= rhs_size); + BUMP_REDS(p, (lhs_size - rhs_size + 1) * rhs_size / + ERTS_ARITH_WORDS_PER_REDUCTION + 1); } return arg1; default: @@ -1282,7 +1309,9 @@ erts_int_rem(Process* p, Eterm arg1, Eterm arg2) if (ires == 0) { arg1 = SMALL_ZERO; } else if (ires > 0) { - Uint need = BIG_NEED_SIZE(big_size(arg1)); + unsigned lhs_size = big_size(arg1); + unsigned rhs_size; + Uint need = BIG_NEED_SIZE(lhs_size); Eterm* hp = HeapFragOnlyAlloc(p, need); arg1 = big_rem(arg1, arg2, hp); @@ -1291,6 +1320,14 @@ erts_int_rem(Process* p, Eterm arg1, Eterm arg2) p->freason = SYSTEM_LIMIT; return THE_NON_VALUE; } + + /* Conservative estimate of effort, assuming shoolbook + * long division. */ + rhs_size = big_size(arg2); + ASSERT(lhs_size >= rhs_size); + BUMP_REDS(p, (lhs_size - rhs_size + 1) * rhs_size / + ERTS_ARITH_WORDS_PER_REDUCTION + 1); + } return arg1; default: @@ -1323,6 +1360,7 @@ Eterm erts_band(Process* p, Eterm arg1, Eterm arg2) arg1 = big_band(arg1, arg2, hp); ASSERT(is_not_nil(arg1)); maybe_shrink(p, hp, arg1, need); + BUMP_REDS(p, need / ERTS_ARITH_WORDS_PER_REDUCTION + 1); return arg1; } @@ -1350,6 +1388,7 @@ Eterm erts_bor(Process* p, Eterm arg1, Eterm arg2) arg1 = big_bor(arg1, arg2, hp); ASSERT(is_not_nil(arg1)); maybe_shrink(p, hp, arg1, need); + BUMP_REDS(p, need / ERTS_ARITH_WORDS_PER_REDUCTION + 1); return arg1; } @@ -1377,6 +1416,7 @@ Eterm erts_bxor(Process* p, Eterm arg1, Eterm arg2) arg1 = big_bxor(arg1, arg2, hp); ASSERT(is_not_nil(arg1)); maybe_shrink(p, hp, arg1, need); + BUMP_REDS(p, need / ERTS_ARITH_WORDS_PER_REDUCTION + 1); return arg1; } @@ -1394,6 +1434,7 @@ Eterm erts_bnot(Process* p, Eterm arg) p->freason = SYSTEM_LIMIT; return THE_NON_VALUE; } + BUMP_REDS(p, need / ERTS_ARITH_WORDS_PER_REDUCTION + 1); } else { p->freason = BADARITH; return THE_NON_VALUE; diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index f918cbfe6b06..c2556955f83f 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1033,6 +1033,11 @@ void erts_print_base64(fmtfn_t to, void *to_arg, const byte* src, Uint size); int erts_set_signal(Eterm signal, Eterm type); /* erl_arith.c */ + +/* The number of bignum words that we assume can be processed in one + * reduction. */ +#define ERTS_ARITH_WORDS_PER_REDUCTION 16 + double erts_get_positive_zero_float(void); /* config.c */ diff --git a/erts/emulator/beam/jit/arm/instr_arith.cpp b/erts/emulator/beam/jit/arm/instr_arith.cpp index 4dcdda1769f3..daf372ebc4a3 100644 --- a/erts/emulator/beam/jit/arm/instr_arith.cpp +++ b/erts/emulator/beam/jit/arm/instr_arith.cpp @@ -188,16 +188,16 @@ void BeamModuleAssembler::emit_i_plus(const ArgLabel &Fail, mov_var(ARG3, rhs); if (Fail.get() != 0) { - emit_enter_runtime(Live.get()); + emit_enter_runtime(Live.get()); a.mov(ARG1, c_p); runtime_call(); - emit_leave_runtime(Live.get()); + emit_leave_runtime(Live.get()); emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown)); } else { - emit_enter_runtime(Live.get()); + emit_enter_runtime(Live.get()); fragment_call(ga->get_plus_body_shared()); - emit_leave_runtime(Live.get()); + emit_leave_runtime(Live.get()); } a.bind(next); @@ -275,16 +275,16 @@ void BeamModuleAssembler::emit_i_unary_minus(const ArgLabel &Fail, mov_var(ARG2, src); if (Fail.get() != 0) { - emit_enter_runtime(Live.get()); + emit_enter_runtime(Live.get()); a.mov(ARG1, c_p); runtime_call(); - emit_leave_runtime(Live.get()); + emit_leave_runtime(Live.get()); emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown)); } else { - emit_enter_runtime(Live.get()); + emit_enter_runtime(Live.get()); fragment_call(ga->get_unary_minus_body_shared()); - emit_leave_runtime(Live.get()); + emit_leave_runtime(Live.get()); } a.bind(next); @@ -376,15 +376,15 @@ void BeamModuleAssembler::emit_i_minus(const ArgLabel &Fail, mov_var(ARG3, rhs); if (Fail.get() != 0) { - emit_enter_runtime(Live.get()); + emit_enter_runtime(Live.get()); a.mov(ARG1, c_p); runtime_call(); - emit_leave_runtime(Live.get()); + emit_leave_runtime(Live.get()); emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown)); } else { - emit_enter_runtime(Live.get()); + emit_enter_runtime(Live.get()); fragment_call(ga->get_minus_body_shared()); - emit_leave_runtime(Live.get()); + emit_leave_runtime(Live.get()); } a.bind(next); @@ -439,7 +439,7 @@ void BeamGlobalAssembler::emit_mul_add_body_shared() { mul_error = a.new_label(), do_error = a.new_label(); emit_enter_runtime_frame(); - emit_enter_runtime(); + emit_enter_runtime(); /* Save original arguments. */ a.stp(ARG2, ARG3, TMP_MEM1q); @@ -452,7 +452,7 @@ void BeamGlobalAssembler::emit_mul_add_body_shared() { runtime_call(); - emit_leave_runtime(); + emit_leave_runtime(); emit_leave_runtime_frame(); emit_branch_if_not_value(ARG1, error); @@ -462,7 +462,7 @@ void BeamGlobalAssembler::emit_mul_add_body_shared() { { runtime_call(); - emit_leave_runtime(); + emit_leave_runtime(); emit_leave_runtime_frame(); emit_branch_if_not_value(ARG1, mul_error); @@ -500,7 +500,7 @@ void BeamGlobalAssembler::emit_mul_add_guard_shared() { a.str(ARG4, TMP_MEM1q); emit_enter_runtime_frame(); - emit_enter_runtime(); + emit_enter_runtime(); a.mov(ARG1, c_p); runtime_call(); @@ -512,7 +512,7 @@ void BeamGlobalAssembler::emit_mul_add_guard_shared() { runtime_call(); a.bind(mul_failed); - emit_leave_runtime(); + emit_leave_runtime(); emit_leave_runtime_frame(); a.ret(a64::x30); @@ -775,7 +775,7 @@ void BeamGlobalAssembler::emit_int_div_rem_guard_shared() { a.bind(generic); { emit_enter_runtime_frame(); - emit_enter_runtime(); + emit_enter_runtime(); a.mov(ARG1, c_p); lea(ARG4, TMP_MEM4q); @@ -783,7 +783,7 @@ void BeamGlobalAssembler::emit_int_div_rem_guard_shared() { runtime_call(); - emit_leave_runtime(); + emit_leave_runtime(); emit_leave_runtime_frame(); a.tst(ARG1, ARG1); @@ -835,7 +835,7 @@ void BeamGlobalAssembler::emit_int_div_rem_body_shared() { a.bind(generic_div); { emit_enter_runtime_frame(); - emit_enter_runtime(); + emit_enter_runtime(); /* Save MFA and original arguments for the error path. */ a.stp(ARG2, ARG3, TMP_MEM1q); @@ -847,7 +847,7 @@ void BeamGlobalAssembler::emit_int_div_rem_body_shared() { runtime_call(); - emit_leave_runtime(); + emit_leave_runtime(); emit_leave_runtime_frame(); a.tst(ARG1, ARG1); @@ -1231,17 +1231,17 @@ void BeamModuleAssembler::emit_i_band(const ArgLabel &Fail, mov_var(ARG3, rhs); if (Fail.get() != 0) { - emit_enter_runtime(Live.get()); + emit_enter_runtime(Live.get()); a.mov(ARG1, c_p); runtime_call(); - emit_leave_runtime(Live.get()); + emit_leave_runtime(Live.get()); emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown)); } else { - emit_enter_runtime(Live.get()); + emit_enter_runtime(Live.get()); fragment_call(ga->get_i_band_body_shared()); - emit_leave_runtime(Live.get()); + emit_leave_runtime(Live.get()); } a.bind(next); @@ -1307,16 +1307,16 @@ void BeamModuleAssembler::emit_i_bor(const ArgLabel &Fail, mov_var(ARG3, rhs); if (Fail.get() != 0) { - emit_enter_runtime(Live.get()); + emit_enter_runtime(Live.get()); a.mov(ARG1, c_p); runtime_call(); - emit_leave_runtime(Live.get()); + emit_leave_runtime(Live.get()); emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown)); } else { - emit_enter_runtime(Live.get()); + emit_enter_runtime(Live.get()); fragment_call(ga->get_i_bor_body_shared()); - emit_leave_runtime(Live.get()); + emit_leave_runtime(Live.get()); } a.bind(next); @@ -1371,15 +1371,15 @@ void BeamModuleAssembler::emit_i_bxor(const ArgLabel &Fail, mov_var(ARG3, rhs); if (Fail.get() != 0) { - emit_enter_runtime(Live.get()); + emit_enter_runtime(Live.get()); a.mov(ARG1, c_p); runtime_call(); - emit_leave_runtime(Live.get()); + emit_leave_runtime(Live.get()); emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown)); } else { - emit_enter_runtime(Live.get()); + emit_enter_runtime(Live.get()); fragment_call(ga->get_i_bxor_body_shared()); - emit_leave_runtime(Live.get()); + emit_leave_runtime(Live.get()); } a.bind(next); @@ -1474,14 +1474,14 @@ void BeamModuleAssembler::emit_i_bnot(const ArgLabel &Fail, } if (Fail.get() != 0) { - emit_enter_runtime(Live.get()); + emit_enter_runtime(Live.get()); fragment_call(ga->get_i_bnot_guard_shared()); - emit_leave_runtime(Live.get()); + emit_leave_runtime(Live.get()); emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown)); } else { - emit_enter_runtime(Live.get()); + emit_enter_runtime(Live.get()); fragment_call(ga->get_i_bnot_body_shared()); - emit_leave_runtime(Live.get()); + emit_leave_runtime(Live.get()); } a.bind(next); @@ -1581,16 +1581,16 @@ void BeamModuleAssembler::emit_i_bsr(const ArgLabel &Fail, mov_arg(ARG3, RHS); if (Fail.get() != 0) { - emit_enter_runtime(Live.get()); + emit_enter_runtime(Live.get()); a.mov(ARG1, c_p); runtime_call(); - emit_leave_runtime(Live.get()); + emit_leave_runtime(Live.get()); emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown)); } else { - emit_enter_runtime(Live.get()); + emit_enter_runtime(Live.get()); fragment_call(ga->get_i_bsr_body_shared()); - emit_leave_runtime(Live.get()); + emit_leave_runtime(Live.get()); } mov_var(dst, ARG1); @@ -1751,16 +1751,16 @@ void BeamModuleAssembler::emit_i_bsl(const ArgLabel &Fail, mov_var(ARG3, rhs); if (Fail.get() != 0) { - emit_enter_runtime(Live.get()); + emit_enter_runtime(Live.get()); a.mov(ARG1, c_p); runtime_call(); - emit_leave_runtime(Live.get()); + emit_leave_runtime(Live.get()); emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown)); } else { - emit_enter_runtime(Live.get()); + emit_enter_runtime(Live.get()); fragment_call(ga->get_i_bsl_body_shared()); - emit_leave_runtime(Live.get()); + emit_leave_runtime(Live.get()); } mov_var(dst, ARG1); diff --git a/erts/emulator/beam/jit/x86/instr_arith.cpp b/erts/emulator/beam/jit/x86/instr_arith.cpp index bb24d80869c6..0d1c1cbb7028 100644 --- a/erts/emulator/beam/jit/x86/instr_arith.cpp +++ b/erts/emulator/beam/jit/x86/instr_arith.cpp @@ -127,7 +127,7 @@ void BeamGlobalAssembler::emit_plus_body_shared() { Label error = a.new_label(); emit_enter_frame(); - emit_enter_runtime(); + emit_enter_runtime(); /* Save original arguments for the error path. */ a.mov(TMP_MEM1q, ARG2); @@ -136,7 +136,7 @@ void BeamGlobalAssembler::emit_plus_body_shared() { a.mov(ARG1, c_p); runtime_call(); - emit_leave_runtime(); + emit_leave_runtime(); emit_leave_frame(); emit_test_the_non_value(RET); @@ -158,13 +158,13 @@ void BeamGlobalAssembler::emit_plus_body_shared() { void BeamGlobalAssembler::emit_plus_guard_shared() { emit_enter_frame(); - emit_enter_runtime(); + emit_enter_runtime(); a.mov(ARG1, c_p); /* ARG2 and ARG3 were set by the caller */ runtime_call(); - emit_leave_runtime(); + emit_leave_runtime(); emit_leave_frame(); /* Set ZF if the addition failed. */ @@ -264,7 +264,7 @@ void BeamGlobalAssembler::emit_minus_body_shared() { Label error = a.new_label(); emit_enter_frame(); - emit_enter_runtime(); + emit_enter_runtime(); /* Save original arguments for the error path. */ a.mov(TMP_MEM1q, ARG2); @@ -273,7 +273,7 @@ void BeamGlobalAssembler::emit_minus_body_shared() { a.mov(ARG1, c_p); runtime_call(); - emit_leave_runtime(); + emit_leave_runtime(); emit_leave_frame(); emit_test_the_non_value(RET); @@ -295,13 +295,13 @@ void BeamGlobalAssembler::emit_minus_body_shared() { void BeamGlobalAssembler::emit_minus_guard_shared() { emit_enter_frame(); - emit_enter_runtime(); + emit_enter_runtime(); a.mov(ARG1, c_p); /* ARG2 and ARG3 were set by the caller */ runtime_call(); - emit_leave_runtime(); + emit_leave_runtime(); emit_leave_frame(); /* Set ZF if the addition failed. */ @@ -397,7 +397,7 @@ void BeamGlobalAssembler::emit_unary_minus_body_shared() { Label error = a.new_label(); emit_enter_frame(); - emit_enter_runtime(); + emit_enter_runtime(); /* Save original arguments for the error path. */ a.mov(TMP_MEM1q, ARG2); @@ -405,7 +405,7 @@ void BeamGlobalAssembler::emit_unary_minus_body_shared() { a.mov(ARG1, c_p); runtime_call(); - emit_leave_runtime(); + emit_leave_runtime(); emit_leave_frame(); emit_test_the_non_value(RET); @@ -425,13 +425,13 @@ void BeamGlobalAssembler::emit_unary_minus_body_shared() { void BeamGlobalAssembler::emit_unary_minus_guard_shared() { emit_enter_frame(); - emit_enter_runtime(); + emit_enter_runtime(); a.mov(ARG1, c_p); /* ARG2 was set by the caller */ runtime_call(); - emit_leave_runtime(); + emit_leave_runtime(); emit_leave_frame(); /* Set ZF if the negation failed. */ @@ -538,7 +538,7 @@ void BeamGlobalAssembler::emit_int_div_rem_guard_shared() { a.bind(generic); { - emit_enter_runtime(); + emit_enter_runtime(); a.mov(ARG2, ARG1); a.mov(ARG3, ARG4); @@ -548,7 +548,7 @@ void BeamGlobalAssembler::emit_int_div_rem_guard_shared() { runtime_call(); - emit_leave_runtime(); + emit_leave_runtime(); /* erts_int_div returns 0 on failure and 1 on success. */ a.test(RETd, RETd); @@ -620,7 +620,7 @@ void BeamGlobalAssembler::emit_int_div_rem_body_shared() { a.bind(generic_div); { - emit_enter_runtime(); + emit_enter_runtime(); /* Save MFA and original arguments for the error path. */ a.mov(TMP_MEM1q, ARG1); @@ -635,7 +635,7 @@ void BeamGlobalAssembler::emit_int_div_rem_body_shared() { runtime_call(); - emit_leave_runtime(); + emit_leave_runtime(); emit_leave_frame(); /* erts_int_div returns 0 on failure and 1 on success. */ @@ -916,7 +916,7 @@ void BeamGlobalAssembler::emit_mul_add_guard_shared() { Label done = a.new_label(); emit_enter_frame(); - emit_enter_runtime(); + emit_enter_runtime(); a.mov(TMP_MEM1q, ARG4); @@ -933,7 +933,7 @@ void BeamGlobalAssembler::emit_mul_add_guard_shared() { runtime_call(); a.bind(done); - emit_leave_runtime(); + emit_leave_runtime(); emit_leave_frame(); emit_test_the_non_value(RET); /* Sets ZF for use in caller */ @@ -951,7 +951,7 @@ void BeamGlobalAssembler::emit_mul_add_body_shared() { mul_error = a.new_label(), do_error = a.new_label(); emit_enter_frame(); - emit_enter_runtime(); + emit_enter_runtime(); /* Save original arguments for the error path. */ a.mov(TMP_MEM1q, ARG2); @@ -965,7 +965,7 @@ void BeamGlobalAssembler::emit_mul_add_body_shared() { runtime_call(); - emit_leave_runtime(); + emit_leave_runtime(); emit_leave_frame(); emit_test_the_non_value(RET); @@ -977,7 +977,7 @@ void BeamGlobalAssembler::emit_mul_add_body_shared() { { runtime_call(); - emit_leave_runtime(); + emit_leave_runtime(); emit_leave_frame(); emit_test_the_non_value(RET); @@ -1217,14 +1217,14 @@ void BeamModuleAssembler::emit_i_mul_add(const ArgLabel &Fail, template void BeamGlobalAssembler::emit_bitwise_fallback_guard() { emit_enter_frame(); - emit_enter_runtime(); + emit_enter_runtime(); a.mov(ARG1, c_p); /* ARG2 is already set to LHS */ a.mov(ARG3, RET); runtime_call(); - emit_leave_runtime(); + emit_leave_runtime(); emit_leave_frame(); emit_test_the_non_value(RET); @@ -1239,7 +1239,7 @@ void BeamGlobalAssembler::emit_bitwise_fallback_body(const ErtsCodeMFA *mfa) { Label error = a.new_label(); emit_enter_frame(); - emit_enter_runtime(); + emit_enter_runtime(); /* Save original arguments for the error path. */ a.mov(TMP_MEM1q, ARG2); @@ -1250,7 +1250,7 @@ void BeamGlobalAssembler::emit_bitwise_fallback_body(const ErtsCodeMFA *mfa) { a.mov(ARG3, RET); runtime_call(); - emit_leave_runtime(); + emit_leave_runtime(); emit_leave_frame(); emit_test_the_non_value(RET); @@ -1479,13 +1479,13 @@ void BeamGlobalAssembler::emit_i_bnot_guard_shared() { /* Undo the speculative inversion in module code */ a.xor_(RET, imm(~_TAG_IMMED1_MASK)); - emit_enter_runtime(); + emit_enter_runtime(); a.mov(ARG1, c_p); a.mov(ARG2, RET); runtime_call(); - emit_leave_runtime(); + emit_leave_runtime(); emit_leave_frame(); emit_test_the_non_value(RET); @@ -1505,7 +1505,7 @@ void BeamGlobalAssembler::emit_i_bnot_body_shared() { /* Undo the speculative inversion in module code */ a.xor_(RET, imm(~_TAG_IMMED1_MASK)); - emit_enter_runtime(); + emit_enter_runtime(); /* Save original arguments for the error path. */ a.mov(TMP_MEM1q, RET); @@ -1514,7 +1514,7 @@ void BeamGlobalAssembler::emit_i_bnot_body_shared() { a.mov(ARG2, RET); runtime_call(); - emit_leave_runtime(); + emit_leave_runtime(); emit_leave_frame(); emit_test_the_non_value(RET); diff --git a/erts/emulator/test/big_SUITE.erl b/erts/emulator/test/big_SUITE.erl index 935950e0790f..4101cba26161 100644 --- a/erts/emulator/test/big_SUITE.erl +++ b/erts/emulator/test/big_SUITE.erl @@ -30,7 +30,7 @@ big_float_1/1, big_float_2/1, bxor_2pow/1, band_2pow/1, shift_limit_1/1, powmod/1, system_limit/1, toobig/1, otp_6692/1, - properties/1]). + properties/1, reductions/1]). %% Internal exports. -export([eval/1]). @@ -52,7 +52,7 @@ all() -> {group, big_float}, shift_limit_1, bxor_2pow, band_2pow, powmod, system_limit, toobig, otp_6692, - properties]. + properties, reductions]. groups() -> [{big_float, [], [big_float_1, big_float_2]}]. @@ -214,7 +214,12 @@ eval_op('band', A, B) -> A band B; eval_op('bor', A, B) -> A bor B; eval_op('bxor', A, B) -> A bxor B; eval_op('bsl', A, B) -> A bsl B; -eval_op('bsr', A, B) -> A bsr B. +eval_op('bsr', A, B) -> A bsr B; +eval_op(madd, A, B) -> A * B + 1. + +eval_op_guard('-', A, Res) when Res =:= -A -> ok; +eval_op_guard('+', A, Res) when Res =:= +A -> ok; +eval_op_guard('bnot', A, Res) when Res =:= bnot A -> ok. eval_op_guard('-', A, B, Res) when Res =:= A - B -> ok; eval_op_guard('+', A, B, Res) when Res =:= A + B -> ok; @@ -226,6 +231,7 @@ eval_op_guard('bor', A, B, Res) when Res =:= A bor B -> ok; eval_op_guard('bxor', A, B, Res) when Res =:= A bxor B -> ok; eval_op_guard('bsl', A, B, Res) when Res =:= A bsl B -> ok; eval_op_guard('bsr', A, B, Res) when Res =:= A bsr B -> ok; +eval_op_guard(madd, A, B, Res) when Res =:= A * B + 1-> ok; eval_op_guard(Op, A, B, Res) -> {error,{Op,A,B,Res}}. test_squaring(I) -> @@ -597,6 +603,121 @@ rand_int() -> <> = rand:bytes(Sz), Int. +reductions(_Config) -> + MaxInt = erlang:system_info(max_integer), + + %% Ensure that each arithmetic operation increments + %% the reuduction count with a reasonable amount. + %% + %% Assumptions: + %% * We have 4000 reductions after a context switch. + %% * One reduction can handle 16 bignum words. + %% * `*`, `div`, and `rem` are conservatively assumed + %% to be quadratic. + %% * All the others are linear. + %% + %% {Op, Args, MinReds} + Ops = [{'*', [id(1) bsl 64, 7], 1}, + {'*', [id(1) bsl (64*20), 7], 2}, + {'*', [id(1) bsl (64*100), id(1) bsl (64*50)], 322}, + {'*', [MaxInt div 8, 8], 3900}, + + {madd, [id(1) bsl 64, 7], 1}, + {madd, [id(1) bsl (64*20), 7], 2}, + {madd, [id(1) bsl (64*100), id(1) bsl (64*50)], 322}, + {madd, [MaxInt div 8, 8], 3900}, + + {'div', [id(1) bsl 64, 7], 1}, + {'div', [id(1) bsl (64*100), 7], 7}, + {'div', [id(1) bsl (64*100), id(1) bsl (64*50)], 4}, + {'div', [MaxInt, 7], 3900}, + + {'rem', [id(1) bsl 64, 7], 1}, + {'rem', [id(1) bsl (64*100), 7], 7}, + {'rem', [id(1) bsl (64*100), id(1) bsl (64*50)], 4}, + {'rem', [MaxInt, 7], 3900}, + + {'+', [id(1) bsl (64*100), id(1) bsl (64*75)], 7}, + {'+', [id(1) bsl (64*1000), id(1) bsl (64*75)], 63}, + + {'-', [id(1) bsl (64*100), id(1) bsl (64*75)], 7}, + {'-', [id(1) bsl (64*1000), id(1) bsl (64*120)], 63}, + + {'-', [id(1) bsl (64*100)], 7}, + {'-', [id(1) bsl (64*1000)], 63}, + + {'band', [id(1) bsl (64*100), id(1) bsl (64*75)], 7}, + {'band', [id(1) bsl (64*1000), id(1) bsl (64*75)], 63}, + + {'bor', [id(1) bsl (64*100), id(1) bsl (64*75)], 7}, + {'bor', [id(1) bsl (64*1000), id(1) bsl (64*75)], 63}, + + {'bxor', [id(1) bsl (64*100), id(1) bsl (64*75)], 7}, + {'bxor', [id(1) bsl (64*1000), id(1) bsl (64*75)], 63}, + + {'bnot', [id(1) bsl (64*100)], 7}, + {'bnot', [id(1) bsl (64*1000)], 63}, + + {'bsl', [id(1) bsl (64*100), 1], 7}, + {'bsl', [id(1) bsl (64*1000), 1], 63}, + + {'bsr', [id(1) bsl (64*100), 1], 7}, + {'bsr', [id(1) bsl (64*1000), 1], 63} + ], + lists:foreach(fun red_eval/1, Ops), + ok. + +red_eval({Op, Args, MinReds}) + when is_atom(Op), is_list(Args), is_integer(MinReds) -> + erlang:yield(), + case Args of + [Arg1,Arg2] -> + {reductions,Reds0} = process_info(self(), reductions), + Res = eval_op(Op, Arg1, Arg2), + {reductions,Reds1} = process_info(self(), reductions), + check_reds(Op, Args, Reds0, Reds1, MinReds), + + {reductions,Reds2} = process_info(self(), reductions), + eval_op_guard(Op, Arg1, Arg2, Res), + {reductions,Reds3} = process_info(self(), reductions), + check_reds(Op, Args, Reds2, Reds3, MinReds), + + case Op of + madd -> ok; + _ -> + {reductions,Reds4} = process_info(self(), reductions), + _ = erlang:Op(Arg1, Arg2), + {reductions,Reds5} = process_info(self(), reductions), + check_reds(Op, Args, Reds4, Reds5, MinReds) + end; + [Arg1] -> + {reductions,Reds0} = process_info(self(), reductions), + Res = eval_op(Op, Arg1), + {reductions,Reds1} = process_info(self(), reductions), + check_reds(Op, Args, Reds0, Reds1, MinReds), + + {reductions,Reds2} = process_info(self(), reductions), + eval_op_guard(Op, Arg1, Res), + {reductions,Reds3} = process_info(self(), reductions), + check_reds(Op, Args, Reds2, Reds3, MinReds), + + {reductions,Reds4} = process_info(self(), reductions), + _ = erlang:Op(Arg1), + {reductions,Reds5} = process_info(self(), reductions), + check_reds(Op, Args, Reds4, Reds5, MinReds) + end. + +check_reds(Op, _Args, Reds0, Reds1, MinReds) -> + Reds = Reds1 - Reds0 - 1, + if + Reds >= MinReds -> + ok; + true -> + io:format("~p: expected at least ~p reductions; " + "got only ~p", [Op, MinReds, Reds]), + error(failed) + end. + %%% %%% Common utilities. %%%