diff --git a/erts/emulator/beam/erl_arith.c b/erts/emulator/beam/erl_arith.c
index b608df2b4fc0..b86b97bc5e2d 100644
--- a/erts/emulator/beam/erl_arith.c
+++ b/erts/emulator/beam/erl_arith.c
@@ -1364,8 +1364,11 @@ Eterm erts_bxor(Process* p, Eterm arg1, Eterm arg2)
     need = BIG_NEED_SIZE(MAX(big_size(arg1), big_size(arg2)) + 1);
     hp = HeapFragOnlyAlloc(p, need);
     arg1 = big_bxor(arg1, arg2, hp);
-    ASSERT(is_not_nil(arg1));
     maybe_shrink(p, hp, arg1, need);
+    if (is_nil(arg1)) {
+        p->freason = SYSTEM_LIMIT;
+        return THE_NON_VALUE;
+    }
     return arg1;
 }
 
diff --git a/erts/emulator/test/big_SUITE.erl b/erts/emulator/test/big_SUITE.erl
index 5e121e30c570..7048345cb019 100644
--- a/erts/emulator/test/big_SUITE.erl
+++ b/erts/emulator/test/big_SUITE.erl
@@ -37,6 +37,7 @@
 
 
 -include_lib("common_test/include/ct.hrl").
+-include_lib("stdlib/include/assert.hrl").
 
 suite() ->
     [{ct_hooks,[ts_install_cth]},
@@ -380,6 +381,23 @@ system_limit(Config) when is_list(Config) ->
     {'EXIT',{system_limit,_}} = (catch id(1) bsl (1 bsl 45)),
     {'EXIT',{system_limit,_}} = (catch id(1) bsl (1 bsl 69)),
 
+    ?assertError(system_limit, Maxbig bxor -1),
+    ?assertError(system_limit, apply(erlang, id('bxor'), [Maxbig,-1])),
+    if
+        Maxbig bxor -1 -> error(should_fail);
+        true -> ok
+    end,
+
+    %% bnot -Maxbig should not raise an exception.
+    MaxMinusOne = Maxbig - 1,
+    MinusMaxbig = -Maxbig,
+    MaxMinusOne = bnot MinusMaxbig,
+    MaxMinusOne = apply(erlang, id('bnot'), [MinusMaxbig]),
+
+    %% -Maxbig bxor -1 should not raise an exception.
+    MaxMinusOne = MinusMaxbig bxor -1,
+    MaxMinusOne = apply(erlang, id('bxor'), [MinusMaxbig,-1]),
+
     %% There should be no system_limit exception when shifting a zero.
     0 = id(0) bsl (1 bsl 128),
     0 = id(0) bsr -(1 bsl 128),