From c5a12719194f8a04813b1a5dc62f0fee9fc67733 Mon Sep 17 00:00:00 2001 From: Wordman Date: Tue, 10 Dec 2019 23:48:33 +0200 Subject: [PATCH 01/48] Properly fix `Jump out of too many nested blocks` error (overrides previous fix, as merging them would be too complex and unneeded) --- goto.py | 68 +++++++++++++++++------------ test_goto.py | 120 ++++++++++++++++++++++++++++++--------------------- 2 files changed, 113 insertions(+), 75 deletions(-) diff --git a/goto.py b/goto.py index 3c67459..81a4ae5 100644 --- a/goto.py +++ b/goto.py @@ -86,6 +86,19 @@ def _parse_instructions(code): extended_arg_offset = None yield (dis.opname[opcode], oparg, offset) +def _get_instruction_size(opname, oparg=0): + size = 1 + + extended_arg = oparg >> _BYTECODE.argument_bits + if extended_arg != 0: + size += _get_instruction_size('EXTENDED_ARG', extended_arg) + oparg &= (1 << _BYTECODE.argument_bits) - 1 + + opcode = dis.opmap[opname] + if opcode >= _BYTECODE.have_argument: + size += _BYTECODE.argument.size + + return size def _write_instruction(buf, pos, opname, oparg=0): extended_arg = oparg >> _BYTECODE.argument_bits @@ -164,37 +177,38 @@ def _patch_code(code): target_depth = len(target_stack) if origin_stack[:target_depth] != target_stack: raise SyntaxError('Jump into different block') + + size = 0 + for i in range(len(origin_stack) - target_depth): + size += _get_instruction_size('POP_BLOCK') + size += _get_instruction_size('JUMP_ABSOLUTE', target // _BYTECODE.jump_unit) + + moved_to_end = False + if pos + size > end: + # not enough space, add at end + pos = _write_instruction(buf, pos, 'JUMP_ABSOLUTE', len(buf) // _BYTECODE.jump_unit) + + if pos > end: + raise SyntaxError('Internal error - not enough bytecode space') + + size += _get_instruction_size('JUMP_ABSOLUTE', end // _BYTECODE.jump_unit) + + moved_to_end = True + pos = len(buf) + buf.extend([0] * size) - failed = False try: for i in range(len(origin_stack) - target_depth): pos = _write_instruction(buf, pos, 'POP_BLOCK') - - if target >= end: - rel_target = (target - pos) // _BYTECODE.jump_unit - oparg_bits = 0 - - while True: - rel_target -= (1 + _BYTECODE.argument.size) // _BYTECODE.jump_unit - if rel_target >> oparg_bits == 0: - pos = _write_instruction(buf, pos, 'EXTENDED_ARG', 0) - break - - oparg_bits += _BYTECODE.argument_bits - if rel_target >> oparg_bits == 0: - break - - pos = _write_instruction(buf, pos, 'JUMP_FORWARD', rel_target) - else: - pos = _write_instruction(buf, pos, 'JUMP_ABSOLUTE', target // _BYTECODE.jump_unit) - - except (IndexError, struct.error): - failed = True - - if failed or pos > end: - raise SyntaxError('Jump out of too many nested blocks') - - _inject_nop_sled(buf, pos, end) + pos = _write_instruction(buf, pos, 'JUMP_ABSOLUTE', target // _BYTECODE.jump_unit) + except (IndexError, struct.error) as e: + raise SyntaxError("Internal error", e) + + if moved_to_end: + pos = _write_instruction(buf, pos, 'JUMP_ABSOLUTE', end // _BYTECODE.jump_unit) + + else: + _inject_nop_sled(buf, pos, end) return _make_code(code, _array_to_bytes(buf)) diff --git a/test_goto.py b/test_goto.py index 7ff1dc1..cabc7b4 100644 --- a/test_goto.py +++ b/test_goto.py @@ -71,63 +71,87 @@ def func(): pytest.raises(SyntaxError, with_goto, func) -if sys.version_info >= (3, 6): - def test_jump_out_of_nested_2_loops(): - @with_goto - def func(): - x = 1 - for i in range(2): - for j in range(2): - # These are more than 256 bytes of bytecode, requiring - # a JUMP_FORWARD below on Python 3.6+, since the absolute - # address would be too large, after leaving two blocks. - x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x - x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x - x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x +def test_jump_out_of_nested_2_loops(): + @with_goto + def func(): + x = 1 + for i in range(2): + for j in range(2): + # These are more than 256 bytes of bytecode + x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x + x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x + x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x + + goto .end + label .end + return (i, j) + assert func() == (0, 0) + +def test_jump_out_of_nested_3_loops(): + @with_goto + def func(): + for i in range(2): + for j in range(2): + for k in range(2): goto .end - label .end - return (i, j) + label .end + return (i, j, k) - assert func() == (0, 0) + assert func() == (0, 0, 0) - def test_jump_out_of_nested_3_loops(): - def func(): - for i in range(2): - for j in range(2): - for k in range(2): +def test_jump_out_of_nested_4_loops(): + @with_goto + def func(): + for i in range(2): + for j in range(2): + for k in range(2): + for m in range(2): goto .end - label .end - return (i, j, k) - - pytest.raises(SyntaxError, with_goto, func) -else: - def test_jump_out_of_nested_4_loops(): - @with_goto - def func(): - for i in range(2): - for j in range(2): - for k in range(2): - for m in range(2): - goto .end - label .end - return (i, j, k, m) + label .end + return (i, j, k, m) - assert func() == (0, 0, 0, 0) + assert func() == (0, 0, 0, 0) - def test_jump_out_of_nested_5_loops(): - def func(): - for i in range(2): - for j in range(2): - for k in range(2): - for m in range(2): - for n in range(2): - goto .end - label .end - return (i, j, k, m, n) +def test_jump_out_of_nested_5_loops(): + @with_goto + def func(): + for i in range(2): + for j in range(2): + for k in range(2): + for m in range(2): + for n in range(2): + goto .end + label .end + return (i, j, k, m, n) - pytest.raises(SyntaxError, with_goto, func) + assert func() == (0, 0, 0, 0, 0) + +def test_jump_out_of_nested_11_loops(): + @with_goto + def func(): + x = 1 + for i1 in range(2): + for i2 in range(2): + for i3 in range(2): + for i4 in range(2): + for i5 in range(2): + for i6 in range(2): + for i7 in range(2): + for i8 in range(2): + for i9 in range(2): + for i10 in range(2): + for i11 in range(2): + # These are more than 256 bytes of bytecode + x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x + x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x + x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x + + goto .end + label .end + return (i1, i2, i3, i4, i5, i6, i7, i8, i9, i10, i11) + assert func() == (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) def test_jump_across_loops(): def func(): From 8b431a23da4d52e1291690640fad05a549d4a819 Mon Sep 17 00:00:00 2001 From: condut Date: Wed, 11 Dec 2019 00:46:46 +0200 Subject: [PATCH 02/48] Rename new SyntaxError to be more descriptive in case ever encountered. I was not able to get it to happen for pretty huge functions (much much larger than the ones in the test), though, so there's a good chance it's unreachable. --- goto.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/goto.py b/goto.py index 81a4ae5..18fcf1d 100644 --- a/goto.py +++ b/goto.py @@ -189,7 +189,7 @@ def _patch_code(code): pos = _write_instruction(buf, pos, 'JUMP_ABSOLUTE', len(buf) // _BYTECODE.jump_unit) if pos > end: - raise SyntaxError('Internal error - not enough bytecode space') + raise SyntaxError('Goto in an incredibly huge function') # not sure if reachable size += _get_instruction_size('JUMP_ABSOLUTE', end // _BYTECODE.jump_unit) From 2b155c193aac6c7287f2249c9b627c53afac8acc Mon Sep 17 00:00:00 2001 From: condut Date: Fri, 13 Dec 2019 00:18:50 +0200 Subject: [PATCH 03/48] Refactor previous changes to avoid duplication & fix safety issues (Candidate for moving to 'fix' branch?) --- goto.py | 54 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/goto.py b/goto.py index 18fcf1d..e7db8f7 100644 --- a/goto.py +++ b/goto.py @@ -98,7 +98,16 @@ def _get_instruction_size(opname, oparg=0): if opcode >= _BYTECODE.have_argument: size += _BYTECODE.argument.size - return size + return size + +def _get_instructions_size(ops): + size = 0 + for op in ops: + if isinstance(op, str): + size += _get_instruction_size(op) + else: + size += _get_instruction_size(*op) + return size def _write_instruction(buf, pos, opname, oparg=0): extended_arg = oparg >> _BYTECODE.argument_bits @@ -116,6 +125,13 @@ def _write_instruction(buf, pos, opname, oparg=0): return pos +def _write_instructions(buf, pos, ops): + for op in ops: + if isinstance(op, str): + pos = _write_instruction(buf, pos, op) + else: + pos = _write_instruction(buf, pos, *op) + return pos def _find_labels_and_gotos(code): labels = {} @@ -178,36 +194,30 @@ def _patch_code(code): if origin_stack[:target_depth] != target_stack: raise SyntaxError('Jump into different block') - size = 0 + ops = [] for i in range(len(origin_stack) - target_depth): - size += _get_instruction_size('POP_BLOCK') - size += _get_instruction_size('JUMP_ABSOLUTE', target // _BYTECODE.jump_unit) + ops.append('POP_BLOCK') + ops.append(('JUMP_ABSOLUTE', target // _BYTECODE.jump_unit)) - moved_to_end = False - if pos + size > end: - # not enough space, add at end - pos = _write_instruction(buf, pos, 'JUMP_ABSOLUTE', len(buf) // _BYTECODE.jump_unit) + if pos + _get_instructions_size(ops) > end: + # not enough space, add code at buffer end and jump there & back + buf_end = len(buf) - if pos > end: + go_to_end_ops = [('JUMP_ABSOLUTE', buf_end // _BYTECODE.jump_unit)] + + if pos + _get_instructions_size(go_to_end_ops) > end: raise SyntaxError('Goto in an incredibly huge function') # not sure if reachable - size += _get_instruction_size('JUMP_ABSOLUTE', end // _BYTECODE.jump_unit) + pos = _write_instructions(buf, pos, go_to_end_ops) + _inject_nop_sled(buf, pos, end) - moved_to_end = True - pos = len(buf) - buf.extend([0] * size) - - try: - for i in range(len(origin_stack) - target_depth): - pos = _write_instruction(buf, pos, 'POP_BLOCK') - pos = _write_instruction(buf, pos, 'JUMP_ABSOLUTE', target // _BYTECODE.jump_unit) - except (IndexError, struct.error) as e: - raise SyntaxError("Internal error", e) + ops.append(('JUMP_ABSOLUTE', end // _BYTECODE.jump_unit)) - if moved_to_end: - pos = _write_instruction(buf, pos, 'JUMP_ABSOLUTE', end // _BYTECODE.jump_unit) + buf.extend([0] * _get_instructions_size(ops)) + _write_instructions(buf, buf_end, ops) else: + pos = _write_instructions(buf, pos, ops) _inject_nop_sled(buf, pos, end) return _make_code(code, _array_to_bytes(buf)) From 66f0dee79a4f1edb4af8bea9c3c126060260a44b Mon Sep 17 00:00:00 2001 From: condut Date: Fri, 13 Dec 2019 00:35:35 +0200 Subject: [PATCH 04/48] Added python 3.8 support Added 4 tests, 3 currently disabled (pre-existing issues, especially on pypy). --- goto.py | 48 +++++++++++++++++++++++++++++--------------- test_goto.py | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 16 deletions(-) diff --git a/goto.py b/goto.py index e7db8f7..5746c36 100644 --- a/goto.py +++ b/goto.py @@ -33,6 +33,8 @@ def __init__(self): self.argument = struct.Struct(' end: diff --git a/test_goto.py b/test_goto.py index cabc7b4..fea5e8b 100644 --- a/test_goto.py +++ b/test_goto.py @@ -63,6 +63,18 @@ def func(): assert func() == 0 +def test_jump_out_of_loop_and_survive(): + @with_goto + def func(): + for i in range(10): + for j in range(10): + goto .end + label .end + return (i, j) + + assert func() == (9, 0) + + def test_jump_into_loop(): def func(): for i in range(10): @@ -180,6 +192,22 @@ def func(): assert func() == None +"""def test_jump_out_of_try_block_and_survive(): + @with_goto + def func(): + for i in range(10): + try: + rv = None + goto .end + except: + rv = 'except' + finally: + rv = 'finally' + label .end + return (i, rv) + + assert func() == (9, None)""" + def test_jump_into_try_block(): def func(): try: @@ -191,6 +219,34 @@ def func(): pytest.raises(SyntaxError, with_goto, func) +"""def test_jump_out_of_except_block(): + @with_goto + def func(): + try: + rv = 1 / 0 + except: + rv = 'except' + goto .end + finally: + rv = 'finally' + label .end + return rv + + assert func() == 'except'""" + + +"""def test_jump_into_except_block(): + def func(): + try: + pass + except: + label .block + pass + goto .block + + pytest.raises(SyntaxError, with_goto, func)""" + + def test_jump_to_unknown_label(): def func(): goto .unknown From 8db24987c812ebb6cea428e003d32e439d2bbaa8 Mon Sep 17 00:00:00 2001 From: condut Date: Fri, 13 Dec 2019 15:58:18 +0200 Subject: [PATCH 05/48] Fix test_jump_out_of_try_block_and_survive. Add new test_jump_out_of_try_block_and_live --- goto.py | 9 ++++++++- test_goto.py | 36 +++++++++++++++++++++++++++++++++--- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/goto.py b/goto.py index 5746c36..1ccbe11 100644 --- a/goto.py +++ b/goto.py @@ -4,6 +4,10 @@ import types import functools +try: + import __pypy__ +except: + __pypy__ = None try: _array_to_bytes = array.array.tobytes @@ -208,11 +212,14 @@ def _patch_code(code): raise SyntaxError('Jump into different block') ops = [] - for block, _ in origin_stack[target_depth:]: + for block, _ in reversed(origin_stack[target_depth:]): if block == 'FOR_ITER': ops.append('POP_TOP') else: ops.append('POP_BLOCK') + if __pypy__ and block == 'SETUP_FINALLY': # pypy 3.6 keep a block around until END_FINALLY; python 3.8 reuses SETUP_FINALLY for SETUP_EXCEPT. What will pypy 3.8 do? + ops.append(('LOAD_CONST', code.co_consts.index(None))) + ops.append('END_FINALLY') ops.append(('JUMP_ABSOLUTE', target // _BYTECODE.jump_unit)) if pos + _get_instructions_size(ops) > end: diff --git a/test_goto.py b/test_goto.py index fea5e8b..9bea4d6 100644 --- a/test_goto.py +++ b/test_goto.py @@ -139,6 +139,20 @@ def func(): assert func() == (0, 0, 0, 0, 0) +def test_jump_out_of_nested_4_loops_and_survive(): + @with_goto + def func(): + for i in range(2): + for j in range(2): + for k in range(2): + for m in range(2): + for n in range(2): + goto .end + label .end + return (i, j, k, m, n) + + assert func() == (1, 0, 0, 0, 0) + def test_jump_out_of_nested_11_loops(): @with_goto def func(): @@ -191,8 +205,7 @@ def func(): assert func() == None - -"""def test_jump_out_of_try_block_and_survive(): +def test_jump_out_of_try_block_and_survive(): @with_goto def func(): for i in range(10): @@ -206,7 +219,24 @@ def func(): label .end return (i, rv) - assert func() == (9, None)""" + assert func() == (9, None) + +def test_jump_out_of_try_block_and_live(): + @with_goto + def func(): + for i in range(3): + for j in range(3): + try: + rv = None + goto .end + except: + rv = 'except' + finally: + rv = 'finally' + label .end + return (i, j, rv) + + assert func() == (2, 2, None) def test_jump_into_try_block(): def func(): From 4c811283e18fc3f925f7a60ea38204d9690c27cf Mon Sep 17 00:00:00 2001 From: condut Date: Fri, 13 Dec 2019 16:05:27 +0200 Subject: [PATCH 06/48] Avoid patching code again and again in nested functions Uses a weak dictionary just in case code objects can be gc'd (can they? maybe on some implementations?) --- goto.py | 11 ++++++++++- test_goto.py | 12 ++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/goto.py b/goto.py index 1ccbe11..f0b9d29 100644 --- a/goto.py +++ b/goto.py @@ -3,6 +3,7 @@ import array import types import functools +import weakref try: import __pypy__ @@ -47,6 +48,7 @@ def argument_bits(self): _BYTECODE = _Bytecode() +_patched_code_cache = weakref.WeakKeyDictionary() def _make_code(code, codestring): try: @@ -195,6 +197,10 @@ def _inject_nop_sled(buf, pos, end): def _patch_code(code): + new_code = _patched_code_cache.get(code) + if new_code is not None: + return new_code + labels, gotos = _find_labels_and_gotos(code) buf = array.array('B', code.co_code) @@ -243,7 +249,10 @@ def _patch_code(code): pos = _write_instructions(buf, pos, ops) _inject_nop_sled(buf, pos, end) - return _make_code(code, _array_to_bytes(buf)) + new_code = _make_code(code, _array_to_bytes(buf)) + + _patched_code_cache[code] = new_code + return new_code def with_goto(func_or_code): diff --git a/test_goto.py b/test_goto.py index 9bea4d6..4238f6b 100644 --- a/test_goto.py +++ b/test_goto.py @@ -293,3 +293,15 @@ def func(): assert newfunc is not func assert newfunc.foo == 'bar' + +def test_code_is_not_copy(): + def outer_func(): + @with_goto + def inner_func(): + goto .test + label .test + return inner_func + + assert outer_func() is not outer_func() + assert outer_func().__code__ is outer_func().__code__ + From 536b16e3c74de5b018d7e1ba23b27fc8ae87f0c5 Mon Sep 17 00:00:00 2001 From: condut Date: Fri, 13 Dec 2019 16:49:24 +0200 Subject: [PATCH 07/48] Fix previous change for python2.6 (doesn't support Code weakrefs) --- goto.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/goto.py b/goto.py index f0b9d29..869611a 100644 --- a/goto.py +++ b/goto.py @@ -48,7 +48,11 @@ def argument_bits(self): _BYTECODE = _Bytecode() -_patched_code_cache = weakref.WeakKeyDictionary() +_patched_code_cache = weakref.WeakKeyDictionary() # use a weak dictionary in case code objects can be garbage-collected +try: + _patched_code_cache[_Bytecode.__init__.__code__] = None +except TypeError: + _patched_code_cache = {} # ...unless not supported def _make_code(code, codestring): try: From 42b5518190cbab8aeb90d0f66184ac8374f5d0e7 Mon Sep 17 00:00:00 2001 From: condut Date: Fri, 13 Dec 2019 17:16:13 +0200 Subject: [PATCH 08/48] Fix with blocks (add tests) add warnings for easier debugging (for peace of heart - they weren't currently triggered by any tests) --- goto.py | 33 ++++++++++++++++++++++++++------- test_goto.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 7 deletions(-) diff --git a/goto.py b/goto.py index 869611a..d1a2faa 100644 --- a/goto.py +++ b/goto.py @@ -4,6 +4,7 @@ import types import functools import weakref +import warnings try: import __pypy__ @@ -74,7 +75,7 @@ def _make_code(code, codestring): return types.CodeType(*args) -def _parse_instructions(code): +def _parse_instructions(code, yield_nones_at_end=0): extended_arg = 0 extended_arg_offset = None pos = 0 @@ -100,6 +101,9 @@ def _parse_instructions(code): extended_arg = 0 extended_arg_offset = None yield (dis.opname[opcode], oparg, offset) + + for _ in range(yield_nones_at_end): + yield (None, None, None) def _get_instruction_size(opname, oparg=0): size = 1 @@ -148,6 +152,9 @@ def _write_instructions(buf, pos, ops): pos = _write_instruction(buf, pos, *op) return pos +def _warn_bug(msg): + warnings.warn("Internal error detected - result of with_goto may be incorrect. (%s)" % msg) + def _find_labels_and_gotos(code): labels = {} gotos = [] @@ -159,8 +166,14 @@ def _find_labels_and_gotos(code): opname1 = oparg1 = offset1 = None opname2 = oparg2 = offset2 = None opname3 = oparg3 = offset3 = None + + def pop_block(): + if block_stack: + block_stack.pop() + else: + _warn_bug("can't pop block") - for opname4, oparg4, offset4 in _parse_instructions(code.co_code): + for opname4, oparg4, offset4 in _parse_instructions(code.co_code, 3): if opname1 in ('LOAD_GLOBAL', 'LOAD_NAME'): if opname2 == 'LOAD_ATTR' and opname3 == 'POP_TOP': name = code.co_names[oparg1] @@ -182,16 +195,19 @@ def _find_labels_and_gotos(code): block_counter += 1 block_stack.append((opname1, block_counter)) block_exits.append(offset1 + oparg1) - elif opname1 == 'POP_BLOCK' and block_stack: - block_stack.pop() - elif block_exits and offset1 == block_exits[-1] and block_stack: - block_stack.pop() + elif opname1 == 'POP_BLOCK': + pop_block() + elif block_exits and offset1 == block_exits[-1]: + pop_block() block_exits.pop() opname1, oparg1, offset1 = opname2, oparg2, offset2 opname2, oparg2, offset2 = opname3, oparg3, offset3 opname3, oparg3, offset3 = opname4, oparg4, offset4 + if block_stack: + _warn_bug("block stack not empty") + return labels, gotos @@ -227,7 +243,10 @@ def _patch_code(code): ops.append('POP_TOP') else: ops.append('POP_BLOCK') - if __pypy__ and block == 'SETUP_FINALLY': # pypy 3.6 keep a block around until END_FINALLY; python 3.8 reuses SETUP_FINALLY for SETUP_EXCEPT. What will pypy 3.8 do? + if block in ('SETUP_WITH', 'SETUP_ASYNC_WITH'): + ops.append('POP_TOP') + # pypy 3.6 keeps a block around until END_FINALLY; python 3.8 reuses SETUP_FINALLY for SETUP_EXCEPT (where END_FINALLY is not accepted). What will pypy 3.8 do? + if __pypy__ and block in ('SETUP_FINALLY', 'SETUP_WITH', 'SETUP_ASYNC_WITH'): ops.append(('LOAD_CONST', code.co_consts.index(None))) ops.append('END_FINALLY') ops.append(('JUMP_ABSOLUTE', target // _BYTECODE.jump_unit)) diff --git a/test_goto.py b/test_goto.py index 4238f6b..4285b66 100644 --- a/test_goto.py +++ b/test_goto.py @@ -75,6 +75,19 @@ def func(): assert func() == (9, 0) +def test_jump_out_of_loop_and_live(): + @with_goto + def func(): + for i in range(10): + for j in range(10): + for k in range(10): + goto .end + label .end + return (i, j, k) + + assert func() == (9, 0, 0) + + def test_jump_into_loop(): def func(): for i in range(10): @@ -189,6 +202,41 @@ def func(): pytest.raises(SyntaxError, with_goto, func) +class Context: + def __init__(self): + self.enters = 0 + self.exits = 0 + def __enter__(self): + self.enters += 1 + return self + def __exit__(self, a, b, c): + self.exits += 1 + return self + def data(self): + return (self.enters, self.exits) + +def test_jump_out_of_with_block(): + @with_goto + def func(): + with Context() as c: + goto .out + label .out + return c.data() + + assert func()== (1, 0) + +def test_jump_out_of_with_block_and_live(): + @with_goto + def func(): + c = Context() + for i in range(3): + for j in range(3): + with c: + goto .out + label .out + return (i, j, c.data()) + + assert func() == (2, 0, (3, 0)) def test_jump_out_of_try_block(): @with_goto From 2bf1e423300df02fbdfc3a716c0a931a3dfa98db Mon Sep 17 00:00:00 2001 From: condut Date: Fri, 13 Dec 2019 18:27:17 +0200 Subject: [PATCH 09/48] Support jumping out of except & finally blocks! --- goto.py | 68 ++++++++++++++++++++++++++++++++++++++++++++----- test_goto.py | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 131 insertions(+), 9 deletions(-) diff --git a/goto.py b/goto.py index d1a2faa..9c801c5 100644 --- a/goto.py +++ b/goto.py @@ -41,6 +41,9 @@ def __init__(self): self.jump_unit = 1 self.has_loop_blocks = 'SETUP_LOOP' in dis.opmap + self.has_pop_except = 'POP_EXCEPT' in dis.opmap + self.has_setup_with = 'SETUP_WITH' in dis.opmap + self.has_setup_except = 'SETUP_EXCEPT' in dis.opmap @property def argument_bits(self): @@ -161,46 +164,93 @@ def _find_labels_and_gotos(code): block_stack = [] block_counter = 0 - block_exits = [] + for_exits = [] + excepts = [] + finallies = [] - opname1 = oparg1 = offset1 = None + opname0 = oparg0 = offset0 = None + opname1 = oparg1 = offset1 = None # the main one we're looking at each loop iteration opname2 = oparg2 = offset2 = None opname3 = oparg3 = offset3 = None + def replace_block_in_stack(stack, old_block, new_block): + for i, block in enumerate(stack): + if block == old_block: + stack[i] = new_block + + def replace_block(old_block, new_block): + replace_block_in_stack(block_stack, old_block, new_block) + for label in labels: + replace_block_in_stack(labels[label][2], old_block, new_block) + for goto in gotos: + replace_block_in_stack(goto[3], old_block, new_block) + def pop_block(): if block_stack: block_stack.pop() else: _warn_bug("can't pop block") + + def pop_block_of_type(type): + if block_stack and block_stack[-1][0] != type: + # in 3.8, only finally blocks are supported, so we must determine the except/finally nature ourselves, and replace the block afterwards + if not _BYTECODE.has_setup_except and type == "" and block_stack[-1][0] == '': + replace_block(block_stack[-1], (type, block_stack[-1][1])) + else: + _warn_bug("mismatched block type") + pop_block() for opname4, oparg4, offset4 in _parse_instructions(code.co_code, 3): + # check for special opcodes if opname1 in ('LOAD_GLOBAL', 'LOAD_NAME'): if opname2 == 'LOAD_ATTR' and opname3 == 'POP_TOP': name = code.co_names[oparg1] if name == 'label': labels[oparg2] = (offset1, offset4, - tuple(block_stack)) + list(block_stack)) elif name == 'goto': gotos.append((offset1, offset4, oparg2, - tuple(block_stack))) + list(block_stack))) elif opname1 in ('SETUP_LOOP', 'SETUP_EXCEPT', 'SETUP_FINALLY', 'SETUP_WITH', 'SETUP_ASYNC_WITH'): block_counter += 1 block_stack.append((opname1, block_counter)) + if opname1 == 'SETUP_EXCEPT' and _BYTECODE.has_pop_except: + excepts.append(offset1 + oparg1) + elif opname1 == 'SETUP_FINALLY': + finallies.append(offset1 + oparg1) elif not _BYTECODE.has_loop_blocks and opname1 == 'FOR_ITER': block_counter += 1 block_stack.append((opname1, block_counter)) - block_exits.append(offset1 + oparg1) + for_exits.append(offset1 + oparg1) elif opname1 == 'POP_BLOCK': pop_block() - elif block_exits and offset1 == block_exits[-1]: + elif opname1 == 'POP_EXCEPT': + pop_block_of_type('') + elif opname1 == 'END_FINALLY': + if opname0 != 'JUMP_FORWARD': # hack for dummy end-finally in except block (correct fix would be a jump-aware reading of instructions!) + pop_block_of_type('') + elif opname1 in ('WITH_CLEANUP', 'WITH_CLEANUP_START') and _BYTECODE.has_setup_with: + block_stack.append(('', -1)) # temporary block to match END_FINALLY + + # check for special offsets + if for_exits and offset1 == for_exits[-1]: pop_block() - block_exits.pop() + for_exits.pop() + if excepts and offset1 == excepts[-1]: + block_counter += 1 + block_stack.append(('', block_counter)) + excepts.pop() + if finallies and offset1 == finallies[-1]: + block_counter += 1 + block_stack.append(('', block_counter)) + finallies.pop() + opname0, oparg0, offset0 = opname1, oparg1, offset1 opname1, oparg1, offset1 = opname2, oparg2, offset2 opname2, oparg2, offset2 = opname3, oparg3, offset3 opname3, oparg3, offset3 = opname4, oparg4, offset4 @@ -241,6 +291,10 @@ def _patch_code(code): for block, _ in reversed(origin_stack[target_depth:]): if block == 'FOR_ITER': ops.append('POP_TOP') + elif block == '': + ops.append('POP_EXCEPT') + elif block == '': + ops.append('END_FINALLY') else: ops.append('POP_BLOCK') if block in ('SETUP_WITH', 'SETUP_ASYNC_WITH'): diff --git a/test_goto.py b/test_goto.py index 4285b66..f7ff3c2 100644 --- a/test_goto.py +++ b/test_goto.py @@ -237,6 +237,28 @@ def func(): return (i, j, c.data()) assert func() == (2, 0, (3, 0)) + +def test_jump_into_with_block(): + def func(): + with Context() as c: + label .block + goto .block + + pytest.raises(SyntaxError, with_goto, func) + +def test_generator(): + @with_goto + def func(): + yield 0 + yield 1 + goto .x + yield 2 + yield 3 + label .x + yield 4 + yield 5 + + assert tuple(func()) == (0, 1, 4, 5) def test_jump_out_of_try_block(): @with_goto @@ -297,7 +319,7 @@ def func(): pytest.raises(SyntaxError, with_goto, func) -"""def test_jump_out_of_except_block(): +def test_jump_out_of_except_block(): @with_goto def func(): try: @@ -310,8 +332,24 @@ def func(): label .end return rv - assert func() == 'except'""" + assert func() == 'except' + +def test_jump_out_of_except_block_and_live(): + @with_goto + def func(): + for i in range(3): + for j in range(3): + try: + rv = 1 / 0 + except: + rv = 'except' + goto .end + finally: + rv = 'finally' + label .end + return (i, j, rv) + assert func() == (2, 0, 'except') """def test_jump_into_except_block(): def func(): @@ -324,6 +362,36 @@ def func(): pytest.raises(SyntaxError, with_goto, func)""" +def test_jump_out_of_finally_block(): + @with_goto + def func(): + try: + rv = None + finally: + rv = 'finally' + goto .end + rv = 'end' + label .end + return rv + + assert func() == 'finally' + +def test_jump_out_of_finally_block_and_live(): + @with_goto + def func(): + for i in range(3): + for j in range(3): + try: + rv = None + finally: + rv = 'finally' + goto .end + rv = 'end' + label .end + return i, j, rv + + assert func() == (2, 0, 'finally') + def test_jump_to_unknown_label(): def func(): From 9b8dd65320b4753830421b02242f9d9e1446a21f Mon Sep 17 00:00:00 2001 From: condut Date: Fri, 13 Dec 2019 18:34:47 +0200 Subject: [PATCH 10/48] Add combined try/catch/finally test and remove unneccessary op (later fix could go into fix branch as well) --- goto.py | 7 +++---- test_goto.py | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/goto.py b/goto.py index 9c801c5..501c7e3 100644 --- a/goto.py +++ b/goto.py @@ -305,7 +305,8 @@ def _patch_code(code): ops.append('END_FINALLY') ops.append(('JUMP_ABSOLUTE', target // _BYTECODE.jump_unit)) - if pos + _get_instructions_size(ops) > end: + size = _get_instructions_size(ops) + if pos + size > end: # not enough space, add code at buffer end and jump there & back buf_end = len(buf) @@ -317,9 +318,7 @@ def _patch_code(code): pos = _write_instructions(buf, pos, go_to_end_ops) _inject_nop_sled(buf, pos, end) - ops.append(('JUMP_ABSOLUTE', end // _BYTECODE.jump_unit)) - - buf.extend([0] * _get_instructions_size(ops)) + buf.extend([0] * size) _write_instructions(buf, buf_end, ops) else: diff --git a/test_goto.py b/test_goto.py index f7ff3c2..c8d51b5 100644 --- a/test_goto.py +++ b/test_goto.py @@ -392,6 +392,32 @@ def func(): assert func() == (2, 0, 'finally') +def test_jump_out_of_try_in_except_in_finally_and_live(): + @with_goto + def func(): + for i in range(3): + for j in range(3): + try: + rv = None + finally: + rv = 'finally' + try: + rv = 1 / 0 + except: + rv = 'except' + try: + rv = 'try' + goto .end + except: + rv = 'except2' + finally: + rv = 'finally2' + rv = 'end' + label .end + return i, j, rv + + assert func() == (2, 0, 'try') + def test_jump_to_unknown_label(): def func(): From 900317a307ae95ff2b845d5fa57b4508c597a093 Mon Sep 17 00:00:00 2001 From: condut Date: Fri, 13 Dec 2019 20:46:51 +0200 Subject: [PATCH 11/48] Support jumping into 'for' blocks with extend 'into' syntax --- goto.py | 136 ++++++++++++++++++++++++++++++++++----------------- test_goto.py | 11 +++++ 2 files changed, 103 insertions(+), 44 deletions(-) diff --git a/goto.py b/goto.py index 501c7e3..f11ec98 100644 --- a/goto.py +++ b/goto.py @@ -58,14 +58,18 @@ def argument_bits(self): except TypeError: _patched_code_cache = {} # ...unless not supported -def _make_code(code, codestring): +def _make_code(code, codestring, varnames, consts): + nlocals = len(varnames) + consts = tuple(consts) + try: - return code.replace(co_code=codestring) # new in 3.8+ - except: + # code.replace is new in 3.8+ + return code.replace(co_code=codestring, co_nlocals=nlocals, co_varnames=varnames, co_consts=consts) + except: args = [ - code.co_argcount, code.co_nlocals, code.co_stacksize, - code.co_flags, codestring, code.co_consts, - code.co_names, code.co_varnames, code.co_filename, + code.co_argcount, nlocals, code.co_stacksize, + code.co_flags, codestring, consts, + code.co_names, varnames, code.co_filename, code.co_name, code.co_firstlineno, code.co_lnotab, code.co_freevars, code.co_cellvars ] @@ -163,7 +167,7 @@ def _find_labels_and_gotos(code): gotos = [] block_stack = [] - block_counter = 0 + block_counter = 1 for_exits = [] excepts = [] finallies = [] @@ -185,6 +189,10 @@ def replace_block(old_block, new_block): for goto in gotos: replace_block_in_stack(goto[3], old_block, new_block) + def push_block(opname, oparg=0): + block_stack.append((opname, oparg, block_counter)) + return block_counter + 1 # to be assigned to block_counter + def pop_block(): if block_stack: block_stack.pop() @@ -195,7 +203,7 @@ def pop_block_of_type(type): if block_stack and block_stack[-1][0] != type: # in 3.8, only finally blocks are supported, so we must determine the except/finally nature ourselves, and replace the block afterwards if not _BYTECODE.has_setup_except and type == "" and block_stack[-1][0] == '': - replace_block(block_stack[-1], (type, block_stack[-1][1])) + replace_block(block_stack[-1], (type,) + block_stack[-1][1:]) else: _warn_bug("mismatched block type") pop_block() @@ -213,19 +221,25 @@ def pop_block_of_type(type): gotos.append((offset1, offset4, oparg2, - list(block_stack))) + list(block_stack), + False)) + elif opname2 == 'LOAD_ATTR' and opname3 == 'STORE_ATTR': + if code.co_names[oparg1] == 'goto' and code.co_names[oparg2] == 'into': + gotos.append((offset1, + offset4, + oparg3, + list(block_stack), + True)) elif opname1 in ('SETUP_LOOP', 'SETUP_EXCEPT', 'SETUP_FINALLY', 'SETUP_WITH', 'SETUP_ASYNC_WITH'): - block_counter += 1 - block_stack.append((opname1, block_counter)) + block_counter = push_block(opname1, oparg1) if opname1 == 'SETUP_EXCEPT' and _BYTECODE.has_pop_except: excepts.append(offset1 + oparg1) elif opname1 == 'SETUP_FINALLY': finallies.append(offset1 + oparg1) elif not _BYTECODE.has_loop_blocks and opname1 == 'FOR_ITER': - block_counter += 1 - block_stack.append((opname1, block_counter)) + block_counter = push_block(opname1, oparg1) for_exits.append(offset1 + oparg1) elif opname1 == 'POP_BLOCK': pop_block() @@ -235,19 +249,17 @@ def pop_block_of_type(type): if opname0 != 'JUMP_FORWARD': # hack for dummy end-finally in except block (correct fix would be a jump-aware reading of instructions!) pop_block_of_type('') elif opname1 in ('WITH_CLEANUP', 'WITH_CLEANUP_START') and _BYTECODE.has_setup_with: - block_stack.append(('', -1)) # temporary block to match END_FINALLY + block_counter = push_block('') # temporary block to match END_FINALLY # check for special offsets if for_exits and offset1 == for_exits[-1]: pop_block() for_exits.pop() if excepts and offset1 == excepts[-1]: - block_counter += 1 - block_stack.append(('', block_counter)) + block_counter = push_block('') excepts.pop() if finallies and offset1 == finallies[-1]: - block_counter += 1 - block_stack.append(('', block_counter)) + block_counter = push_block('') finallies.pop() opname0, oparg0, offset0 = opname1, oparg1, offset1 @@ -265,6 +277,26 @@ def _inject_nop_sled(buf, pos, end): while pos < end: pos = _write_instruction(buf, pos, 'NOP') +def _inject_ops(buf, pos, end, ops): + size = _get_instructions_size(ops) + if pos + size > end: + # not enough space, add code at buffer end and jump there & back + buf_end = len(buf) + + go_to_end_ops = [('JUMP_ABSOLUTE', buf_end // _BYTECODE.jump_unit)] + + if pos + _get_instructions_size(go_to_end_ops) > end: + raise SyntaxError('Goto in an incredibly huge function') # not sure if reachable + + pos = _write_instructions(buf, pos, go_to_end_ops) + _inject_nop_sled(buf, pos, end) + + buf.extend([0] * size) + _write_instructions(buf, buf_end, ops) + + else: + pos = _write_instructions(buf, pos, ops) + _inject_nop_sled(buf, pos, end) def _patch_code(code): new_code = _patched_code_cache.get(code) @@ -273,22 +305,32 @@ def _patch_code(code): labels, gotos = _find_labels_and_gotos(code) buf = array.array('B', code.co_code) + temp_var = None + + varnames = code.co_varnames + consts = list(code.co_consts) + + def get_const(value): + try: + i = consts.index(value) + except ValueError: + i = len(consts) + consts.append(value) + return i for pos, end, _ in labels.values(): _inject_nop_sled(buf, pos, end) - for pos, end, label, origin_stack in gotos: + for pos, end, label, origin_stack, is_into in gotos: try: _, target, target_stack = labels[label] except KeyError: raise SyntaxError('Unknown label {0!r}'.format(code.co_names[label])) - target_depth = len(target_stack) - if origin_stack[:target_depth] != target_stack: - raise SyntaxError('Jump into different block') - ops = [] - for block, _ in reversed(origin_stack[target_depth:]): + + target_depth = len(target_stack) + for block, _, _ in reversed(origin_stack[target_depth:]): if block == 'FOR_ITER': ops.append('POP_TOP') elif block == '': @@ -301,31 +343,37 @@ def _patch_code(code): ops.append('POP_TOP') # pypy 3.6 keeps a block around until END_FINALLY; python 3.8 reuses SETUP_FINALLY for SETUP_EXCEPT (where END_FINALLY is not accepted). What will pypy 3.8 do? if __pypy__ and block in ('SETUP_FINALLY', 'SETUP_WITH', 'SETUP_ASYNC_WITH'): - ops.append(('LOAD_CONST', code.co_consts.index(None))) + ops.append(('LOAD_CONST', get_const(None))) ops.append('END_FINALLY') - ops.append(('JUMP_ABSOLUTE', target // _BYTECODE.jump_unit)) - - size = _get_instructions_size(ops) - if pos + size > end: - # not enough space, add code at buffer end and jump there & back - buf_end = len(buf) - - go_to_end_ops = [('JUMP_ABSOLUTE', buf_end // _BYTECODE.jump_unit)] - - if pos + _get_instructions_size(go_to_end_ops) > end: - raise SyntaxError('Goto in an incredibly huge function') # not sure if reachable + + if origin_stack[:target_depth] != target_stack: + if not is_into: + raise SyntaxError('Jump into different block without "into" syntax') - pos = _write_instructions(buf, pos, go_to_end_ops) - _inject_nop_sled(buf, pos, end) + if temp_var is None: + temp_var = len(varnames) + varnames += ('goto.into.temp',) - buf.extend([0] * size) - _write_instructions(buf, buf_end, ops) + origin_depth = len(origin_stack) + ops.append(('STORE_FAST', temp_var)) # each block must see the right amount of data on the stack - else: - pos = _write_instructions(buf, pos, ops) - _inject_nop_sled(buf, pos, end) + tuple_i = 0 + for block, blockarg, _ in target_stack[origin_depth:]: + if block in ('SETUP_LOOP', 'FOR_ITER'): + if block != 'FOR_ITER': + ops.append((block, blockarg)) + ops.append(('LOAD_FAST', temp_var)) + ops.append(('LOAD_CONST', get_const(tuple_i))) + ops.append('BINARY_SUBSCR') + tuple_i += 1 + else: + raise SyntaxError('Being worked on...') + + ops.append(('JUMP_ABSOLUTE', target // _BYTECODE.jump_unit)) + + _inject_ops(buf, pos, end, ops) - new_code = _make_code(code, _array_to_bytes(buf)) + new_code = _make_code(code, _array_to_bytes(buf), varnames, consts) _patched_code_cache[code] = new_code return new_code diff --git a/test_goto.py b/test_goto.py index c8d51b5..ad316dc 100644 --- a/test_goto.py +++ b/test_goto.py @@ -96,6 +96,17 @@ def func(): pytest.raises(SyntaxError, with_goto, func) + +def test_jump_into_loop_params(): + @with_goto + def func(): + goto.into .loop = iter(range(5)), + for i in range(10): + label .loop + return i + + assert func() == 4 + def test_jump_out_of_nested_2_loops(): @with_goto def func(): From b0361b5e1545d12c2e96a5222a8e7f18d62cb554 Mon Sep 17 00:00:00 2001 From: condut Date: Fri, 13 Dec 2019 20:47:08 +0200 Subject: [PATCH 12/48] Amend previous commit... --- goto.py | 2 ++ test_goto.py | 35 +++++++++++++++++++++++++++++++++-- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/goto.py b/goto.py index f11ec98..48efbee 100644 --- a/goto.py +++ b/goto.py @@ -366,6 +366,8 @@ def get_const(value): ops.append(('LOAD_CONST', get_const(tuple_i))) ops.append('BINARY_SUBSCR') tuple_i += 1 + if block in ('SETUP_LOOP', 'FOR_ITER'): + ops.append('GET_ITER') # ensure the stack item is an iter, to avoid FOR_ITER crashing. Side-effect: convert iterables to iterators else: raise SyntaxError('Being worked on...') diff --git a/test_goto.py b/test_goto.py index ad316dc..b45caac 100644 --- a/test_goto.py +++ b/test_goto.py @@ -97,15 +97,46 @@ def func(): pytest.raises(SyntaxError, with_goto, func) -def test_jump_into_loop_params(): +def test_jump_into_loop_iter_params(): @with_goto def func(): - goto.into .loop = iter(range(5)), + my_iter = iter(range(5)) + goto.into .loop = my_iter, + for i in range(10): + label .loop + return i, sum(1 for _ in my_iter) + + assert func() == (4, 0) + +def test_jump_into_loop_iterable_params(): # wasn't planned on being accepted, but works due to an implemenation detail + @with_goto + def func(): + goto.into .loop = range(5), for i in range(10): label .loop return i assert func() == 4 + +def test_jump_into_loop_bad_params(): + @with_goto + def func(): + goto.into .loop = 1, + for i in range(10): + label .loop + return i + + pytest.raises(TypeError, func) + +def test_jump_into_loop_bad_param_format(): + @with_goto + def func(): + goto.into .loop = iter(range(5)) + for i in range(10): + label .loop + return i + + pytest.raises(TypeError, func) def test_jump_out_of_nested_2_loops(): @with_goto From d273711efc2fbd5d2a26155324b12af055843b42 Mon Sep 17 00:00:00 2001 From: condut Date: Fri, 13 Dec 2019 20:55:12 +0200 Subject: [PATCH 13/48] More tests --- test_goto.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/test_goto.py b/test_goto.py index b45caac..8d5c87e 100644 --- a/test_goto.py +++ b/test_goto.py @@ -138,6 +138,45 @@ def func(): pytest.raises(TypeError, func) +def test_jump_into_loop_params_with_index(): + @with_goto + def func(): + lst = [] + i = -1 + goto.into .loop = iter(range(5)), + for i in range(10): + label .loop + lst.append(i) + return lst + + assert func() == [-1, 0, 1, 2, 3, 4] + +def test_jump_into_loop_params_without_index(): + @with_goto + def func(): + lst = [] + goto.into .loop = iter(range(5)), + for i in range(10): + label .loop + lst.append(i) + return lst + + pytest.raises(UnboundLocalError, func) + +def test_jump_into_2_loops_params_and_live(): + @with_goto + def func(): + for i in range(3): + c = 0 + goto.into .loop = iter(range(3)), iter(range(10)) + for j in None: + for k in range(2): + label .loop + c += 1 + return i, c + + assert func() == (2, 11 + 6) + def test_jump_out_of_nested_2_loops(): @with_goto def func(): From 2312a3fc69a1ebb24057aaedfb93d76c2c92209c Mon Sep 17 00:00:00 2001 From: condut Date: Fri, 13 Dec 2019 21:08:57 +0200 Subject: [PATCH 14/48] Add and fix tests for cross-loop jumping with params --- goto.py | 62 +++++++++++++++++++++++++++++----------------------- test_goto.py | 40 +++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 27 deletions(-) diff --git a/goto.py b/goto.py index 48efbee..f13eb68 100644 --- a/goto.py +++ b/goto.py @@ -16,6 +16,10 @@ except AttributeError: _array_to_bytes = array.array.tostring +try: + _range = xrange +except NameError: + _range = range class _Bytecode: def __init__(self): @@ -109,7 +113,7 @@ def _parse_instructions(code, yield_nones_at_end=0): extended_arg_offset = None yield (dis.opname[opcode], oparg, offset) - for _ in range(yield_nones_at_end): + for _ in _range(yield_nones_at_end): yield (None, None, None) def _get_instruction_size(opname, oparg=0): @@ -329,8 +333,23 @@ def get_const(value): ops = [] - target_depth = len(target_stack) - for block, _, _ in reversed(origin_stack[target_depth:]): + common_depth = min(len(origin_stack), len(target_stack)) + for i in _range(common_depth): + if origin_stack[i] != target_stack[i]: + common_depth = i + break + + if is_into: + if temp_var is None: + temp_var = len(varnames) + varnames += ('goto.into.temp',) + + ops.append(('STORE_FAST', temp_var)) # must do this before any blocks are pushed/popped + + elif common_depth < len(target_stack): + raise SyntaxError('Jump into different block without "into" syntax') + + for block, _, _ in reversed(origin_stack[common_depth:]): if block == 'FOR_ITER': ops.append('POP_TOP') elif block == '': @@ -345,31 +364,20 @@ def get_const(value): if __pypy__ and block in ('SETUP_FINALLY', 'SETUP_WITH', 'SETUP_ASYNC_WITH'): ops.append(('LOAD_CONST', get_const(None))) ops.append('END_FINALLY') - - if origin_stack[:target_depth] != target_stack: - if not is_into: - raise SyntaxError('Jump into different block without "into" syntax') - - if temp_var is None: - temp_var = len(varnames) - varnames += ('goto.into.temp',) - - origin_depth = len(origin_stack) - ops.append(('STORE_FAST', temp_var)) # each block must see the right amount of data on the stack - - tuple_i = 0 - for block, blockarg, _ in target_stack[origin_depth:]: + + tuple_i = 0 + for block, blockarg, _ in target_stack[common_depth:]: + if block in ('SETUP_LOOP', 'FOR_ITER'): + if block != 'FOR_ITER': + ops.append((block, blockarg)) + ops.append(('LOAD_FAST', temp_var)) + ops.append(('LOAD_CONST', get_const(tuple_i))) + ops.append('BINARY_SUBSCR') + tuple_i += 1 if block in ('SETUP_LOOP', 'FOR_ITER'): - if block != 'FOR_ITER': - ops.append((block, blockarg)) - ops.append(('LOAD_FAST', temp_var)) - ops.append(('LOAD_CONST', get_const(tuple_i))) - ops.append('BINARY_SUBSCR') - tuple_i += 1 - if block in ('SETUP_LOOP', 'FOR_ITER'): - ops.append('GET_ITER') # ensure the stack item is an iter, to avoid FOR_ITER crashing. Side-effect: convert iterables to iterators - else: - raise SyntaxError('Being worked on...') + ops.append('GET_ITER') # ensure the stack item is an iter, to avoid FOR_ITER crashing. Side-effect: convert iterables to iterators + else: + raise SyntaxError('Being worked on...') ops.append(('JUMP_ABSOLUTE', target // _BYTECODE.jump_unit)) diff --git a/test_goto.py b/test_goto.py index 8d5c87e..6e73624 100644 --- a/test_goto.py +++ b/test_goto.py @@ -283,6 +283,46 @@ def func(): pytest.raises(SyntaxError, with_goto, func) +def test_jump_across_loops_with_params(): + @with_goto + def func(): + for i in range(10): + goto.into .other_loop = iter(range(3)), + + for i in range(10): + label .other_loop + + return i + + assert func() == 2 + +def test_jump_across_loops_with_params_and_live(): + @with_goto + def func(): + for i in range(5): + for j in range(10): + for k in range(10): + goto.into .other_loop = iter(range(3)), + + for j in range(10): + label .other_loop + + return (i, j) + + assert func() == (4, 2) + +def test_jump_into_with_unneeded_params_and_live(): + @with_goto + def func(): + for i in range(10): + j = 0 + goto.into .not_loop = () + j = 1 + label .not_loop + return (i, j) + + assert func() == (9, 0) + class Context: def __init__(self): self.enters = 0 From d78a7b344418207840a9fc9e905825d0bc6fbf66 Mon Sep 17 00:00:00 2001 From: condut Date: Fri, 13 Dec 2019 22:19:00 +0200 Subject: [PATCH 15/48] Added jump into except/finally blocks Renamed goto.into to goto.params (more appropriate, maybe) Fixed two dumb dumb bugs that cancelled each other out (in most cases...) --- goto.py | 150 +++++++++++++++++++++++++++++---------------------- test_goto.py | 124 ++++++++++++++++++++++++++++++++++++------ 2 files changed, 194 insertions(+), 80 deletions(-) diff --git a/goto.py b/goto.py index f13eb68..d9b5268 100644 --- a/goto.py +++ b/goto.py @@ -6,11 +6,6 @@ import weakref import warnings -try: - import __pypy__ -except: - __pypy__ = None - try: _array_to_bytes = array.array.tobytes except AttributeError: @@ -48,6 +43,13 @@ def __init__(self): self.has_pop_except = 'POP_EXCEPT' in dis.opmap self.has_setup_with = 'SETUP_WITH' in dis.opmap self.has_setup_except = 'SETUP_EXCEPT' in dis.opmap + self.has_begin_finally = 'BEGIN_FINALLY' in dis.opmap + + try: + import __pypy__ + self.pypy_finally_semantics = True + except: + self.pypy_finally_semantics = False @property def argument_bits(self): @@ -211,60 +213,71 @@ def pop_block_of_type(type): else: _warn_bug("mismatched block type") pop_block() + + jump_targets = set(dis.findlabels(code.co_code)) + dead = False for opname4, oparg4, offset4 in _parse_instructions(code.co_code, 3): - # check for special opcodes - if opname1 in ('LOAD_GLOBAL', 'LOAD_NAME'): - if opname2 == 'LOAD_ATTR' and opname3 == 'POP_TOP': - name = code.co_names[oparg1] - if name == 'label': - labels[oparg2] = (offset1, + if offset1 in jump_targets: + dead = False + + if not dead: + endoffset1 = offset2 + + # check for special offsets + if for_exits and offset1 == for_exits[-1]: + pop_block() + for_exits.pop() + if excepts and offset1 == excepts[-1]: + block_counter = push_block('') + excepts.pop() + if finallies and offset1 == finallies[-1]: + block_counter = push_block('') + finallies.pop() + + # check for special opcodes + if opname1 in ('LOAD_GLOBAL', 'LOAD_NAME'): + if opname2 == 'LOAD_ATTR' and opname3 == 'POP_TOP': + name = code.co_names[oparg1] + if name == 'label': + labels[oparg2] = (offset1, + offset4, + list(block_stack)) + elif name == 'goto': + gotos.append((offset1, + offset4, + oparg2, + list(block_stack), + False)) + elif opname2 == 'LOAD_ATTR' and opname3 == 'STORE_ATTR': + if code.co_names[oparg1] == 'goto' and code.co_names[oparg2] == 'params': + gotos.append((offset1, offset4, - list(block_stack)) - elif name == 'goto': - gotos.append((offset1, - offset4, - oparg2, - list(block_stack), - False)) - elif opname2 == 'LOAD_ATTR' and opname3 == 'STORE_ATTR': - if code.co_names[oparg1] == 'goto' and code.co_names[oparg2] == 'into': - gotos.append((offset1, - offset4, - oparg3, - list(block_stack), - True)) - elif opname1 in ('SETUP_LOOP', - 'SETUP_EXCEPT', 'SETUP_FINALLY', - 'SETUP_WITH', 'SETUP_ASYNC_WITH'): - block_counter = push_block(opname1, oparg1) - if opname1 == 'SETUP_EXCEPT' and _BYTECODE.has_pop_except: - excepts.append(offset1 + oparg1) - elif opname1 == 'SETUP_FINALLY': - finallies.append(offset1 + oparg1) - elif not _BYTECODE.has_loop_blocks and opname1 == 'FOR_ITER': - block_counter = push_block(opname1, oparg1) - for_exits.append(offset1 + oparg1) - elif opname1 == 'POP_BLOCK': - pop_block() - elif opname1 == 'POP_EXCEPT': - pop_block_of_type('') - elif opname1 == 'END_FINALLY': - if opname0 != 'JUMP_FORWARD': # hack for dummy end-finally in except block (correct fix would be a jump-aware reading of instructions!) + oparg3, + list(block_stack), + True)) + elif opname1 in ('SETUP_LOOP', + 'SETUP_EXCEPT', 'SETUP_FINALLY', + 'SETUP_WITH', 'SETUP_ASYNC_WITH'): + block_counter = push_block(opname1, oparg1) + if opname1 == 'SETUP_EXCEPT' and _BYTECODE.has_pop_except: + excepts.append(endoffset1 + oparg1) + elif opname1 == 'SETUP_FINALLY': + finallies.append(endoffset1 + oparg1) + elif not _BYTECODE.has_loop_blocks and opname1 == 'FOR_ITER': + block_counter = push_block(opname1, oparg1) + for_exits.append(endoffset1 + oparg1) + elif opname1 == 'POP_BLOCK': + pop_block() + elif opname1 == 'POP_EXCEPT': + pop_block_of_type('') + elif opname1 == 'END_FINALLY': pop_block_of_type('') - elif opname1 in ('WITH_CLEANUP', 'WITH_CLEANUP_START') and _BYTECODE.has_setup_with: - block_counter = push_block('') # temporary block to match END_FINALLY + elif opname1 in ('WITH_CLEANUP', 'WITH_CLEANUP_START') and _BYTECODE.has_setup_with: + block_counter = push_block('') # temporary block to match END_FINALLY - # check for special offsets - if for_exits and offset1 == for_exits[-1]: - pop_block() - for_exits.pop() - if excepts and offset1 == excepts[-1]: - block_counter = push_block('') - excepts.pop() - if finallies and offset1 == finallies[-1]: - block_counter = push_block('') - finallies.pop() + if opname1 in ('JUMP_ABSOLUTE', 'JUMP_FORWARD'): + dead = True opname0, oparg0, offset0 = opname1, oparg1, offset1 opname1, oparg1, offset1 = opname2, oparg2, offset2 @@ -325,7 +338,7 @@ def get_const(value): for pos, end, _ in labels.values(): _inject_nop_sled(buf, pos, end) - for pos, end, label, origin_stack, is_into in gotos: + for pos, end, label, origin_stack, has_params in gotos: try: _, target, target_stack = labels[label] except KeyError: @@ -339,15 +352,12 @@ def get_const(value): common_depth = i break - if is_into: + if has_params: if temp_var is None: temp_var = len(varnames) - varnames += ('goto.into.temp',) + varnames += ('goto.params',) - ops.append(('STORE_FAST', temp_var)) # must do this before any blocks are pushed/popped - - elif common_depth < len(target_stack): - raise SyntaxError('Jump into different block without "into" syntax') + ops.append(('STORE_FAST', temp_var)) # must do this before any blocks are pushed/popped for block, _, _ in reversed(origin_stack[common_depth:]): if block == 'FOR_ITER': @@ -361,13 +371,15 @@ def get_const(value): if block in ('SETUP_WITH', 'SETUP_ASYNC_WITH'): ops.append('POP_TOP') # pypy 3.6 keeps a block around until END_FINALLY; python 3.8 reuses SETUP_FINALLY for SETUP_EXCEPT (where END_FINALLY is not accepted). What will pypy 3.8 do? - if __pypy__ and block in ('SETUP_FINALLY', 'SETUP_WITH', 'SETUP_ASYNC_WITH'): - ops.append(('LOAD_CONST', get_const(None))) + if _BYTECODE.pypy_finally_semantics and block in ('SETUP_FINALLY', 'SETUP_WITH', 'SETUP_ASYNC_WITH'): + ops.append('BEGIN_FINALLY' if _BYTECODE.has_begin_finally else ('LOAD_CONST', get_const(None))) ops.append('END_FINALLY') tuple_i = 0 for block, blockarg, _ in target_stack[common_depth:]: if block in ('SETUP_LOOP', 'FOR_ITER'): + if not has_params: + raise SyntaxError('Jump into block without the necessary params') if block != 'FOR_ITER': ops.append((block, blockarg)) ops.append(('LOAD_FAST', temp_var)) @@ -376,6 +388,18 @@ def get_const(value): tuple_i += 1 if block in ('SETUP_LOOP', 'FOR_ITER'): ops.append('GET_ITER') # ensure the stack item is an iter, to avoid FOR_ITER crashing. Side-effect: convert iterables to iterators + elif block == '': + if _BYTECODE.pypy_finally_semantics: + ops.append('SETUP_FINALLY') + ops.append('POP_BLOCK') + ops.append('BEGIN_FINALLY' if _BYTECODE.has_begin_finally else ('LOAD_CONST', get_const(None))) + elif block == '': + # we raise an exception to get the right block pushed + raise_ops = [('LOAD_CONST', get_const(None)), ('RAISE_VARARGS', 1)] + ops.append(('SETUP_EXCEPT' if _BYTECODE.has_setup_except else 'SETUP_FINALLY', _get_instructions_size(raise_ops))) + ops += raise_ops + for _ in _range(3): + ops.append("POP_TOP") else: raise SyntaxError('Being worked on...') diff --git a/test_goto.py b/test_goto.py index 6e73624..aa3121d 100644 --- a/test_goto.py +++ b/test_goto.py @@ -101,17 +101,17 @@ def test_jump_into_loop_iter_params(): @with_goto def func(): my_iter = iter(range(5)) - goto.into .loop = my_iter, + goto.params .loop = my_iter, for i in range(10): label .loop return i, sum(1 for _ in my_iter) assert func() == (4, 0) -def test_jump_into_loop_iterable_params(): # wasn't planned on being accepted, but works due to an implemenation detail +def test_jump_into_loop_iterable_params(): # wasn't planned on being accepted, but works due to an implementation detail, and... probably for the best @with_goto def func(): - goto.into .loop = range(5), + goto.params .loop = range(5), for i in range(10): label .loop return i @@ -121,7 +121,7 @@ def func(): def test_jump_into_loop_bad_params(): @with_goto def func(): - goto.into .loop = 1, + goto.params .loop = 1, for i in range(10): label .loop return i @@ -131,7 +131,7 @@ def func(): def test_jump_into_loop_bad_param_format(): @with_goto def func(): - goto.into .loop = iter(range(5)) + goto.params .loop = iter(range(5)) for i in range(10): label .loop return i @@ -143,7 +143,7 @@ def test_jump_into_loop_params_with_index(): def func(): lst = [] i = -1 - goto.into .loop = iter(range(5)), + goto.params .loop = iter(range(5)), for i in range(10): label .loop lst.append(i) @@ -155,7 +155,7 @@ def test_jump_into_loop_params_without_index(): @with_goto def func(): lst = [] - goto.into .loop = iter(range(5)), + goto.params .loop = iter(range(5)), for i in range(10): label .loop lst.append(i) @@ -168,7 +168,7 @@ def test_jump_into_2_loops_params_and_live(): def func(): for i in range(3): c = 0 - goto.into .loop = iter(range(3)), iter(range(10)) + goto.params .loop = iter(range(3)), iter(range(10)) for j in None: for k in range(2): label .loop @@ -287,7 +287,7 @@ def test_jump_across_loops_with_params(): @with_goto def func(): for i in range(10): - goto.into .other_loop = iter(range(3)), + goto.params .other_loop = iter(range(3)), for i in range(10): label .other_loop @@ -302,7 +302,7 @@ def func(): for i in range(5): for j in range(10): for k in range(10): - goto.into .other_loop = iter(range(3)), + goto.params .other_loop = iter(range(3)), for j in range(10): label .other_loop @@ -316,7 +316,7 @@ def test_jump_into_with_unneeded_params_and_live(): def func(): for i in range(10): j = 0 - goto.into .not_loop = () + goto.params .not_loop = () j = 1 label .not_loop return (i, j) @@ -366,6 +366,16 @@ def func(): goto .block pytest.raises(SyntaxError, with_goto, func) + +"""def test_jump_into_with_block_with_params(): + def func(): + c = Context() + goto.params .block = c + with 123 as c: + label .block + return c.data() + + assert func() == (0, 1)""" def test_generator(): @with_goto @@ -472,16 +482,34 @@ def func(): assert func() == (2, 0, 'except') -"""def test_jump_into_except_block(): +def test_jump_into_except_block(): + @with_goto def func(): + i = 1 + goto .block try: - pass + i = 2 except: label .block - pass - goto .block - - pytest.raises(SyntaxError, with_goto, func)""" + i = 3 + return i + + assert func() == 3 + +def test_jump_into_except_block_and_live(): + @with_goto + def func(): + for i in range(10): + j = 1 + goto .block + try: + j = 2 + except: + label .block + j = 3 + return i, j + + assert func() == (9, 3) def test_jump_out_of_finally_block(): @with_goto @@ -539,6 +567,34 @@ def func(): assert func() == (2, 0, 'try') +def test_jump_into_finally_block(): + @with_goto + def func(): + rv = 1 + goto .fin + try: + rv = 1 / 0 + finally: + label .fin + rv = 2 + return rv + + assert func() == 2 + +def test_jump_into_finally_block_and_live(): + @with_goto + def func(): + for i in range(3): + rv = 1 + goto .fin + try: + rv = 1 / 0 + finally: + label .fin + rv = 2 + return i, rv + + assert func() == (2, 2) def test_jump_to_unknown_label(): def func(): @@ -547,6 +603,40 @@ def func(): pytest.raises(SyntaxError, with_goto, func) +def test_jump_with_for_break(): # to see it doesn't confuse parser + @with_goto + def func(): + for i in range(4): + goto .x + break + label .x + return i + + assert func() == 0 + +def test_jump_with_for_continue(): # to see it doesn't confuse parser + @with_goto + def func(): + for i in range(4): + goto .x + continue + label .x + return i + + assert func() == 0 + +def test_jump_with_for_return(): # to see it doesn't confuse parser + @with_goto + def func(): + for i in range(4): + goto .x + return + label .x + return i + + assert func() == 0 + + def test_function_is_copy(): def func(): pass From 7de176d950c0b3089028012cda89d2bcf846788a Mon Sep 17 00:00:00 2001 From: condut Date: Fri, 13 Dec 2019 23:01:48 +0200 Subject: [PATCH 16/48] Added goto to try block & fix py26 goto out of with bug --- goto.py | 24 +++++---- test_goto.py | 136 +++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 146 insertions(+), 14 deletions(-) diff --git a/goto.py b/goto.py index d9b5268..a08ca49 100644 --- a/goto.py +++ b/goto.py @@ -177,6 +177,7 @@ def _find_labels_and_gotos(code): for_exits = [] excepts = [] finallies = [] + last_block = None opname0 = oparg0 = offset0 = None opname1 = oparg1 = offset1 = None # the main one we're looking at each loop iteration @@ -201,7 +202,7 @@ def push_block(opname, oparg=0): def pop_block(): if block_stack: - block_stack.pop() + return block_stack.pop() else: _warn_bug("can't pop block") @@ -212,7 +213,7 @@ def pop_block_of_type(type): replace_block(block_stack[-1], (type,) + block_stack[-1][1:]) else: _warn_bug("mismatched block type") - pop_block() + return pop_block() jump_targets = set(dis.findlabels(code.co_code)) dead = False @@ -226,7 +227,7 @@ def pop_block_of_type(type): # check for special offsets if for_exits and offset1 == for_exits[-1]: - pop_block() + last_block = pop_block() for_exits.pop() if excepts and offset1 == excepts[-1]: block_counter = push_block('') @@ -268,13 +269,16 @@ def pop_block_of_type(type): block_counter = push_block(opname1, oparg1) for_exits.append(endoffset1 + oparg1) elif opname1 == 'POP_BLOCK': - pop_block() + last_block = pop_block() elif opname1 == 'POP_EXCEPT': - pop_block_of_type('') + last_block = pop_block_of_type('') elif opname1 == 'END_FINALLY': - pop_block_of_type('') - elif opname1 in ('WITH_CLEANUP', 'WITH_CLEANUP_START') and _BYTECODE.has_setup_with: - block_counter = push_block('') # temporary block to match END_FINALLY + last_block = pop_block_of_type('') + elif opname1 in ('WITH_CLEANUP', 'WITH_CLEANUP_START'): + if _BYTECODE.has_setup_with: + block_counter = push_block('') # temporary block to match END_FINALLY + else: + replace_block(last_block, ('SETUP_WITH',) + last_block[1:]) # python 2.6 - finally was actually with if opname1 in ('JUMP_ABSOLUTE', 'JUMP_FORWARD'): dead = True @@ -388,6 +392,8 @@ def get_const(value): tuple_i += 1 if block in ('SETUP_LOOP', 'FOR_ITER'): ops.append('GET_ITER') # ensure the stack item is an iter, to avoid FOR_ITER crashing. Side-effect: convert iterables to iterators + elif block in ('SETUP_EXCEPT', 'SETUP_FINALLY'): + ops.append((block, blockarg)) elif block == '': if _BYTECODE.pypy_finally_semantics: ops.append('SETUP_FINALLY') @@ -399,7 +405,7 @@ def get_const(value): ops.append(('SETUP_EXCEPT' if _BYTECODE.has_setup_except else 'SETUP_FINALLY', _get_instructions_size(raise_ops))) ops += raise_ops for _ in _range(3): - ops.append("POP_TOP") + ops.append("POP_TOP") else: raise SyntaxError('Being worked on...') diff --git a/test_goto.py b/test_goto.py index aa3121d..e680512 100644 --- a/test_goto.py +++ b/test_goto.py @@ -2,6 +2,8 @@ import pytest from goto import with_goto +NonConstFalse = False + CODE = '''\ i = 0 result = [] @@ -247,6 +249,39 @@ def func(): assert func() == (1, 0, 0, 0, 0) +def test_large_jumps_in_diff_orders(): + @with_goto + def func(): + goto .start + + if NonConstFalse: + label .finalle + return (i, j, k, m, n, i1, j1, k1, m1, n1, i2, j2, k2, m2, n2) + + label .start + for i in range(2): + for j in range(2): + for k in range(2): + for m in range(2): + for n in range(2): + goto .end + label .end + for i1 in range(2): + for j1 in range(2): + for k1 in range(2): + for m1 in range(2): + for n1 in range(2): + goto .end2 + label .end2 + for i2 in range(2): + for j2 in range(2): + for k2 in range(2): + for m2 in range(2): + for n2 in range(2): + goto .finalle + + assert func() == (0,) * 15 + def test_jump_out_of_nested_11_loops(): @with_goto def func(): @@ -346,6 +381,18 @@ def func(): assert func()== (1, 0) +def test_jump_out_of_with_block_and_survive(): + @with_goto + def func(): + c = Context() + for i in range(3): + with c: + goto .out + label .out + return (i, c.data()) + + assert func() == (2, (3, 0)) + def test_jump_out_of_with_block_and_live(): @with_goto def func(): @@ -391,6 +438,32 @@ def func(): assert tuple(func()) == (0, 1, 4, 5) +def test_jump_out_of_try_except_block(): + @with_goto + def func(): + try: + rv = None + goto .end + except: + rv = 'except' + label .end + return rv + + assert func() == None + +def test_jump_out_of_try_finally_block(): + @with_goto + def func(): + try: + rv = None + goto .end + finally: + rv = 'finally' + label .end + return rv + + assert func() == None + def test_jump_out_of_try_block(): @with_goto def func(): @@ -438,17 +511,70 @@ def func(): return (i, j, rv) assert func() == (2, 2, None) - + def test_jump_into_try_block(): + @with_goto def func(): + rv = 0 + goto .block try: + rv = 1 label .block except: - pass - goto .block - - pytest.raises(SyntaxError, with_goto, func) + rv = 2 + finally: + rv = 3 + return rv + + assert func() == 3 +def test_jump_into_try_except_block_and_survive(): + @with_goto + def func(): + for i in range(10): + rv = 0 + goto .block + try: + rv = 1 + label .block + except: + rv = 2 + return i, rv + + assert func() == (9, 0) + +def test_jump_into_try_finally_block_and_survive(): + @with_goto + def func(): + for i in range(10): + rv, fv = 0, 0 + goto .block + try: + rv = 1 + label .block + finally: + fv = 1 + return i, rv, fv + + assert func() == (9, 0, 1) + +def test_jump_into_try_block_and_survive(): + @with_goto + def func(): + for i in range(10): + rv, fv = 0, 0 + goto .block + try: + rv = 1 + label .block + except: + rv = 2 + finally: + fv = 1 + return i, rv, fv + + assert func() == (9, 0, 1) + def test_jump_out_of_except_block(): @with_goto From d52c7e0cfb11ce97e1e61d2086926a294e26d858 Mon Sep 17 00:00:00 2001 From: condut Date: Fri, 13 Dec 2019 23:43:25 +0200 Subject: [PATCH 17/48] And finally(?), jump into with support! --- goto.py | 85 ++++++++++++++++++++++++++++++++++------------------ test_goto.py | 80 +++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 133 insertions(+), 32 deletions(-) diff --git a/goto.py b/goto.py index a08ca49..7063a38 100644 --- a/goto.py +++ b/goto.py @@ -64,18 +64,16 @@ def argument_bits(self): except TypeError: _patched_code_cache = {} # ...unless not supported -def _make_code(code, codestring, varnames, consts): - nlocals = len(varnames) - consts = tuple(consts) - +def _make_code(code, codestring, data): try: # code.replace is new in 3.8+ - return code.replace(co_code=codestring, co_nlocals=nlocals, co_varnames=varnames, co_consts=consts) + return code.replace(co_code=codestring, co_nlocals=data.nlocals, co_varnames=data.varnames, + co_consts=data.consts, co_names=data.names) except: args = [ - code.co_argcount, nlocals, code.co_stacksize, - code.co_flags, codestring, consts, - code.co_names, varnames, code.co_filename, + code.co_argcount, data.nlocals, code.co_stacksize, + code.co_flags, codestring, data.consts, + data.names, data.varnames, code.co_filename, code.co_name, code.co_firstlineno, code.co_lnotab, code.co_freevars, code.co_cellvars ] @@ -318,6 +316,35 @@ def _inject_ops(buf, pos, end, ops): else: pos = _write_instructions(buf, pos, ops) _inject_nop_sled(buf, pos, end) + +class _CodeData: + def __init__(self, code): + self.nlocals = code.co_nlocals + self.varnames = code.co_varnames + self.consts = code.co_consts + self.names = code.co_names + + def get_const(self, value): + try: + i = self.consts.index(value) + except ValueError: + i = len(self.consts) + self.consts += (value,) + return i + + def get_name(self, value): + try: + i = self.names.index(value) + except ValueError: + i = len(self.names) + self.names += (value,) + return i + + def add_var(self, name): + idx = len(self.varnames) + self.varnames += (name,) + self.nlocals += 1 + return idx def _patch_code(code): new_code = _patched_code_cache.get(code) @@ -328,16 +355,7 @@ def _patch_code(code): buf = array.array('B', code.co_code) temp_var = None - varnames = code.co_varnames - consts = list(code.co_consts) - - def get_const(value): - try: - i = consts.index(value) - except ValueError: - i = len(consts) - consts.append(value) - return i + data = _CodeData(code) for pos, end, _ in labels.values(): _inject_nop_sled(buf, pos, end) @@ -356,10 +374,9 @@ def get_const(value): common_depth = i break - if has_params: + if has_params: if temp_var is None: - temp_var = len(varnames) - varnames += ('goto.params',) + temp_var = data.add_var('goto.params') ops.append(('STORE_FAST', temp_var)) # must do this before any blocks are pushed/popped @@ -376,7 +393,7 @@ def get_const(value): ops.append('POP_TOP') # pypy 3.6 keeps a block around until END_FINALLY; python 3.8 reuses SETUP_FINALLY for SETUP_EXCEPT (where END_FINALLY is not accepted). What will pypy 3.8 do? if _BYTECODE.pypy_finally_semantics and block in ('SETUP_FINALLY', 'SETUP_WITH', 'SETUP_ASYNC_WITH'): - ops.append('BEGIN_FINALLY' if _BYTECODE.has_begin_finally else ('LOAD_CONST', get_const(None))) + ops.append('BEGIN_FINALLY' if _BYTECODE.has_begin_finally else ('LOAD_CONST', data.get_const(None))) ops.append('END_FINALLY') tuple_i = 0 @@ -387,33 +404,43 @@ def get_const(value): if block != 'FOR_ITER': ops.append((block, blockarg)) ops.append(('LOAD_FAST', temp_var)) - ops.append(('LOAD_CONST', get_const(tuple_i))) + ops.append(('LOAD_CONST', data.get_const(tuple_i))) ops.append('BINARY_SUBSCR') tuple_i += 1 - if block in ('SETUP_LOOP', 'FOR_ITER'): - ops.append('GET_ITER') # ensure the stack item is an iter, to avoid FOR_ITER crashing. Side-effect: convert iterables to iterators + ops.append('GET_ITER') # this both converts iterables to iterators for convenience, and prevents FOR_ITER from crashing on non-iter objects. (this is a no-op for iterators) + + elif block in ('SETUP_WITH', 'SETUP_ASYNC_WITH'): + if not has_params: + raise SyntaxError('Jump into block without the necessary params') + ops.append(('LOAD_FAST', temp_var)) + ops.append(('LOAD_CONST', data.get_const(tuple_i))) + ops.append('BINARY_SUBSCR') + ops.append(('LOAD_ATTR', data.get_name('__exit__'))) + # SETUP_WITH executes __enter__ and so is inappropriate here (a goto must bypass any and all side-effects) + ops.append(('SETUP_FINALLY', blockarg)) + elif block in ('SETUP_EXCEPT', 'SETUP_FINALLY'): ops.append((block, blockarg)) elif block == '': if _BYTECODE.pypy_finally_semantics: ops.append('SETUP_FINALLY') ops.append('POP_BLOCK') - ops.append('BEGIN_FINALLY' if _BYTECODE.has_begin_finally else ('LOAD_CONST', get_const(None))) + ops.append('BEGIN_FINALLY' if _BYTECODE.has_begin_finally else ('LOAD_CONST', data.get_const(None))) elif block == '': # we raise an exception to get the right block pushed - raise_ops = [('LOAD_CONST', get_const(None)), ('RAISE_VARARGS', 1)] + raise_ops = [('LOAD_CONST', data.get_const(None)), ('RAISE_VARARGS', 1)] ops.append(('SETUP_EXCEPT' if _BYTECODE.has_setup_except else 'SETUP_FINALLY', _get_instructions_size(raise_ops))) ops += raise_ops for _ in _range(3): ops.append("POP_TOP") else: - raise SyntaxError('Being worked on...') + _warn_bug("ignoring %s" % block) ops.append(('JUMP_ABSOLUTE', target // _BYTECODE.jump_unit)) _inject_ops(buf, pos, end, ops) - new_code = _make_code(code, _array_to_bytes(buf), varnames, consts) + new_code = _make_code(code, _array_to_bytes(buf), data) _patched_code_cache[code] = new_code return new_code diff --git a/test_goto.py b/test_goto.py index e680512..5396f4c 100644 --- a/test_goto.py +++ b/test_goto.py @@ -179,6 +179,26 @@ def func(): assert func() == (2, 11 + 6) +def test_jump_out_then_back_in_for_loop_and_survive(): + @with_goto + def func(): + it1 = iter(range(5)) + cc = 0 + for i in it1: + it2 = iter(range(4)) + for j in it2: + goto .out + raise None + label .back + + if NonConstFalse: + label .out + cc += 1 + goto.params .back = it1, it2 + return cc, i, j + + assert func() == (20, 4, 3) + def test_jump_out_of_nested_2_loops(): @with_goto def func(): @@ -414,15 +434,69 @@ def func(): pytest.raises(SyntaxError, with_goto, func) -"""def test_jump_into_with_block_with_params(): +def test_jump_into_with_block_with_params(): + @with_goto def func(): c = Context() - goto.params .block = c + goto.params .block = c, with 123 as c: label .block return c.data() - assert func() == (0, 1)""" + assert func() == (0, 1) + +def test_jump_into_with_block_with_params_and_survive(): + @with_goto + def func(): + c = Context() + for i in range(10): + goto.params .block = c, + with 123 as c: + label .block + return i, c.data() + + assert func() == (9, (0, 10)) + +def test_jump_into_with_block_with_bad_params(): + @with_goto + def func(): + with Context() as c: + label .block + goto.params .block = 123, + + pytest.raises(AttributeError, func) + +def test_jump_into_with_block_with_bad_exit_params(): + class BadAttr: + __exit__ = 123 + + @with_goto + def func(): + with Context() as c: + label .block + goto.params .block = BadAttr, + + pytest.raises(TypeError, func) + +def test_jump_out_of_then_back_into_with_block_with_params_and_survive(): + @with_goto + def func(): + c = Context() + cc = 0 + for i in range(10): + with c: + goto .out + cc -= 100 + label .back + + if NonConstFalse: + label .out + cc += 1 + goto.params .back = c, + + return i, cc, c.data() + + assert func() == (9, 10, (10, 10)) def test_generator(): @with_goto From a75a912e19692a57bfcd9646ade05f1e59aa8d40 Mon Sep 17 00:00:00 2001 From: condut Date: Sat, 14 Dec 2019 00:09:40 +0200 Subject: [PATCH 18/48] Added support to goto.param flavor & fix stupid bug with with. --- goto.py | 48 ++++++++++++++++---------------- test_goto.py | 77 +++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 85 insertions(+), 40 deletions(-) diff --git a/goto.py b/goto.py index 7063a38..269baf3 100644 --- a/goto.py +++ b/goto.py @@ -247,14 +247,15 @@ def pop_block_of_type(type): offset4, oparg2, list(block_stack), - False)) + 0)) elif opname2 == 'LOAD_ATTR' and opname3 == 'STORE_ATTR': - if code.co_names[oparg1] == 'goto' and code.co_names[oparg2] == 'params': + if code.co_names[oparg1] == 'goto' and code.co_names[oparg2] in ('param', 'params'): gotos.append((offset1, offset4, oparg3, list(block_stack), - True)) + code.co_names[oparg2])) + elif opname1 in ('SETUP_LOOP', 'SETUP_EXCEPT', 'SETUP_FINALLY', 'SETUP_WITH', 'SETUP_ASYNC_WITH'): @@ -266,6 +267,7 @@ def pop_block_of_type(type): elif not _BYTECODE.has_loop_blocks and opname1 == 'FOR_ITER': block_counter = push_block(opname1, oparg1) for_exits.append(endoffset1 + oparg1) + elif opname1 == 'POP_BLOCK': last_block = pop_block() elif opname1 == 'POP_EXCEPT': @@ -360,7 +362,7 @@ def _patch_code(code): for pos, end, _ in labels.values(): _inject_nop_sled(buf, pos, end) - for pos, end, label, origin_stack, has_params in gotos: + for pos, end, label, origin_stack, params in gotos: try: _, target, target_stack = labels[label] except KeyError: @@ -374,11 +376,12 @@ def _patch_code(code): common_depth = i break - if has_params: + if params: if temp_var is None: - temp_var = data.add_var('goto.params') + temp_var = data.add_var('goto.temp') - ops.append(('STORE_FAST', temp_var)) # must do this before any blocks are pushed/popped + ops.append(('STORE_FAST', temp_var)) # must do this before any blocks are pushed/popped + many_params = (params != 'param') for block, _, _ in reversed(origin_stack[common_depth:]): if block == 'FOR_ITER': @@ -398,26 +401,23 @@ def _patch_code(code): tuple_i = 0 for block, blockarg, _ in target_stack[common_depth:]: - if block in ('SETUP_LOOP', 'FOR_ITER'): - if not has_params: + if block in ('SETUP_LOOP', 'FOR_ITER', + 'SETUP_WITH', 'SETUP_ASYNC_WITH'): + if not params: raise SyntaxError('Jump into block without the necessary params') - if block != 'FOR_ITER': + if block == 'SETUP_LOOP': ops.append((block, blockarg)) ops.append(('LOAD_FAST', temp_var)) - ops.append(('LOAD_CONST', data.get_const(tuple_i))) - ops.append('BINARY_SUBSCR') - tuple_i += 1 - ops.append('GET_ITER') # this both converts iterables to iterators for convenience, and prevents FOR_ITER from crashing on non-iter objects. (this is a no-op for iterators) - - elif block in ('SETUP_WITH', 'SETUP_ASYNC_WITH'): - if not has_params: - raise SyntaxError('Jump into block without the necessary params') - ops.append(('LOAD_FAST', temp_var)) - ops.append(('LOAD_CONST', data.get_const(tuple_i))) - ops.append('BINARY_SUBSCR') - ops.append(('LOAD_ATTR', data.get_name('__exit__'))) - # SETUP_WITH executes __enter__ and so is inappropriate here (a goto must bypass any and all side-effects) - ops.append(('SETUP_FINALLY', blockarg)) + if many_params: + ops.append(('LOAD_CONST', data.get_const(tuple_i))) + ops.append('BINARY_SUBSCR') + tuple_i += 1 + if block in ('SETUP_LOOP', 'FOR_ITER'): + ops.append('GET_ITER') # this both converts iterables to iterators for convenience, and prevents FOR_ITER from crashing on non-iter objects. (this is a no-op for iterators) + elif block in ('SETUP_WITH', 'SETUP_ASYNC_WITH'): + # SETUP_WITH executes __enter__ and so would be inappropriate (a goto must bypass any and all side-effects) + ops.append(('LOAD_ATTR', data.get_name('__exit__'))) + ops.append(('SETUP_FINALLY', blockarg)) elif block in ('SETUP_EXCEPT', 'SETUP_FINALLY'): ops.append((block, blockarg)) diff --git a/test_goto.py b/test_goto.py index 5396f4c..6a96997 100644 --- a/test_goto.py +++ b/test_goto.py @@ -99,6 +99,17 @@ def func(): pytest.raises(SyntaxError, with_goto, func) +def test_jump_into_loop_iter_param(): + @with_goto + def func(): + my_iter = iter(range(5)) + goto.param .loop = my_iter + for i in range(10): + label .loop + return i, sum(1 for _ in my_iter) + + assert func() == (4, 0) + def test_jump_into_loop_iter_params(): @with_goto def func(): @@ -110,10 +121,10 @@ def func(): assert func() == (4, 0) -def test_jump_into_loop_iterable_params(): # wasn't planned on being accepted, but works due to an implementation detail, and... probably for the best +def test_jump_into_loop_iterable_param(): # wasn't planned on being accepted, but works due to an implementation detail, and... probably for the best @with_goto def func(): - goto.params .loop = range(5), + goto.param .loop = range(5) for i in range(10): label .loop return i @@ -123,14 +134,14 @@ def func(): def test_jump_into_loop_bad_params(): @with_goto def func(): - goto.params .loop = 1, + goto.param .loop = 1 for i in range(10): label .loop return i pytest.raises(TypeError, func) -def test_jump_into_loop_bad_param_format(): +def test_jump_into_loop_params_not_seq(): @with_goto def func(): goto.params .loop = iter(range(5)) @@ -140,12 +151,12 @@ def func(): pytest.raises(TypeError, func) -def test_jump_into_loop_params_with_index(): +def test_jump_into_loop_param_with_index(): @with_goto def func(): lst = [] i = -1 - goto.params .loop = iter(range(5)), + goto.param .loop = iter(range(5)) for i in range(10): label .loop lst.append(i) @@ -153,11 +164,11 @@ def func(): assert func() == [-1, 0, 1, 2, 3, 4] -def test_jump_into_loop_params_without_index(): +def test_jump_into_loop_param_without_index(): @with_goto def func(): lst = [] - goto.params .loop = iter(range(5)), + goto.param .loop = iter(range(5)) for i in range(10): label .loop lst.append(i) @@ -338,11 +349,11 @@ def func(): pytest.raises(SyntaxError, with_goto, func) -def test_jump_across_loops_with_params(): +def test_jump_across_loops_with_param(): @with_goto def func(): for i in range(10): - goto.params .other_loop = iter(range(3)), + goto.param .other_loop = iter(range(3)) for i in range(10): label .other_loop @@ -351,13 +362,13 @@ def func(): assert func() == 2 -def test_jump_across_loops_with_params_and_live(): +def test_jump_across_loops_with_param_and_live(): @with_goto def func(): for i in range(5): for j in range(10): for k in range(10): - goto.params .other_loop = iter(range(3)), + goto.param .other_loop = iter(range(3)) for j in range(10): label .other_loop @@ -434,6 +445,17 @@ def func(): pytest.raises(SyntaxError, with_goto, func) +def test_jump_into_with_block_with_param(): + @with_goto + def func(): + c = Context() + goto.param .block = c + with 123 as c: + label .block + return c.data() + + assert func() == (0, 1) + def test_jump_into_with_block_with_params(): @with_goto def func(): @@ -450,7 +472,7 @@ def test_jump_into_with_block_with_params_and_survive(): def func(): c = Context() for i in range(10): - goto.params .block = c, + goto.param .block = c with 123 as c: label .block return i, c.data() @@ -462,7 +484,7 @@ def test_jump_into_with_block_with_bad_params(): def func(): with Context() as c: label .block - goto.params .block = 123, + goto.param .block = 123 pytest.raises(AttributeError, func) @@ -474,7 +496,7 @@ class BadAttr: def func(): with Context() as c: label .block - goto.params .block = BadAttr, + goto.param .block = BadAttr pytest.raises(TypeError, func) @@ -492,11 +514,34 @@ def func(): if NonConstFalse: label .out cc += 1 - goto.params .back = c, + goto.param .back = c return i, cc, c.data() assert func() == (9, 10, (10, 10)) + +def test_jump_out_of_then_back_into_2_nested_with_blocks_with_params_and_survive(): + @with_goto + def func(): + c1 = Context() + c2 = Context() + cc = 0 + for i in range(11): + with c1: + if i != 10: + with c2: + goto .out + cc -= 100 + label .back + + if NonConstFalse: + label .out + cc += 1 + goto.params .back = c1, c2 + + return i, cc, c1.data(), c2.data() + + assert func() == (10, 10, (11, 11), (10, 10)) def test_generator(): @with_goto From ee35b2a1881bafb8245c97360b90c0e1c93c77f6 Mon Sep 17 00:00:00 2001 From: condut Date: Sat, 14 Dec 2019 01:15:44 +0200 Subject: [PATCH 19/48] Various style updates per initial comments Let me know what you want me to do with the long ifs and the like - the format I've chosen here may not be the one you'll want. (I haven't split things up into more functions than I already have since I'm not sure it'll ease readability. Let me know your preferences here as well.) I've also changed empty lines between if/elif to be consistent (either always 1, or always 0, depending on what's more readable.) Also remove unused variables I forgot about. --- goto.py | 179 +++++++++++++++++++++++++++++++------------------- test_goto.py | 182 +++++++++++++++++++++------------------------------ 2 files changed, 186 insertions(+), 175 deletions(-) diff --git a/goto.py b/goto.py index 269baf3..ecdab7b 100644 --- a/goto.py +++ b/goto.py @@ -38,13 +38,13 @@ def __init__(self): self.argument = struct.Struct('> _BYTECODE.argument_bits if extended_arg != 0: size += _get_instruction_size('EXTENDED_ARG', extended_arg) oparg &= (1 << _BYTECODE.argument_bits) - 1 - + opcode = dis.opmap[opname] - if opcode >= _BYTECODE.have_argument: + if opcode >= _BYTECODE.have_argument: size += _BYTECODE.argument.size - - return size + + return size def _get_instructions_size(ops): size = 0 @@ -136,7 +140,7 @@ def _get_instructions_size(ops): if isinstance(op, str): size += _get_instruction_size(op) else: - size += _get_instruction_size(*op) + size += _get_instruction_size(*op) return size def _write_instruction(buf, pos, opname, oparg=0): @@ -164,7 +168,8 @@ def _write_instructions(buf, pos, ops): return pos def _warn_bug(msg): - warnings.warn("Internal error detected - result of with_goto may be incorrect. (%s)" % msg) + warnings.warn("Internal error detected" + + " - result of with_goto may be incorrect. (%s)" % msg) def _find_labels_and_gotos(code): labels = {} @@ -177,52 +182,54 @@ def _find_labels_and_gotos(code): finallies = [] last_block = None - opname0 = oparg0 = offset0 = None - opname1 = oparg1 = offset1 = None # the main one we're looking at each loop iteration + opname1 = oparg1 = offset1 = None opname2 = oparg2 = offset2 = None opname3 = oparg3 = offset3 = None - + def replace_block_in_stack(stack, old_block, new_block): for i, block in enumerate(stack): if block == old_block: stack[i] = new_block - + def replace_block(old_block, new_block): replace_block_in_stack(block_stack, old_block, new_block) for label in labels: replace_block_in_stack(labels[label][2], old_block, new_block) for goto in gotos: replace_block_in_stack(goto[3], old_block, new_block) - + def push_block(opname, oparg=0): block_stack.append((opname, oparg, block_counter)) return block_counter + 1 # to be assigned to block_counter - + def pop_block(): if block_stack: return block_stack.pop() else: _warn_bug("can't pop block") - + def pop_block_of_type(type): if block_stack and block_stack[-1][0] != type: - # in 3.8, only finally blocks are supported, so we must determine the except/finally nature ourselves, and replace the block afterwards - if not _BYTECODE.has_setup_except and type == "" and block_stack[-1][0] == '': + # in 3.8, only finally blocks are supported, so we must determine + # except/finally ourselves, and replace the block's type + if not _BYTECODE.has_setup_except and \ + type == "" and \ + block_stack[-1][0] == '': replace_block(block_stack[-1], (type,) + block_stack[-1][1:]) else: _warn_bug("mismatched block type") return pop_block() - + jump_targets = set(dis.findlabels(code.co_code)) dead = False for opname4, oparg4, offset4 in _parse_instructions(code.co_code, 3): if offset1 in jump_targets: dead = False - + if not dead: endoffset1 = offset2 - + # check for special offsets if for_exits and offset1 == for_exits[-1]: last_block = pop_block() @@ -233,7 +240,7 @@ def pop_block_of_type(type): if finallies and offset1 == finallies[-1]: block_counter = push_block('') finallies.pop() - + # check for special opcodes if opname1 in ('LOAD_GLOBAL', 'LOAD_NAME'): if opname2 == 'LOAD_ATTR' and opname3 == 'POP_TOP': @@ -249,13 +256,14 @@ def pop_block_of_type(type): list(block_stack), 0)) elif opname2 == 'LOAD_ATTR' and opname3 == 'STORE_ATTR': - if code.co_names[oparg1] == 'goto' and code.co_names[oparg2] in ('param', 'params'): + if code.co_names[oparg1] == 'goto' and \ + code.co_names[oparg2] in ('param', 'params'): gotos.append((offset1, offset4, oparg3, list(block_stack), code.co_names[oparg2])) - + elif opname1 in ('SETUP_LOOP', 'SETUP_EXCEPT', 'SETUP_FINALLY', 'SETUP_WITH', 'SETUP_ASYNC_WITH'): @@ -264,26 +272,32 @@ def pop_block_of_type(type): excepts.append(endoffset1 + oparg1) elif opname1 == 'SETUP_FINALLY': finallies.append(endoffset1 + oparg1) + elif not _BYTECODE.has_loop_blocks and opname1 == 'FOR_ITER': block_counter = push_block(opname1, oparg1) for_exits.append(endoffset1 + oparg1) - + elif opname1 == 'POP_BLOCK': last_block = pop_block() + elif opname1 == 'POP_EXCEPT': last_block = pop_block_of_type('') + elif opname1 == 'END_FINALLY': last_block = pop_block_of_type('') + elif opname1 in ('WITH_CLEANUP', 'WITH_CLEANUP_START'): if _BYTECODE.has_setup_with: - block_counter = push_block('') # temporary block to match END_FINALLY + # temporary block to match END_FINALLY + block_counter = push_block('') else: - replace_block(last_block, ('SETUP_WITH',) + last_block[1:]) # python 2.6 - finally was actually with - + # python 2.6 - finally was actually with + replace_block(last_block, + ('SETUP_WITH',) + last_block[1:]) + if opname1 in ('JUMP_ABSOLUTE', 'JUMP_FORWARD'): dead = True - opname0, oparg0, offset0 = opname1, oparg1, offset1 opname1, oparg1, offset1 = opname2, oparg2, offset2 opname2, oparg2, offset2 = opname3, oparg3, offset3 opname3, oparg3, offset3 = opname4, oparg4, offset4 @@ -303,29 +317,30 @@ def _inject_ops(buf, pos, end, ops): if pos + size > end: # not enough space, add code at buffer end and jump there & back buf_end = len(buf) - + go_to_end_ops = [('JUMP_ABSOLUTE', buf_end // _BYTECODE.jump_unit)] - + if pos + _get_instructions_size(go_to_end_ops) > end: - raise SyntaxError('Goto in an incredibly huge function') # not sure if reachable - + # not sure if reachable + raise SyntaxError('Goto in an incredibly huge function') + pos = _write_instructions(buf, pos, go_to_end_ops) _inject_nop_sled(buf, pos, end) - + buf.extend([0] * size) _write_instructions(buf, buf_end, ops) - + else: pos = _write_instructions(buf, pos, ops) _inject_nop_sled(buf, pos, end) - + class _CodeData: def __init__(self, code): self.nlocals = code.co_nlocals self.varnames = code.co_varnames self.consts = code.co_consts self.names = code.co_names - + def get_const(self, value): try: i = self.consts.index(value) @@ -333,7 +348,7 @@ def get_const(self, value): i = len(self.consts) self.consts += (value,) return i - + def get_name(self, value): try: i = self.names.index(value) @@ -341,7 +356,7 @@ def get_name(self, value): i = len(self.names) self.names += (value,) return i - + def add_var(self, name): idx = len(self.varnames) self.varnames += (name,) @@ -352,12 +367,12 @@ def _patch_code(code): new_code = _patched_code_cache.get(code) if new_code is not None: return new_code - + labels, gotos = _find_labels_and_gotos(code) buf = array.array('B', code.co_code) temp_var = None - - data = _CodeData(code) + + data = _CodeData(code) for pos, end, _ in labels.values(): _inject_nop_sled(buf, pos, end) @@ -375,12 +390,13 @@ def _patch_code(code): if origin_stack[i] != target_stack[i]: common_depth = i break - + if params: if temp_var is None: temp_var = data.add_var('goto.temp') - - ops.append(('STORE_FAST', temp_var)) # must do this before any blocks are pushed/popped + + # must do this before any blocks are pushed/popped + ops.append(('STORE_FAST', temp_var)) many_params = (params != 'param') for block, _, _ in reversed(origin_stack[common_depth:]): @@ -394,54 +410,81 @@ def _patch_code(code): ops.append('POP_BLOCK') if block in ('SETUP_WITH', 'SETUP_ASYNC_WITH'): ops.append('POP_TOP') - # pypy 3.6 keeps a block around until END_FINALLY; python 3.8 reuses SETUP_FINALLY for SETUP_EXCEPT (where END_FINALLY is not accepted). What will pypy 3.8 do? - if _BYTECODE.pypy_finally_semantics and block in ('SETUP_FINALLY', 'SETUP_WITH', 'SETUP_ASYNC_WITH'): - ops.append('BEGIN_FINALLY' if _BYTECODE.has_begin_finally else ('LOAD_CONST', data.get_const(None))) + # pypy 3.6 keeps a block around until END_FINALLY; + # python 3.8 reuses SETUP_FINALLY for SETUP_EXCEPT + # (where END_FINALLY is not accepted). + # What will pypy 3.8 do? + if _BYTECODE.pypy_finally_semantics and \ + block in ('SETUP_FINALLY', 'SETUP_WITH', + 'SETUP_ASYNC_WITH'): + if _BYTECODE.has_begin_finally: + ops.append('BEGIN_FINALLY') + else: + ops.append(('LOAD_CONST', data.get_const(None))) ops.append('END_FINALLY') - + tuple_i = 0 for block, blockarg, _ in target_stack[common_depth:]: if block in ('SETUP_LOOP', 'FOR_ITER', 'SETUP_WITH', 'SETUP_ASYNC_WITH'): if not params: - raise SyntaxError('Jump into block without the necessary params') + raise SyntaxError( + 'Jump into block without the necessary params') + if block == 'SETUP_LOOP': ops.append((block, blockarg)) ops.append(('LOAD_FAST', temp_var)) if many_params: ops.append(('LOAD_CONST', data.get_const(tuple_i))) ops.append('BINARY_SUBSCR') - tuple_i += 1 + tuple_i += 1 + if block in ('SETUP_LOOP', 'FOR_ITER'): - ops.append('GET_ITER') # this both converts iterables to iterators for convenience, and prevents FOR_ITER from crashing on non-iter objects. (this is a no-op for iterators) + # this both converts iterables to iterators for + # convenience, and prevents FOR_ITER from crashing + # on non-iter objects. (this is a no-op for iterators) + ops.append('GET_ITER') + elif block in ('SETUP_WITH', 'SETUP_ASYNC_WITH'): - # SETUP_WITH executes __enter__ and so would be inappropriate (a goto must bypass any and all side-effects) + # SETUP_WITH executes __enter__ and so would be + # inappropriate + # (a goto must bypass any and all side-effects) ops.append(('LOAD_ATTR', data.get_name('__exit__'))) ops.append(('SETUP_FINALLY', blockarg)) - + elif block in ('SETUP_EXCEPT', 'SETUP_FINALLY'): ops.append((block, blockarg)) + elif block == '': if _BYTECODE.pypy_finally_semantics: ops.append('SETUP_FINALLY') ops.append('POP_BLOCK') - ops.append('BEGIN_FINALLY' if _BYTECODE.has_begin_finally else ('LOAD_CONST', data.get_const(None))) + if _BYTECODE.has_begin_finally: + ops.append('BEGIN_FINALLY') + else: + ops.append(('LOAD_CONST', data.get_const(None))) + elif block == '': # we raise an exception to get the right block pushed - raise_ops = [('LOAD_CONST', data.get_const(None)), ('RAISE_VARARGS', 1)] - ops.append(('SETUP_EXCEPT' if _BYTECODE.has_setup_except else 'SETUP_FINALLY', _get_instructions_size(raise_ops))) + raise_ops = [('LOAD_CONST', data.get_const(None)), + ('RAISE_VARARGS', 1)] + + setup_except = 'SETUP_EXCEPT' if _BYTECODE.has_setup_except else \ + 'SETUP_FINALLY' + ops.append((setup_except, _get_instructions_size(raise_ops))) ops += raise_ops for _ in _range(3): - ops.append("POP_TOP") + ops.append("POP_TOP") + else: _warn_bug("ignoring %s" % block) - + ops.append(('JUMP_ABSOLUTE', target // _BYTECODE.jump_unit)) - + _inject_ops(buf, pos, end, ops) new_code = _make_code(code, _array_to_bytes(buf), data) - + _patched_code_cache[code] = new_code return new_code diff --git a/test_goto.py b/test_goto.py index 6a96997..007c51f 100644 --- a/test_goto.py +++ b/test_goto.py @@ -121,7 +121,7 @@ def func(): assert func() == (4, 0) -def test_jump_into_loop_iterable_param(): # wasn't planned on being accepted, but works due to an implementation detail, and... probably for the best +def test_jump_into_loop_iterable_param(): @with_goto def func(): goto.param .loop = range(5) @@ -140,7 +140,7 @@ def func(): return i pytest.raises(TypeError, func) - + def test_jump_into_loop_params_not_seq(): @with_goto def func(): @@ -150,7 +150,7 @@ def func(): return i pytest.raises(TypeError, func) - + def test_jump_into_loop_param_with_index(): @with_goto def func(): @@ -163,7 +163,7 @@ def func(): return lst assert func() == [-1, 0, 1, 2, 3, 4] - + def test_jump_into_loop_param_without_index(): @with_goto def func(): @@ -175,12 +175,12 @@ def func(): return lst pytest.raises(UnboundLocalError, func) - -def test_jump_into_2_loops_params_and_live(): + +def test_jump_into_2_loops_and_live(): @with_goto def func(): for i in range(3): - c = 0 + c = 0 goto.params .loop = iter(range(3)), iter(range(10)) for j in None: for k in range(2): @@ -189,7 +189,7 @@ def func(): return i, c assert func() == (2, 11 + 6) - + def test_jump_out_then_back_in_for_loop_and_survive(): @with_goto def func(): @@ -201,15 +201,15 @@ def func(): goto .out raise None label .back - + if NonConstFalse: label .out cc += 1 goto.params .back = it1, it2 return cc, i, j - + assert func() == (20, 4, 3) - + def test_jump_out_of_nested_2_loops(): @with_goto def func(): @@ -227,45 +227,6 @@ def func(): assert func() == (0, 0) -def test_jump_out_of_nested_3_loops(): - @with_goto - def func(): - for i in range(2): - for j in range(2): - for k in range(2): - goto .end - label .end - return (i, j, k) - - assert func() == (0, 0, 0) - -def test_jump_out_of_nested_4_loops(): - @with_goto - def func(): - for i in range(2): - for j in range(2): - for k in range(2): - for m in range(2): - goto .end - label .end - return (i, j, k, m) - - assert func() == (0, 0, 0, 0) - -def test_jump_out_of_nested_5_loops(): - @with_goto - def func(): - for i in range(2): - for j in range(2): - for k in range(2): - for m in range(2): - for n in range(2): - goto .end - label .end - return (i, j, k, m, n) - - assert func() == (0, 0, 0, 0, 0) - def test_jump_out_of_nested_4_loops_and_survive(): @with_goto def func(): @@ -284,11 +245,11 @@ def test_large_jumps_in_diff_orders(): @with_goto def func(): goto .start - + if NonConstFalse: label .finalle return (i, j, k, m, n, i1, j1, k1, m1, n1, i2, j2, k2, m2, n2) - + label .start for i in range(2): for j in range(2): @@ -328,11 +289,18 @@ def func(): for i9 in range(2): for i10 in range(2): for i11 in range(2): - # These are more than 256 bytes of bytecode - x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x - x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x - x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x - + # These are more than + # 256 bytes of bytecode + x += (x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x) + x += (x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x) + x += (x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x) + goto .end label .end return (i1, i2, i3, i4, i5, i6, i7, i8, i9, i10, i11) @@ -357,7 +325,7 @@ def func(): for i in range(10): label .other_loop - + return i assert func() == 2 @@ -369,14 +337,14 @@ def func(): for j in range(10): for k in range(10): goto.param .other_loop = iter(range(3)) - + for j in range(10): label .other_loop - + return (i, j) assert func() == (4, 2) - + def test_jump_into_with_unneeded_params_and_live(): @with_goto def func(): @@ -386,9 +354,9 @@ def func(): j = 1 label .not_loop return (i, j) - + assert func() == (9, 0) - + class Context: def __init__(self): self.enters = 0 @@ -409,7 +377,7 @@ def func(): goto .out label .out return c.data() - + assert func()== (1, 0) def test_jump_out_of_with_block_and_survive(): @@ -421,7 +389,7 @@ def func(): goto .out label .out return (i, c.data()) - + assert func() == (2, (3, 0)) def test_jump_out_of_with_block_and_live(): @@ -434,17 +402,17 @@ def func(): goto .out label .out return (i, j, c.data()) - + assert func() == (2, 0, (3, 0)) - -def test_jump_into_with_block(): + +def test_jump_into_with_block_without_params(): def func(): with Context() as c: label .block goto .block pytest.raises(SyntaxError, with_goto, func) - + def test_jump_into_with_block_with_param(): @with_goto def func(): @@ -455,7 +423,7 @@ def func(): return c.data() assert func() == (0, 1) - + def test_jump_into_with_block_with_params(): @with_goto def func(): @@ -466,8 +434,8 @@ def func(): return c.data() assert func() == (0, 1) - -def test_jump_into_with_block_with_params_and_survive(): + +def test_jump_into_with_block_and_survive(): @with_goto def func(): c = Context() @@ -478,7 +446,7 @@ def func(): return i, c.data() assert func() == (9, (0, 10)) - + def test_jump_into_with_block_with_bad_params(): @with_goto def func(): @@ -487,11 +455,11 @@ def func(): goto.param .block = 123 pytest.raises(AttributeError, func) - + def test_jump_into_with_block_with_bad_exit_params(): class BadAttr: __exit__ = 123 - + @with_goto def func(): with Context() as c: @@ -499,8 +467,8 @@ def func(): goto.param .block = BadAttr pytest.raises(TypeError, func) - -def test_jump_out_of_then_back_into_with_block_with_params_and_survive(): + +def test_jump_out_then_in_with_block_and_survive(): @with_goto def func(): c = Context() @@ -510,17 +478,17 @@ def func(): goto .out cc -= 100 label .back - + if NonConstFalse: label .out cc += 1 goto.param .back = c - + return i, cc, c.data() assert func() == (9, 10, (10, 10)) - -def test_jump_out_of_then_back_into_2_nested_with_blocks_with_params_and_survive(): + +def test_jump_out_then_in_2_nested_with_blocks_and_survive(): @with_goto def func(): c1 = Context() @@ -533,12 +501,12 @@ def func(): goto .out cc -= 100 label .back - + if NonConstFalse: label .out cc += 1 goto.params .back = c1, c2 - + return i, cc, c1.data(), c2.data() assert func() == (10, 10, (11, 11), (10, 10)) @@ -554,7 +522,7 @@ def func(): label .x yield 4 yield 5 - + assert tuple(func()) == (0, 1, 4, 5) def test_jump_out_of_try_except_block(): @@ -569,7 +537,7 @@ def func(): return rv assert func() == None - + def test_jump_out_of_try_finally_block(): @with_goto def func(): @@ -582,7 +550,7 @@ def func(): return rv assert func() == None - + def test_jump_out_of_try_block(): @with_goto def func(): @@ -630,7 +598,7 @@ def func(): return (i, j, rv) assert func() == (2, 2, None) - + def test_jump_into_try_block(): @with_goto def func(): @@ -644,7 +612,7 @@ def func(): finally: rv = 3 return rv - + assert func() == 3 def test_jump_into_try_except_block_and_survive(): @@ -659,9 +627,9 @@ def func(): except: rv = 2 return i, rv - + assert func() == (9, 0) - + def test_jump_into_try_finally_block_and_survive(): @with_goto def func(): @@ -674,9 +642,9 @@ def func(): finally: fv = 1 return i, rv, fv - + assert func() == (9, 0, 1) - + def test_jump_into_try_block_and_survive(): @with_goto def func(): @@ -691,9 +659,9 @@ def func(): finally: fv = 1 return i, rv, fv - + assert func() == (9, 0, 1) - + def test_jump_out_of_except_block(): @with_goto @@ -738,9 +706,9 @@ def func(): label .block i = 3 return i - + assert func() == 3 - + def test_jump_into_except_block_and_live(): @with_goto def func(): @@ -753,8 +721,8 @@ def func(): label .block j = 3 return i, j - - assert func() == (9, 3) + + assert func() == (9, 3) def test_jump_out_of_finally_block(): @with_goto @@ -856,9 +824,9 @@ def func(): break label .x return i - + assert func() == 0 - + def test_jump_with_for_continue(): # to see it doesn't confuse parser @with_goto def func(): @@ -867,9 +835,9 @@ def func(): continue label .x return i - + assert func() == 0 - + def test_jump_with_for_return(): # to see it doesn't confuse parser @with_goto def func(): @@ -878,9 +846,9 @@ def func(): return label .x return i - + assert func() == 0 - + def test_function_is_copy(): def func(): @@ -891,7 +859,7 @@ def func(): assert newfunc is not func assert newfunc.foo == 'bar' - + def test_code_is_not_copy(): def outer_func(): @with_goto @@ -899,7 +867,7 @@ def inner_func(): goto .test label .test return inner_func - + assert outer_func() is not outer_func() assert outer_func().__code__ is outer_func().__code__ - + From c1613f5c2b4d3aabdd20cb12ffb271e71fd7fedd Mon Sep 17 00:00:00 2001 From: condut Date: Sat, 14 Dec 2019 01:27:03 +0200 Subject: [PATCH 20/48] Style & test updates (other updates to commit in following commits) --- goto.py | 25 +++++++++++------------ test_goto.py | 56 +++++++++++----------------------------------------- 2 files changed, 24 insertions(+), 57 deletions(-) diff --git a/goto.py b/goto.py index 18fcf1d..0e79b3e 100644 --- a/goto.py +++ b/goto.py @@ -88,17 +88,17 @@ def _parse_instructions(code): def _get_instruction_size(opname, oparg=0): size = 1 - + extended_arg = oparg >> _BYTECODE.argument_bits if extended_arg != 0: size += _get_instruction_size('EXTENDED_ARG', extended_arg) oparg &= (1 << _BYTECODE.argument_bits) - 1 - + opcode = dis.opmap[opname] - if opcode >= _BYTECODE.have_argument: + if opcode >= _BYTECODE.have_argument: size += _BYTECODE.argument.size - - return size + + return size def _write_instruction(buf, pos, opname, oparg=0): extended_arg = oparg >> _BYTECODE.argument_bits @@ -177,22 +177,22 @@ def _patch_code(code): target_depth = len(target_stack) if origin_stack[:target_depth] != target_stack: raise SyntaxError('Jump into different block') - + size = 0 for i in range(len(origin_stack) - target_depth): size += _get_instruction_size('POP_BLOCK') size += _get_instruction_size('JUMP_ABSOLUTE', target // _BYTECODE.jump_unit) - + moved_to_end = False if pos + size > end: # not enough space, add at end pos = _write_instruction(buf, pos, 'JUMP_ABSOLUTE', len(buf) // _BYTECODE.jump_unit) - + if pos > end: raise SyntaxError('Goto in an incredibly huge function') # not sure if reachable - + size += _get_instruction_size('JUMP_ABSOLUTE', end // _BYTECODE.jump_unit) - + moved_to_end = True pos = len(buf) buf.extend([0] * size) @@ -202,11 +202,10 @@ def _patch_code(code): pos = _write_instruction(buf, pos, 'POP_BLOCK') pos = _write_instruction(buf, pos, 'JUMP_ABSOLUTE', target // _BYTECODE.jump_unit) except (IndexError, struct.error) as e: - raise SyntaxError("Internal error", e) - + raise SyntaxError("Internal error") + if moved_to_end: pos = _write_instruction(buf, pos, 'JUMP_ABSOLUTE', end // _BYTECODE.jump_unit) - else: _inject_nop_sled(buf, pos, end) diff --git a/test_goto.py b/test_goto.py index cabc7b4..6b12f0c 100644 --- a/test_goto.py +++ b/test_goto.py @@ -88,45 +88,6 @@ def func(): assert func() == (0, 0) -def test_jump_out_of_nested_3_loops(): - @with_goto - def func(): - for i in range(2): - for j in range(2): - for k in range(2): - goto .end - label .end - return (i, j, k) - - assert func() == (0, 0, 0) - -def test_jump_out_of_nested_4_loops(): - @with_goto - def func(): - for i in range(2): - for j in range(2): - for k in range(2): - for m in range(2): - goto .end - label .end - return (i, j, k, m) - - assert func() == (0, 0, 0, 0) - -def test_jump_out_of_nested_5_loops(): - @with_goto - def func(): - for i in range(2): - for j in range(2): - for k in range(2): - for m in range(2): - for n in range(2): - goto .end - label .end - return (i, j, k, m, n) - - assert func() == (0, 0, 0, 0, 0) - def test_jump_out_of_nested_11_loops(): @with_goto def func(): @@ -142,11 +103,18 @@ def func(): for i9 in range(2): for i10 in range(2): for i11 in range(2): - # These are more than 256 bytes of bytecode - x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x - x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x - x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x - + # These are more than + # 256 bytes of bytecode + x += (x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x) + x += (x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x) + x += (x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x) + goto .end label .end return (i1, i2, i3, i4, i5, i6, i7, i8, i9, i10, i11) From df4038234aa2a679177c2512fcdf708056ecfc07 Mon Sep 17 00:00:00 2001 From: condut Date: Sat, 14 Dec 2019 01:35:36 +0200 Subject: [PATCH 21/48] Refactor the ops sizing/writing to avoid code duplication (This was originally in the second PR, but since you want to take this one first, I think it's a good commit to add here, since it's relevant to this change and would make diffing against future PRs easier) --- goto.py | 53 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/goto.py b/goto.py index 0e79b3e..cc6d746 100644 --- a/goto.py +++ b/goto.py @@ -100,6 +100,15 @@ def _get_instruction_size(opname, oparg=0): return size +def _get_instructions_size(ops): + size = 0 + for op in ops: + if isinstance(op, str): + size += _get_instruction_size(op) + else: + size += _get_instruction_size(*op) + return size + def _write_instruction(buf, pos, opname, oparg=0): extended_arg = oparg >> _BYTECODE.argument_bits if extended_arg != 0: @@ -116,6 +125,13 @@ def _write_instruction(buf, pos, opname, oparg=0): return pos +def _write_instructions(buf, pos, ops): + for op in ops: + if isinstance(op, str): + pos = _write_instruction(buf, pos, op) + else: + pos = _write_instruction(buf, pos, *op) + return pos def _find_labels_and_gotos(code): labels = {} @@ -178,35 +194,28 @@ def _patch_code(code): if origin_stack[:target_depth] != target_stack: raise SyntaxError('Jump into different block') - size = 0 + ops = [] for i in range(len(origin_stack) - target_depth): - size += _get_instruction_size('POP_BLOCK') - size += _get_instruction_size('JUMP_ABSOLUTE', target // _BYTECODE.jump_unit) + ops.append('POP_BLOCK') + ops.append(('JUMP_ABSOLUTE', target // _BYTECODE.jump_unit)) - moved_to_end = False - if pos + size > end: - # not enough space, add at end - pos = _write_instruction(buf, pos, 'JUMP_ABSOLUTE', len(buf) // _BYTECODE.jump_unit) + if pos + _get_instructions_size(ops) > end: + # not enough space, add code at buffer end and jump there + buf_end = len(buf) - if pos > end: - raise SyntaxError('Goto in an incredibly huge function') # not sure if reachable + go_to_end_ops = [('JUMP_ABSOLUTE', buf_end // _BYTECODE.jump_unit)] - size += _get_instruction_size('JUMP_ABSOLUTE', end // _BYTECODE.jump_unit) + if pos + _get_instructions_size(go_to_end_ops) > end: + # not sure if reachable + raise SyntaxError('Goto in an incredibly huge function') - moved_to_end = True - pos = len(buf) - buf.extend([0] * size) + pos = _write_instructions(buf, pos, go_to_end_ops) + _inject_nop_sled(buf, pos, end) - try: - for i in range(len(origin_stack) - target_depth): - pos = _write_instruction(buf, pos, 'POP_BLOCK') - pos = _write_instruction(buf, pos, 'JUMP_ABSOLUTE', target // _BYTECODE.jump_unit) - except (IndexError, struct.error) as e: - raise SyntaxError("Internal error") - - if moved_to_end: - pos = _write_instruction(buf, pos, 'JUMP_ABSOLUTE', end // _BYTECODE.jump_unit) + buf.extend([0] * _get_instructions_size(ops)) + _write_instructions(buf, buf_end, ops) else: + pos = _write_instructions(buf, pos, ops) _inject_nop_sled(buf, pos, end) return _make_code(code, _array_to_bytes(buf)) From fedd2cb19f890ebcefae8e1de63fa5931aedb1c5 Mon Sep 17 00:00:00 2001 From: condut Date: Fri, 13 Dec 2019 00:35:35 +0200 Subject: [PATCH 22/48] Added python 3.8 support Added 4 tests, 3 currently disabled (pre-existing issues, especially on pypy). --- goto.py | 48 +++++++++++++++++++++++++++++--------------- test_goto.py | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 16 deletions(-) diff --git a/goto.py b/goto.py index cc6d746..349e1d9 100644 --- a/goto.py +++ b/goto.py @@ -33,6 +33,8 @@ def __init__(self): self.argument = struct.Struct(' end: diff --git a/test_goto.py b/test_goto.py index 6b12f0c..27c7199 100644 --- a/test_goto.py +++ b/test_goto.py @@ -63,6 +63,18 @@ def func(): assert func() == 0 +def test_jump_out_of_loop_and_survive(): + @with_goto + def func(): + for i in range(10): + for j in range(10): + goto .end + label .end + return (i, j) + + assert func() == (9, 0) + + def test_jump_into_loop(): def func(): for i in range(10): @@ -148,6 +160,22 @@ def func(): assert func() == None +"""def test_jump_out_of_try_block_and_survive(): + @with_goto + def func(): + for i in range(10): + try: + rv = None + goto .end + except: + rv = 'except' + finally: + rv = 'finally' + label .end + return (i, rv) + + assert func() == (9, None)""" + def test_jump_into_try_block(): def func(): try: @@ -159,6 +187,34 @@ def func(): pytest.raises(SyntaxError, with_goto, func) +"""def test_jump_out_of_except_block(): + @with_goto + def func(): + try: + rv = 1 / 0 + except: + rv = 'except' + goto .end + finally: + rv = 'finally' + label .end + return rv + + assert func() == 'except'""" + + +"""def test_jump_into_except_block(): + def func(): + try: + pass + except: + label .block + pass + goto .block + + pytest.raises(SyntaxError, with_goto, func)""" + + def test_jump_to_unknown_label(): def func(): goto .unknown From 5133d8c21e8c394672a612a9154cd2a5e777c842 Mon Sep 17 00:00:00 2001 From: condut Date: Sat, 14 Dec 2019 02:00:11 +0200 Subject: [PATCH 23/48] Updated style & added py38 to tox/travis/readme --- .travis.yml | 2 + README.md | 2 +- goto.py | 506 +++++++++++++++++++++++++-------------------------- test_goto.py | 466 +++++++++++++++++++++++------------------------ tox.ini | 2 +- 5 files changed, 490 insertions(+), 488 deletions(-) diff --git a/.travis.yml b/.travis.yml index c448cc2..77433de 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,8 @@ python: - "3.4" - "3.5" - "3.6" + - "3.7" + - "3.8" - "pypy" - "pypy3" diff --git a/README.md b/README.md index 9157dd6..a0c54f7 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![Pypi Entry](https://badge.fury.io/py/goto-statement.svg)](https://pypi.python.org/pypi/goto-statement) A function decorator to use `goto` in Python. -Tested on Python 2.6 through 3.6 and PyPy. +Tested on Python 2.6 through 3.8 and PyPy. [![](https://imgs.xkcd.com/comics/goto.png)](https://xkcd.com/292/) diff --git a/goto.py b/goto.py index 349e1d9..80e802c 100644 --- a/goto.py +++ b/goto.py @@ -1,253 +1,253 @@ -import dis -import struct -import array -import types -import functools - - -try: - _array_to_bytes = array.array.tobytes -except AttributeError: - _array_to_bytes = array.array.tostring - - -class _Bytecode: - def __init__(self): - code = (lambda: x if x else y).__code__.co_code - opcode, oparg = struct.unpack_from('BB', code, 2) - - # Starting with Python 3.6, the bytecode format has been changed to use - # 16-bit words (8-bit opcode + 8-bit argument) for each instruction, - # as opposed to previously 24-bit (8-bit opcode + 16-bit argument) for - # instructions that expect an argument or just 8-bit for those that don't. - # https://bugs.python.org/issue26647 - if dis.opname[opcode] == 'POP_JUMP_IF_FALSE': - self.argument = struct.Struct('B') - self.have_argument = 0 - # As of Python 3.6, jump targets are still addressed by their byte - # unit. This, however, is matter to change, so that jump targets, - # in the future, will refer to the code unit (address in bytes / 2). - # https://bugs.python.org/issue26647 - self.jump_unit = 8 // oparg - else: - self.argument = struct.Struct('= _BYTECODE.have_argument: - oparg = extended_arg | _BYTECODE.argument.unpack_from(code, pos)[0] - pos += _BYTECODE.argument.size - - if opcode == dis.EXTENDED_ARG: - extended_arg = oparg << _BYTECODE.argument_bits - extended_arg_offset = offset - continue - - extended_arg = 0 - extended_arg_offset = None - yield (dis.opname[opcode], oparg, offset) - -def _get_instruction_size(opname, oparg=0): - size = 1 - - extended_arg = oparg >> _BYTECODE.argument_bits - if extended_arg != 0: - size += _get_instruction_size('EXTENDED_ARG', extended_arg) - oparg &= (1 << _BYTECODE.argument_bits) - 1 - - opcode = dis.opmap[opname] - if opcode >= _BYTECODE.have_argument: - size += _BYTECODE.argument.size - - return size - -def _get_instructions_size(ops): - size = 0 - for op in ops: - if isinstance(op, str): - size += _get_instruction_size(op) - else: - size += _get_instruction_size(*op) - return size - -def _write_instruction(buf, pos, opname, oparg=0): - extended_arg = oparg >> _BYTECODE.argument_bits - if extended_arg != 0: - pos = _write_instruction(buf, pos, 'EXTENDED_ARG', extended_arg) - oparg &= (1 << _BYTECODE.argument_bits) - 1 - - opcode = dis.opmap[opname] - buf[pos] = opcode - pos += 1 - - if opcode >= _BYTECODE.have_argument: - _BYTECODE.argument.pack_into(buf, pos, oparg) - pos += _BYTECODE.argument.size - - return pos - -def _write_instructions(buf, pos, ops): - for op in ops: - if isinstance(op, str): - pos = _write_instruction(buf, pos, op) - else: - pos = _write_instruction(buf, pos, *op) - return pos - -def _find_labels_and_gotos(code): - labels = {} - gotos = [] - - block_stack = [] - block_counter = 0 - block_exits = [] - - opname1 = oparg1 = offset1 = None - opname2 = oparg2 = offset2 = None - opname3 = oparg3 = offset3 = None - - for opname4, oparg4, offset4 in _parse_instructions(code.co_code): - if opname1 in ('LOAD_GLOBAL', 'LOAD_NAME'): - if opname2 == 'LOAD_ATTR' and opname3 == 'POP_TOP': - name = code.co_names[oparg1] - if name == 'label': - labels[oparg2] = (offset1, - offset4, - tuple(block_stack)) - elif name == 'goto': - gotos.append((offset1, - offset4, - oparg2, - tuple(block_stack))) - elif opname1 in ('SETUP_LOOP', - 'SETUP_EXCEPT', 'SETUP_FINALLY', - 'SETUP_WITH', 'SETUP_ASYNC_WITH'): - block_counter += 1 - block_stack.append((opname1, block_counter)) - elif not _BYTECODE.has_loop_blocks and opname1 == 'FOR_ITER': - block_counter += 1 - block_stack.append((opname1, block_counter)) - block_exits.append(offset1 + oparg1) - elif opname1 == 'POP_BLOCK' and block_stack: - block_stack.pop() - elif block_exits and offset1 == block_exits[-1] and block_stack: - block_stack.pop() - block_exits.pop() - - opname1, oparg1, offset1 = opname2, oparg2, offset2 - opname2, oparg2, offset2 = opname3, oparg3, offset3 - opname3, oparg3, offset3 = opname4, oparg4, offset4 - - return labels, gotos - - -def _inject_nop_sled(buf, pos, end): - while pos < end: - pos = _write_instruction(buf, pos, 'NOP') - - -def _patch_code(code): - labels, gotos = _find_labels_and_gotos(code) - buf = array.array('B', code.co_code) - - for pos, end, _ in labels.values(): - _inject_nop_sled(buf, pos, end) - - for pos, end, label, origin_stack in gotos: - try: - _, target, target_stack = labels[label] - except KeyError: - raise SyntaxError('Unknown label {0!r}'.format(code.co_names[label])) - - target_depth = len(target_stack) - if origin_stack[:target_depth] != target_stack: - raise SyntaxError('Jump into different block') - - ops = [] - for block, _ in origin_stack[target_depth:]: - if block == 'FOR_ITER': - ops.append('POP_TOP') - else: - ops.append('POP_BLOCK') - ops.append(('JUMP_ABSOLUTE', target // _BYTECODE.jump_unit)) - - if pos + _get_instructions_size(ops) > end: - # not enough space, add code at buffer end and jump there - buf_end = len(buf) - - go_to_end_ops = [('JUMP_ABSOLUTE', buf_end // _BYTECODE.jump_unit)] - - if pos + _get_instructions_size(go_to_end_ops) > end: - # not sure if reachable - raise SyntaxError('Goto in an incredibly huge function') - - pos = _write_instructions(buf, pos, go_to_end_ops) - _inject_nop_sled(buf, pos, end) - - buf.extend([0] * _get_instructions_size(ops)) - _write_instructions(buf, buf_end, ops) - else: - pos = _write_instructions(buf, pos, ops) - _inject_nop_sled(buf, pos, end) - - return _make_code(code, _array_to_bytes(buf)) - - -def with_goto(func_or_code): - if isinstance(func_or_code, types.CodeType): - return _patch_code(func_or_code) - - return functools.update_wrapper( - types.FunctionType( - _patch_code(func_or_code.__code__), - func_or_code.__globals__, - func_or_code.__name__, - func_or_code.__defaults__, - func_or_code.__closure__, - ), - func_or_code - ) +import dis +import struct +import array +import types +import functools + + +try: + _array_to_bytes = array.array.tobytes +except AttributeError: + _array_to_bytes = array.array.tostring + + +class _Bytecode: + def __init__(self): + code = (lambda: x if x else y).__code__.co_code + opcode, oparg = struct.unpack_from('BB', code, 2) + + # Starting with Python 3.6, the bytecode format has been changed to use + # 16-bit words (8-bit opcode + 8-bit argument) for each instruction, + # as opposed to previously 24-bit (8-bit opcode + 16-bit argument) for + # instructions that expect an argument or just 8-bit for those that don't. + # https://bugs.python.org/issue26647 + if dis.opname[opcode] == 'POP_JUMP_IF_FALSE': + self.argument = struct.Struct('B') + self.have_argument = 0 + # As of Python 3.6, jump targets are still addressed by their byte + # unit. This, however, is matter to change, so that jump targets, + # in the future, will refer to the code unit (address in bytes / 2). + # https://bugs.python.org/issue26647 + self.jump_unit = 8 // oparg + else: + self.argument = struct.Struct('= _BYTECODE.have_argument: + oparg = extended_arg | _BYTECODE.argument.unpack_from(code, pos)[0] + pos += _BYTECODE.argument.size + + if opcode == dis.EXTENDED_ARG: + extended_arg = oparg << _BYTECODE.argument_bits + extended_arg_offset = offset + continue + + extended_arg = 0 + extended_arg_offset = None + yield (dis.opname[opcode], oparg, offset) + +def _get_instruction_size(opname, oparg=0): + size = 1 + + extended_arg = oparg >> _BYTECODE.argument_bits + if extended_arg != 0: + size += _get_instruction_size('EXTENDED_ARG', extended_arg) + oparg &= (1 << _BYTECODE.argument_bits) - 1 + + opcode = dis.opmap[opname] + if opcode >= _BYTECODE.have_argument: + size += _BYTECODE.argument.size + + return size + +def _get_instructions_size(ops): + size = 0 + for op in ops: + if isinstance(op, str): + size += _get_instruction_size(op) + else: + size += _get_instruction_size(*op) + return size + +def _write_instruction(buf, pos, opname, oparg=0): + extended_arg = oparg >> _BYTECODE.argument_bits + if extended_arg != 0: + pos = _write_instruction(buf, pos, 'EXTENDED_ARG', extended_arg) + oparg &= (1 << _BYTECODE.argument_bits) - 1 + + opcode = dis.opmap[opname] + buf[pos] = opcode + pos += 1 + + if opcode >= _BYTECODE.have_argument: + _BYTECODE.argument.pack_into(buf, pos, oparg) + pos += _BYTECODE.argument.size + + return pos + +def _write_instructions(buf, pos, ops): + for op in ops: + if isinstance(op, str): + pos = _write_instruction(buf, pos, op) + else: + pos = _write_instruction(buf, pos, *op) + return pos + +def _find_labels_and_gotos(code): + labels = {} + gotos = [] + + block_stack = [] + block_counter = 0 + block_exits = [] + + opname1 = oparg1 = offset1 = None + opname2 = oparg2 = offset2 = None + opname3 = oparg3 = offset3 = None + + for opname4, oparg4, offset4 in _parse_instructions(code.co_code): + if opname1 in ('LOAD_GLOBAL', 'LOAD_NAME'): + if opname2 == 'LOAD_ATTR' and opname3 == 'POP_TOP': + name = code.co_names[oparg1] + if name == 'label': + labels[oparg2] = (offset1, + offset4, + tuple(block_stack)) + elif name == 'goto': + gotos.append((offset1, + offset4, + oparg2, + tuple(block_stack))) + elif opname1 in ('SETUP_LOOP', + 'SETUP_EXCEPT', 'SETUP_FINALLY', + 'SETUP_WITH', 'SETUP_ASYNC_WITH'): + block_counter += 1 + block_stack.append((opname1, block_counter)) + elif not _BYTECODE.has_loop_blocks and opname1 == 'FOR_ITER': + block_counter += 1 + block_stack.append((opname1, block_counter)) + block_exits.append(offset1 + oparg1) + elif opname1 == 'POP_BLOCK' and block_stack: + block_stack.pop() + elif block_exits and offset1 == block_exits[-1] and block_stack: + block_stack.pop() + block_exits.pop() + + opname1, oparg1, offset1 = opname2, oparg2, offset2 + opname2, oparg2, offset2 = opname3, oparg3, offset3 + opname3, oparg3, offset3 = opname4, oparg4, offset4 + + return labels, gotos + + +def _inject_nop_sled(buf, pos, end): + while pos < end: + pos = _write_instruction(buf, pos, 'NOP') + + +def _patch_code(code): + labels, gotos = _find_labels_and_gotos(code) + buf = array.array('B', code.co_code) + + for pos, end, _ in labels.values(): + _inject_nop_sled(buf, pos, end) + + for pos, end, label, origin_stack in gotos: + try: + _, target, target_stack = labels[label] + except KeyError: + raise SyntaxError('Unknown label {0!r}'.format(code.co_names[label])) + + target_depth = len(target_stack) + if origin_stack[:target_depth] != target_stack: + raise SyntaxError('Jump into different block') + + ops = [] + for block, _ in origin_stack[target_depth:]: + if block == 'FOR_ITER': + ops.append('POP_TOP') + else: + ops.append('POP_BLOCK') + ops.append(('JUMP_ABSOLUTE', target // _BYTECODE.jump_unit)) + + if pos + _get_instructions_size(ops) > end: + # not enough space, add code at buffer end and jump there + buf_end = len(buf) + + go_to_end_ops = [('JUMP_ABSOLUTE', buf_end // _BYTECODE.jump_unit)] + + if pos + _get_instructions_size(go_to_end_ops) > end: + # not sure if reachable + raise SyntaxError('Goto in an incredibly huge function') + + pos = _write_instructions(buf, pos, go_to_end_ops) + _inject_nop_sled(buf, pos, end) + + buf.extend([0] * _get_instructions_size(ops)) + _write_instructions(buf, buf_end, ops) + else: + pos = _write_instructions(buf, pos, ops) + _inject_nop_sled(buf, pos, end) + + return _make_code(code, _array_to_bytes(buf)) + + +def with_goto(func_or_code): + if isinstance(func_or_code, types.CodeType): + return _patch_code(func_or_code) + + return functools.update_wrapper( + types.FunctionType( + _patch_code(func_or_code.__code__), + func_or_code.__globals__, + func_or_code.__name__, + func_or_code.__defaults__, + func_or_code.__closure__, + ), + func_or_code + ) diff --git a/test_goto.py b/test_goto.py index 27c7199..83ebb97 100644 --- a/test_goto.py +++ b/test_goto.py @@ -1,233 +1,233 @@ -import sys -import pytest -from goto import with_goto - -CODE = '''\ -i = 0 -result = [] - -label .start -if i == 10: - goto .end - -result.append(i) -i += 1 -goto .start - -label .end -''' - -EXPECTED = list(range(10)) - - -def test_range_as_code(): - ns = {} - exec(with_goto(compile(CODE, '', 'exec')), ns) - assert ns['result'] == EXPECTED - - -def make_function(code): - lines = ['def func():'] - for line in code: - lines.append(' ' + line) - lines.append(' return result') - - ns = {} - exec('\n'.join(lines), ns) - return ns['func'] - - -def test_range_as_function(): - assert with_goto(make_function(CODE.splitlines()))() == EXPECTED - - -def test_EXTENDED_ARG(): - code = [] - code.append('result = True') - code.append('goto .foo') - for i in range(2**16): - code.append('label .l{0}'.format(i)) - code.append('result = "dead code"') - code.append('label .foo') - assert with_goto(make_function(code))() is True - - -def test_jump_out_of_loop(): - @with_goto - def func(): - for i in range(10): - goto .end - label .end - return i - - assert func() == 0 - - -def test_jump_out_of_loop_and_survive(): - @with_goto - def func(): - for i in range(10): - for j in range(10): - goto .end - label .end - return (i, j) - - assert func() == (9, 0) - - -def test_jump_into_loop(): - def func(): - for i in range(10): - label .loop - goto .loop - - pytest.raises(SyntaxError, with_goto, func) - -def test_jump_out_of_nested_2_loops(): - @with_goto - def func(): - x = 1 - for i in range(2): - for j in range(2): - # These are more than 256 bytes of bytecode - x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x - x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x - x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x - - goto .end - label .end - return (i, j) - - assert func() == (0, 0) - -def test_jump_out_of_nested_11_loops(): - @with_goto - def func(): - x = 1 - for i1 in range(2): - for i2 in range(2): - for i3 in range(2): - for i4 in range(2): - for i5 in range(2): - for i6 in range(2): - for i7 in range(2): - for i8 in range(2): - for i9 in range(2): - for i10 in range(2): - for i11 in range(2): - # These are more than - # 256 bytes of bytecode - x += (x+x+x+x+x+x+x+x+x+ - x+x+x+x+x+x+x+x+x+ - x+x+x+x+x+x+x+x+x) - x += (x+x+x+x+x+x+x+x+x+ - x+x+x+x+x+x+x+x+x+ - x+x+x+x+x+x+x+x+x) - x += (x+x+x+x+x+x+x+x+x+ - x+x+x+x+x+x+x+x+x+ - x+x+x+x+x+x+x+x+x) - - goto .end - label .end - return (i1, i2, i3, i4, i5, i6, i7, i8, i9, i10, i11) - - assert func() == (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) - -def test_jump_across_loops(): - def func(): - for i in range(10): - goto .other_loop - - for i in range(10): - label .other_loop - - pytest.raises(SyntaxError, with_goto, func) - - -def test_jump_out_of_try_block(): - @with_goto - def func(): - try: - rv = None - goto .end - except: - rv = 'except' - finally: - rv = 'finally' - label .end - return rv - - assert func() == None - - -"""def test_jump_out_of_try_block_and_survive(): - @with_goto - def func(): - for i in range(10): - try: - rv = None - goto .end - except: - rv = 'except' - finally: - rv = 'finally' - label .end - return (i, rv) - - assert func() == (9, None)""" - -def test_jump_into_try_block(): - def func(): - try: - label .block - except: - pass - goto .block - - pytest.raises(SyntaxError, with_goto, func) - - -"""def test_jump_out_of_except_block(): - @with_goto - def func(): - try: - rv = 1 / 0 - except: - rv = 'except' - goto .end - finally: - rv = 'finally' - label .end - return rv - - assert func() == 'except'""" - - -"""def test_jump_into_except_block(): - def func(): - try: - pass - except: - label .block - pass - goto .block - - pytest.raises(SyntaxError, with_goto, func)""" - - -def test_jump_to_unknown_label(): - def func(): - goto .unknown - - pytest.raises(SyntaxError, with_goto, func) - - -def test_function_is_copy(): - def func(): - pass - - func.foo = 'bar' - newfunc = with_goto(func) - - assert newfunc is not func - assert newfunc.foo == 'bar' +import sys +import pytest +from goto import with_goto + +CODE = '''\ +i = 0 +result = [] + +label .start +if i == 10: + goto .end + +result.append(i) +i += 1 +goto .start + +label .end +''' + +EXPECTED = list(range(10)) + + +def test_range_as_code(): + ns = {} + exec(with_goto(compile(CODE, '', 'exec')), ns) + assert ns['result'] == EXPECTED + + +def make_function(code): + lines = ['def func():'] + for line in code: + lines.append(' ' + line) + lines.append(' return result') + + ns = {} + exec('\n'.join(lines), ns) + return ns['func'] + + +def test_range_as_function(): + assert with_goto(make_function(CODE.splitlines()))() == EXPECTED + + +def test_EXTENDED_ARG(): + code = [] + code.append('result = True') + code.append('goto .foo') + for i in range(2**16): + code.append('label .l{0}'.format(i)) + code.append('result = "dead code"') + code.append('label .foo') + assert with_goto(make_function(code))() is True + + +def test_jump_out_of_loop(): + @with_goto + def func(): + for i in range(10): + goto .end + label .end + return i + + assert func() == 0 + + +def test_jump_out_of_loop_and_survive(): + @with_goto + def func(): + for i in range(10): + for j in range(10): + goto .end + label .end + return (i, j) + + assert func() == (9, 0) + + +def test_jump_into_loop(): + def func(): + for i in range(10): + label .loop + goto .loop + + pytest.raises(SyntaxError, with_goto, func) + +def test_jump_out_of_nested_2_loops(): + @with_goto + def func(): + x = 1 + for i in range(2): + for j in range(2): + # These are more than 256 bytes of bytecode + x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x + x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x + x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x + + goto .end + label .end + return (i, j) + + assert func() == (0, 0) + +def test_jump_out_of_nested_11_loops(): + @with_goto + def func(): + x = 1 + for i1 in range(2): + for i2 in range(2): + for i3 in range(2): + for i4 in range(2): + for i5 in range(2): + for i6 in range(2): + for i7 in range(2): + for i8 in range(2): + for i9 in range(2): + for i10 in range(2): + for i11 in range(2): + # These are more than + # 256 bytes of bytecode + x += (x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x) + x += (x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x) + x += (x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x) + + goto .end + label .end + return (i1, i2, i3, i4, i5, i6, i7, i8, i9, i10, i11) + + assert func() == (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) + +def test_jump_across_loops(): + def func(): + for i in range(10): + goto .other_loop + + for i in range(10): + label .other_loop + + pytest.raises(SyntaxError, with_goto, func) + + +def test_jump_out_of_try_block(): + @with_goto + def func(): + try: + rv = None + goto .end + except: + rv = 'except' + finally: + rv = 'finally' + label .end + return rv + + assert func() == None + + +"""def test_jump_out_of_try_block_and_survive(): + @with_goto + def func(): + for i in range(10): + try: + rv = None + goto .end + except: + rv = 'except' + finally: + rv = 'finally' + label .end + return (i, rv) + + assert func() == (9, None)""" + +def test_jump_into_try_block(): + def func(): + try: + label .block + except: + pass + goto .block + + pytest.raises(SyntaxError, with_goto, func) + + +"""def test_jump_out_of_except_block(): + @with_goto + def func(): + try: + rv = 1 / 0 + except: + rv = 'except' + goto .end + finally: + rv = 'finally' + label .end + return rv + + assert func() == 'except'""" + + +"""def test_jump_into_except_block(): + def func(): + try: + pass + except: + label .block + pass + goto .block + + pytest.raises(SyntaxError, with_goto, func)""" + + +def test_jump_to_unknown_label(): + def func(): + goto .unknown + + pytest.raises(SyntaxError, with_goto, func) + + +def test_function_is_copy(): + def func(): + pass + + func.foo = 'bar' + newfunc = with_goto(func) + + assert newfunc is not func + assert newfunc.foo == 'bar' diff --git a/tox.ini b/tox.ini index 67d5756..e06274a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist=py26,py27,py34,py35,py36,pypy,pypy3 +envlist=py26,py27,py34,py35,py36,py37,py38,pypy,pypy3 [testenv] deps=pytest From 2d9047c50a4a349d030c9ddf29eeba9302f4b13c Mon Sep 17 00:00:00 2001 From: condut Date: Sat, 14 Dec 2019 02:11:18 +0200 Subject: [PATCH 24/48] fix line endings snafu (you'll squash anyway, so I'm keeping history) --- goto.py | 506 +++++++++++++++++++++++++-------------------------- test_goto.py | 466 +++++++++++++++++++++++------------------------ 2 files changed, 486 insertions(+), 486 deletions(-) diff --git a/goto.py b/goto.py index 80e802c..4603685 100644 --- a/goto.py +++ b/goto.py @@ -1,253 +1,253 @@ -import dis -import struct -import array -import types -import functools - - -try: - _array_to_bytes = array.array.tobytes -except AttributeError: - _array_to_bytes = array.array.tostring - - -class _Bytecode: - def __init__(self): - code = (lambda: x if x else y).__code__.co_code - opcode, oparg = struct.unpack_from('BB', code, 2) - - # Starting with Python 3.6, the bytecode format has been changed to use - # 16-bit words (8-bit opcode + 8-bit argument) for each instruction, - # as opposed to previously 24-bit (8-bit opcode + 16-bit argument) for - # instructions that expect an argument or just 8-bit for those that don't. - # https://bugs.python.org/issue26647 - if dis.opname[opcode] == 'POP_JUMP_IF_FALSE': - self.argument = struct.Struct('B') - self.have_argument = 0 - # As of Python 3.6, jump targets are still addressed by their byte - # unit. This, however, is matter to change, so that jump targets, - # in the future, will refer to the code unit (address in bytes / 2). - # https://bugs.python.org/issue26647 - self.jump_unit = 8 // oparg - else: - self.argument = struct.Struct('= _BYTECODE.have_argument: - oparg = extended_arg | _BYTECODE.argument.unpack_from(code, pos)[0] - pos += _BYTECODE.argument.size - - if opcode == dis.EXTENDED_ARG: - extended_arg = oparg << _BYTECODE.argument_bits - extended_arg_offset = offset - continue - - extended_arg = 0 - extended_arg_offset = None - yield (dis.opname[opcode], oparg, offset) - -def _get_instruction_size(opname, oparg=0): - size = 1 - - extended_arg = oparg >> _BYTECODE.argument_bits - if extended_arg != 0: - size += _get_instruction_size('EXTENDED_ARG', extended_arg) - oparg &= (1 << _BYTECODE.argument_bits) - 1 - - opcode = dis.opmap[opname] - if opcode >= _BYTECODE.have_argument: - size += _BYTECODE.argument.size - - return size - -def _get_instructions_size(ops): - size = 0 - for op in ops: - if isinstance(op, str): - size += _get_instruction_size(op) - else: - size += _get_instruction_size(*op) - return size - -def _write_instruction(buf, pos, opname, oparg=0): - extended_arg = oparg >> _BYTECODE.argument_bits - if extended_arg != 0: - pos = _write_instruction(buf, pos, 'EXTENDED_ARG', extended_arg) - oparg &= (1 << _BYTECODE.argument_bits) - 1 - - opcode = dis.opmap[opname] - buf[pos] = opcode - pos += 1 - - if opcode >= _BYTECODE.have_argument: - _BYTECODE.argument.pack_into(buf, pos, oparg) - pos += _BYTECODE.argument.size - - return pos - -def _write_instructions(buf, pos, ops): - for op in ops: - if isinstance(op, str): - pos = _write_instruction(buf, pos, op) - else: - pos = _write_instruction(buf, pos, *op) - return pos - -def _find_labels_and_gotos(code): - labels = {} - gotos = [] - - block_stack = [] - block_counter = 0 - block_exits = [] - - opname1 = oparg1 = offset1 = None - opname2 = oparg2 = offset2 = None - opname3 = oparg3 = offset3 = None - - for opname4, oparg4, offset4 in _parse_instructions(code.co_code): - if opname1 in ('LOAD_GLOBAL', 'LOAD_NAME'): - if opname2 == 'LOAD_ATTR' and opname3 == 'POP_TOP': - name = code.co_names[oparg1] - if name == 'label': - labels[oparg2] = (offset1, - offset4, - tuple(block_stack)) - elif name == 'goto': - gotos.append((offset1, - offset4, - oparg2, - tuple(block_stack))) - elif opname1 in ('SETUP_LOOP', - 'SETUP_EXCEPT', 'SETUP_FINALLY', - 'SETUP_WITH', 'SETUP_ASYNC_WITH'): - block_counter += 1 - block_stack.append((opname1, block_counter)) - elif not _BYTECODE.has_loop_blocks and opname1 == 'FOR_ITER': - block_counter += 1 - block_stack.append((opname1, block_counter)) - block_exits.append(offset1 + oparg1) - elif opname1 == 'POP_BLOCK' and block_stack: - block_stack.pop() - elif block_exits and offset1 == block_exits[-1] and block_stack: - block_stack.pop() - block_exits.pop() - - opname1, oparg1, offset1 = opname2, oparg2, offset2 - opname2, oparg2, offset2 = opname3, oparg3, offset3 - opname3, oparg3, offset3 = opname4, oparg4, offset4 - - return labels, gotos - - -def _inject_nop_sled(buf, pos, end): - while pos < end: - pos = _write_instruction(buf, pos, 'NOP') - - -def _patch_code(code): - labels, gotos = _find_labels_and_gotos(code) - buf = array.array('B', code.co_code) - - for pos, end, _ in labels.values(): - _inject_nop_sled(buf, pos, end) - - for pos, end, label, origin_stack in gotos: - try: - _, target, target_stack = labels[label] - except KeyError: - raise SyntaxError('Unknown label {0!r}'.format(code.co_names[label])) - - target_depth = len(target_stack) - if origin_stack[:target_depth] != target_stack: - raise SyntaxError('Jump into different block') - - ops = [] - for block, _ in origin_stack[target_depth:]: - if block == 'FOR_ITER': - ops.append('POP_TOP') - else: - ops.append('POP_BLOCK') - ops.append(('JUMP_ABSOLUTE', target // _BYTECODE.jump_unit)) - - if pos + _get_instructions_size(ops) > end: - # not enough space, add code at buffer end and jump there - buf_end = len(buf) - - go_to_end_ops = [('JUMP_ABSOLUTE', buf_end // _BYTECODE.jump_unit)] - - if pos + _get_instructions_size(go_to_end_ops) > end: - # not sure if reachable - raise SyntaxError('Goto in an incredibly huge function') - - pos = _write_instructions(buf, pos, go_to_end_ops) - _inject_nop_sled(buf, pos, end) - - buf.extend([0] * _get_instructions_size(ops)) - _write_instructions(buf, buf_end, ops) - else: - pos = _write_instructions(buf, pos, ops) - _inject_nop_sled(buf, pos, end) - - return _make_code(code, _array_to_bytes(buf)) - - -def with_goto(func_or_code): - if isinstance(func_or_code, types.CodeType): - return _patch_code(func_or_code) - - return functools.update_wrapper( - types.FunctionType( - _patch_code(func_or_code.__code__), - func_or_code.__globals__, - func_or_code.__name__, - func_or_code.__defaults__, - func_or_code.__closure__, - ), - func_or_code - ) +import dis +import struct +import array +import types +import functools + + +try: + _array_to_bytes = array.array.tobytes +except AttributeError: + _array_to_bytes = array.array.tostring + + +class _Bytecode: + def __init__(self): + code = (lambda: x if x else y).__code__.co_code + opcode, oparg = struct.unpack_from('BB', code, 2) + + # Starting with Python 3.6, the bytecode format has been changed to use + # 16-bit words (8-bit opcode + 8-bit argument) for each instruction, + # as opposed to previously 24-bit (8-bit opcode + 16-bit argument) for + # instructions that expect an argument or just 8-bit for those that don't. + # https://bugs.python.org/issue26647 + if dis.opname[opcode] == 'POP_JUMP_IF_FALSE': + self.argument = struct.Struct('B') + self.have_argument = 0 + # As of Python 3.6, jump targets are still addressed by their byte + # unit. This, however, is matter to change, so that jump targets, + # in the future, will refer to the code unit (address in bytes / 2). + # https://bugs.python.org/issue26647 + self.jump_unit = 8 // oparg + else: + self.argument = struct.Struct('= _BYTECODE.have_argument: + oparg = extended_arg | _BYTECODE.argument.unpack_from(code, pos)[0] + pos += _BYTECODE.argument.size + + if opcode == dis.EXTENDED_ARG: + extended_arg = oparg << _BYTECODE.argument_bits + extended_arg_offset = offset + continue + + extended_arg = 0 + extended_arg_offset = None + yield (dis.opname[opcode], oparg, offset) + +def _get_instruction_size(opname, oparg=0): + size = 1 + + extended_arg = oparg >> _BYTECODE.argument_bits + if extended_arg != 0: + size += _get_instruction_size('EXTENDED_ARG', extended_arg) + oparg &= (1 << _BYTECODE.argument_bits) - 1 + + opcode = dis.opmap[opname] + if opcode >= _BYTECODE.have_argument: + size += _BYTECODE.argument.size + + return size + +def _get_instructions_size(ops): + size = 0 + for op in ops: + if isinstance(op, str): + size += _get_instruction_size(op) + else: + size += _get_instruction_size(*op) + return size + +def _write_instruction(buf, pos, opname, oparg=0): + extended_arg = oparg >> _BYTECODE.argument_bits + if extended_arg != 0: + pos = _write_instruction(buf, pos, 'EXTENDED_ARG', extended_arg) + oparg &= (1 << _BYTECODE.argument_bits) - 1 + + opcode = dis.opmap[opname] + buf[pos] = opcode + pos += 1 + + if opcode >= _BYTECODE.have_argument: + _BYTECODE.argument.pack_into(buf, pos, oparg) + pos += _BYTECODE.argument.size + + return pos + +def _write_instructions(buf, pos, ops): + for op in ops: + if isinstance(op, str): + pos = _write_instruction(buf, pos, op) + else: + pos = _write_instruction(buf, pos, *op) + return pos + +def _find_labels_and_gotos(code): + labels = {} + gotos = [] + + block_stack = [] + block_counter = 0 + block_exits = [] + + opname1 = oparg1 = offset1 = None + opname2 = oparg2 = offset2 = None + opname3 = oparg3 = offset3 = None + + for opname4, oparg4, offset4 in _parse_instructions(code.co_code): + if opname1 in ('LOAD_GLOBAL', 'LOAD_NAME'): + if opname2 == 'LOAD_ATTR' and opname3 == 'POP_TOP': + name = code.co_names[oparg1] + if name == 'label': + labels[oparg2] = (offset1, + offset4, + tuple(block_stack)) + elif name == 'goto': + gotos.append((offset1, + offset4, + oparg2, + tuple(block_stack))) + elif opname1 in ('SETUP_LOOP', + 'SETUP_EXCEPT', 'SETUP_FINALLY', + 'SETUP_WITH', 'SETUP_ASYNC_WITH'): + block_counter += 1 + block_stack.append((opname1, block_counter)) + elif not _BYTECODE.has_loop_blocks and opname1 == 'FOR_ITER': + block_counter += 1 + block_stack.append((opname1, block_counter)) + block_exits.append(offset1 + oparg1) + elif opname1 == 'POP_BLOCK' and block_stack: + block_stack.pop() + elif block_exits and offset1 == block_exits[-1] and block_stack: + block_stack.pop() + block_exits.pop() + + opname1, oparg1, offset1 = opname2, oparg2, offset2 + opname2, oparg2, offset2 = opname3, oparg3, offset3 + opname3, oparg3, offset3 = opname4, oparg4, offset4 + + return labels, gotos + + +def _inject_nop_sled(buf, pos, end): + while pos < end: + pos = _write_instruction(buf, pos, 'NOP') + + +def _patch_code(code): + labels, gotos = _find_labels_and_gotos(code) + buf = array.array('B', code.co_code) + + for pos, end, _ in labels.values(): + _inject_nop_sled(buf, pos, end) + + for pos, end, label, origin_stack in gotos: + try: + _, target, target_stack = labels[label] + except KeyError: + raise SyntaxError('Unknown label {0!r}'.format(code.co_names[label])) + + target_depth = len(target_stack) + if origin_stack[:target_depth] != target_stack: + raise SyntaxError('Jump into different block') + + ops = [] + for block, _ in origin_stack[target_depth:]: + if block == 'FOR_ITER': + ops.append('POP_TOP') + else: + ops.append('POP_BLOCK') + ops.append(('JUMP_ABSOLUTE', target // _BYTECODE.jump_unit)) + + if pos + _get_instructions_size(ops) > end: + # not enough space, add code at buffer end and jump there + buf_end = len(buf) + + go_to_end_ops = [('JUMP_ABSOLUTE', buf_end // _BYTECODE.jump_unit)] + + if pos + _get_instructions_size(go_to_end_ops) > end: + # not sure if reachable + raise SyntaxError('Goto in an incredibly huge function') + + pos = _write_instructions(buf, pos, go_to_end_ops) + _inject_nop_sled(buf, pos, end) + + buf.extend([0] * _get_instructions_size(ops)) + _write_instructions(buf, buf_end, ops) + else: + pos = _write_instructions(buf, pos, ops) + _inject_nop_sled(buf, pos, end) + + return _make_code(code, _array_to_bytes(buf)) + + +def with_goto(func_or_code): + if isinstance(func_or_code, types.CodeType): + return _patch_code(func_or_code) + + return functools.update_wrapper( + types.FunctionType( + _patch_code(func_or_code.__code__), + func_or_code.__globals__, + func_or_code.__name__, + func_or_code.__defaults__, + func_or_code.__closure__, + ), + func_or_code + ) diff --git a/test_goto.py b/test_goto.py index 83ebb97..27c7199 100644 --- a/test_goto.py +++ b/test_goto.py @@ -1,233 +1,233 @@ -import sys -import pytest -from goto import with_goto - -CODE = '''\ -i = 0 -result = [] - -label .start -if i == 10: - goto .end - -result.append(i) -i += 1 -goto .start - -label .end -''' - -EXPECTED = list(range(10)) - - -def test_range_as_code(): - ns = {} - exec(with_goto(compile(CODE, '', 'exec')), ns) - assert ns['result'] == EXPECTED - - -def make_function(code): - lines = ['def func():'] - for line in code: - lines.append(' ' + line) - lines.append(' return result') - - ns = {} - exec('\n'.join(lines), ns) - return ns['func'] - - -def test_range_as_function(): - assert with_goto(make_function(CODE.splitlines()))() == EXPECTED - - -def test_EXTENDED_ARG(): - code = [] - code.append('result = True') - code.append('goto .foo') - for i in range(2**16): - code.append('label .l{0}'.format(i)) - code.append('result = "dead code"') - code.append('label .foo') - assert with_goto(make_function(code))() is True - - -def test_jump_out_of_loop(): - @with_goto - def func(): - for i in range(10): - goto .end - label .end - return i - - assert func() == 0 - - -def test_jump_out_of_loop_and_survive(): - @with_goto - def func(): - for i in range(10): - for j in range(10): - goto .end - label .end - return (i, j) - - assert func() == (9, 0) - - -def test_jump_into_loop(): - def func(): - for i in range(10): - label .loop - goto .loop - - pytest.raises(SyntaxError, with_goto, func) - -def test_jump_out_of_nested_2_loops(): - @with_goto - def func(): - x = 1 - for i in range(2): - for j in range(2): - # These are more than 256 bytes of bytecode - x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x - x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x - x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x - - goto .end - label .end - return (i, j) - - assert func() == (0, 0) - -def test_jump_out_of_nested_11_loops(): - @with_goto - def func(): - x = 1 - for i1 in range(2): - for i2 in range(2): - for i3 in range(2): - for i4 in range(2): - for i5 in range(2): - for i6 in range(2): - for i7 in range(2): - for i8 in range(2): - for i9 in range(2): - for i10 in range(2): - for i11 in range(2): - # These are more than - # 256 bytes of bytecode - x += (x+x+x+x+x+x+x+x+x+ - x+x+x+x+x+x+x+x+x+ - x+x+x+x+x+x+x+x+x) - x += (x+x+x+x+x+x+x+x+x+ - x+x+x+x+x+x+x+x+x+ - x+x+x+x+x+x+x+x+x) - x += (x+x+x+x+x+x+x+x+x+ - x+x+x+x+x+x+x+x+x+ - x+x+x+x+x+x+x+x+x) - - goto .end - label .end - return (i1, i2, i3, i4, i5, i6, i7, i8, i9, i10, i11) - - assert func() == (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) - -def test_jump_across_loops(): - def func(): - for i in range(10): - goto .other_loop - - for i in range(10): - label .other_loop - - pytest.raises(SyntaxError, with_goto, func) - - -def test_jump_out_of_try_block(): - @with_goto - def func(): - try: - rv = None - goto .end - except: - rv = 'except' - finally: - rv = 'finally' - label .end - return rv - - assert func() == None - - -"""def test_jump_out_of_try_block_and_survive(): - @with_goto - def func(): - for i in range(10): - try: - rv = None - goto .end - except: - rv = 'except' - finally: - rv = 'finally' - label .end - return (i, rv) - - assert func() == (9, None)""" - -def test_jump_into_try_block(): - def func(): - try: - label .block - except: - pass - goto .block - - pytest.raises(SyntaxError, with_goto, func) - - -"""def test_jump_out_of_except_block(): - @with_goto - def func(): - try: - rv = 1 / 0 - except: - rv = 'except' - goto .end - finally: - rv = 'finally' - label .end - return rv - - assert func() == 'except'""" - - -"""def test_jump_into_except_block(): - def func(): - try: - pass - except: - label .block - pass - goto .block - - pytest.raises(SyntaxError, with_goto, func)""" - - -def test_jump_to_unknown_label(): - def func(): - goto .unknown - - pytest.raises(SyntaxError, with_goto, func) - - -def test_function_is_copy(): - def func(): - pass - - func.foo = 'bar' - newfunc = with_goto(func) - - assert newfunc is not func - assert newfunc.foo == 'bar' +import sys +import pytest +from goto import with_goto + +CODE = '''\ +i = 0 +result = [] + +label .start +if i == 10: + goto .end + +result.append(i) +i += 1 +goto .start + +label .end +''' + +EXPECTED = list(range(10)) + + +def test_range_as_code(): + ns = {} + exec(with_goto(compile(CODE, '', 'exec')), ns) + assert ns['result'] == EXPECTED + + +def make_function(code): + lines = ['def func():'] + for line in code: + lines.append(' ' + line) + lines.append(' return result') + + ns = {} + exec('\n'.join(lines), ns) + return ns['func'] + + +def test_range_as_function(): + assert with_goto(make_function(CODE.splitlines()))() == EXPECTED + + +def test_EXTENDED_ARG(): + code = [] + code.append('result = True') + code.append('goto .foo') + for i in range(2**16): + code.append('label .l{0}'.format(i)) + code.append('result = "dead code"') + code.append('label .foo') + assert with_goto(make_function(code))() is True + + +def test_jump_out_of_loop(): + @with_goto + def func(): + for i in range(10): + goto .end + label .end + return i + + assert func() == 0 + + +def test_jump_out_of_loop_and_survive(): + @with_goto + def func(): + for i in range(10): + for j in range(10): + goto .end + label .end + return (i, j) + + assert func() == (9, 0) + + +def test_jump_into_loop(): + def func(): + for i in range(10): + label .loop + goto .loop + + pytest.raises(SyntaxError, with_goto, func) + +def test_jump_out_of_nested_2_loops(): + @with_goto + def func(): + x = 1 + for i in range(2): + for j in range(2): + # These are more than 256 bytes of bytecode + x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x + x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x + x += x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x + + goto .end + label .end + return (i, j) + + assert func() == (0, 0) + +def test_jump_out_of_nested_11_loops(): + @with_goto + def func(): + x = 1 + for i1 in range(2): + for i2 in range(2): + for i3 in range(2): + for i4 in range(2): + for i5 in range(2): + for i6 in range(2): + for i7 in range(2): + for i8 in range(2): + for i9 in range(2): + for i10 in range(2): + for i11 in range(2): + # These are more than + # 256 bytes of bytecode + x += (x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x) + x += (x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x) + x += (x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x+ + x+x+x+x+x+x+x+x+x) + + goto .end + label .end + return (i1, i2, i3, i4, i5, i6, i7, i8, i9, i10, i11) + + assert func() == (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) + +def test_jump_across_loops(): + def func(): + for i in range(10): + goto .other_loop + + for i in range(10): + label .other_loop + + pytest.raises(SyntaxError, with_goto, func) + + +def test_jump_out_of_try_block(): + @with_goto + def func(): + try: + rv = None + goto .end + except: + rv = 'except' + finally: + rv = 'finally' + label .end + return rv + + assert func() == None + + +"""def test_jump_out_of_try_block_and_survive(): + @with_goto + def func(): + for i in range(10): + try: + rv = None + goto .end + except: + rv = 'except' + finally: + rv = 'finally' + label .end + return (i, rv) + + assert func() == (9, None)""" + +def test_jump_into_try_block(): + def func(): + try: + label .block + except: + pass + goto .block + + pytest.raises(SyntaxError, with_goto, func) + + +"""def test_jump_out_of_except_block(): + @with_goto + def func(): + try: + rv = 1 / 0 + except: + rv = 'except' + goto .end + finally: + rv = 'finally' + label .end + return rv + + assert func() == 'except'""" + + +"""def test_jump_into_except_block(): + def func(): + try: + pass + except: + label .block + pass + goto .block + + pytest.raises(SyntaxError, with_goto, func)""" + + +def test_jump_to_unknown_label(): + def func(): + goto .unknown + + pytest.raises(SyntaxError, with_goto, func) + + +def test_function_is_copy(): + def func(): + pass + + func.foo = 'bar' + newfunc = with_goto(func) + + assert newfunc is not func + assert newfunc.foo == 'bar' From 281cc1c1becb754257d551ec79e0abbf88469140 Mon Sep 17 00:00:00 2001 From: condut Date: Sat, 14 Dec 2019 02:16:15 +0200 Subject: [PATCH 25/48] Remove the commented out tests (They'll come back in the next PR). (The new test helps test functionality that needs special attention at least on py38) --- test_goto.py | 45 --------------------------------------------- 1 file changed, 45 deletions(-) diff --git a/test_goto.py b/test_goto.py index 27c7199..41a9f0d 100644 --- a/test_goto.py +++ b/test_goto.py @@ -159,23 +159,6 @@ def func(): assert func() == None - -"""def test_jump_out_of_try_block_and_survive(): - @with_goto - def func(): - for i in range(10): - try: - rv = None - goto .end - except: - rv = 'except' - finally: - rv = 'finally' - label .end - return (i, rv) - - assert func() == (9, None)""" - def test_jump_into_try_block(): def func(): try: @@ -187,34 +170,6 @@ def func(): pytest.raises(SyntaxError, with_goto, func) -"""def test_jump_out_of_except_block(): - @with_goto - def func(): - try: - rv = 1 / 0 - except: - rv = 'except' - goto .end - finally: - rv = 'finally' - label .end - return rv - - assert func() == 'except'""" - - -"""def test_jump_into_except_block(): - def func(): - try: - pass - except: - label .block - pass - goto .block - - pytest.raises(SyntaxError, with_goto, func)""" - - def test_jump_to_unknown_label(): def func(): goto .unknown From e26a2d0683eaf155c026b2981dd3861ca09cfd8c Mon Sep 17 00:00:00 2001 From: condut Date: Fri, 13 Dec 2019 00:18:50 +0200 Subject: [PATCH 26/48] Refactor previous changes to avoid duplication & fix safety issues (Candidate for moving to 'fix' branch?) --- goto.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/goto.py b/goto.py index 074ef76..18c1f54 100644 --- a/goto.py +++ b/goto.py @@ -112,6 +112,15 @@ def _get_instructions_size(ops): size += _get_instruction_size(op) else: size += _get_instruction_size(*op) + return size + +def _get_instructions_size(ops): + size = 0 + for op in ops: + if isinstance(op, str): + size += _get_instruction_size(op) + else: + size += _get_instruction_size(*op) return size def _write_instruction(buf, pos, opname, oparg=0): From 5a8064d492774f0cab986650cdca4113ab93eb38 Mon Sep 17 00:00:00 2001 From: condut Date: Fri, 13 Dec 2019 15:58:18 +0200 Subject: [PATCH 27/48] Fix test_jump_out_of_try_block_and_survive. Add new test_jump_out_of_try_block_and_live --- goto.py | 9 ++++++++- test_goto.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/goto.py b/goto.py index 18c1f54..3ff2865 100644 --- a/goto.py +++ b/goto.py @@ -4,6 +4,10 @@ import types import functools +try: + import __pypy__ +except: + __pypy__ = None try: _array_to_bytes = array.array.tobytes @@ -220,11 +224,14 @@ def _patch_code(code): raise SyntaxError('Jump into different block') ops = [] - for block, _ in origin_stack[target_depth:]: + for block, _ in reversed(origin_stack[target_depth:]): if block == 'FOR_ITER': ops.append('POP_TOP') else: ops.append('POP_BLOCK') + if __pypy__ and block == 'SETUP_FINALLY': # pypy 3.6 keep a block around until END_FINALLY; python 3.8 reuses SETUP_FINALLY for SETUP_EXCEPT. What will pypy 3.8 do? + ops.append(('LOAD_CONST', code.co_consts.index(None))) + ops.append('END_FINALLY') ops.append(('JUMP_ABSOLUTE', target // _BYTECODE.jump_unit)) if pos + _get_instructions_size(ops) > end: diff --git a/test_goto.py b/test_goto.py index cbb7a67..a66cf97 100644 --- a/test_goto.py +++ b/test_goto.py @@ -100,6 +100,20 @@ def func(): assert func() == (0, 0) +def test_jump_out_of_nested_4_loops_and_survive(): + @with_goto + def func(): + for i in range(2): + for j in range(2): + for k in range(2): + for m in range(2): + for n in range(2): + goto .end + label .end + return (i, j, k, m, n) + + assert func() == (1, 0, 0, 0, 0) + def test_jump_out_of_nested_11_loops(): @with_goto def func(): @@ -159,6 +173,39 @@ def func(): assert func() == None +def test_jump_out_of_try_block_and_survive(): + @with_goto + def func(): + for i in range(10): + try: + rv = None + goto .end + except: + rv = 'except' + finally: + rv = 'finally' + label .end + return (i, rv) + + assert func() == (9, None) + +def test_jump_out_of_try_block_and_live(): + @with_goto + def func(): + for i in range(3): + for j in range(3): + try: + rv = None + goto .end + except: + rv = 'except' + finally: + rv = 'finally' + label .end + return (i, j, rv) + + assert func() == (2, 2, None) + def test_jump_into_try_block(): def func(): try: From 3055aa7b4c88df3ef94f469547b25c56f71055ec Mon Sep 17 00:00:00 2001 From: condut Date: Fri, 13 Dec 2019 16:05:27 +0200 Subject: [PATCH 28/48] Avoid patching code again and again in nested functions Uses a weak dictionary just in case code objects can be gc'd (can they? maybe on some implementations?) --- goto.py | 11 ++++++++++- test_goto.py | 12 ++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/goto.py b/goto.py index 3ff2865..04dd38f 100644 --- a/goto.py +++ b/goto.py @@ -3,6 +3,7 @@ import array import types import functools +import weakref try: import __pypy__ @@ -47,6 +48,7 @@ def argument_bits(self): _BYTECODE = _Bytecode() +_patched_code_cache = weakref.WeakKeyDictionary() def _make_code(code, codestring): try: @@ -207,6 +209,10 @@ def _inject_nop_sled(buf, pos, end): def _patch_code(code): + new_code = _patched_code_cache.get(code) + if new_code is not None: + return new_code + labels, gotos = _find_labels_and_gotos(code) buf = array.array('B', code.co_code) @@ -253,7 +259,10 @@ def _patch_code(code): pos = _write_instructions(buf, pos, ops) _inject_nop_sled(buf, pos, end) - return _make_code(code, _array_to_bytes(buf)) + new_code = _make_code(code, _array_to_bytes(buf)) + + _patched_code_cache[code] = new_code + return new_code def with_goto(func_or_code): diff --git a/test_goto.py b/test_goto.py index a66cf97..3d5dd76 100644 --- a/test_goto.py +++ b/test_goto.py @@ -242,3 +242,15 @@ def func(): assert newfunc is not func assert newfunc.foo == 'bar' + +def test_code_is_not_copy(): + def outer_func(): + @with_goto + def inner_func(): + goto .test + label .test + return inner_func + + assert outer_func() is not outer_func() + assert outer_func().__code__ is outer_func().__code__ + From 16ad6934de2188253779658adba6277f8d956cab Mon Sep 17 00:00:00 2001 From: condut Date: Fri, 13 Dec 2019 16:49:24 +0200 Subject: [PATCH 29/48] Fix previous change for python2.6 (doesn't support Code weakrefs) --- goto.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/goto.py b/goto.py index 04dd38f..05f3f13 100644 --- a/goto.py +++ b/goto.py @@ -48,7 +48,11 @@ def argument_bits(self): _BYTECODE = _Bytecode() -_patched_code_cache = weakref.WeakKeyDictionary() +_patched_code_cache = weakref.WeakKeyDictionary() # use a weak dictionary in case code objects can be garbage-collected +try: + _patched_code_cache[_Bytecode.__init__.__code__] = None +except TypeError: + _patched_code_cache = {} # ...unless not supported def _make_code(code, codestring): try: From 5b7eb5aaebec855bf86114818cc3c900a030f307 Mon Sep 17 00:00:00 2001 From: condut Date: Fri, 13 Dec 2019 17:16:13 +0200 Subject: [PATCH 30/48] Fix with blocks (add tests) add warnings for easier debugging (for peace of heart - they weren't currently triggered by any tests) --- goto.py | 33 ++++++++++++++++++++++++++------- test_goto.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 7 deletions(-) diff --git a/goto.py b/goto.py index 05f3f13..ecb4e9b 100644 --- a/goto.py +++ b/goto.py @@ -4,6 +4,7 @@ import types import functools import weakref +import warnings try: import __pypy__ @@ -74,7 +75,7 @@ def _make_code(code, codestring): return types.CodeType(*args) -def _parse_instructions(code): +def _parse_instructions(code, yield_nones_at_end=0): extended_arg = 0 extended_arg_offset = None pos = 0 @@ -100,6 +101,9 @@ def _parse_instructions(code): extended_arg = 0 extended_arg_offset = None yield (dis.opname[opcode], oparg, offset) + + for _ in range(yield_nones_at_end): + yield (None, None, None) def _get_instruction_size(opname, oparg=0): size = 1 @@ -157,6 +161,9 @@ def _write_instructions(buf, pos, ops): pos = _write_instruction(buf, pos, *op) return pos +def _warn_bug(msg): + warnings.warn("Internal error detected - result of with_goto may be incorrect. (%s)" % msg) + def _find_labels_and_gotos(code): labels = {} gotos = [] @@ -168,8 +175,14 @@ def _find_labels_and_gotos(code): opname1 = oparg1 = offset1 = None opname2 = oparg2 = offset2 = None opname3 = oparg3 = offset3 = None + + def pop_block(): + if block_stack: + block_stack.pop() + else: + _warn_bug("can't pop block") - for opname4, oparg4, offset4 in _parse_instructions(code.co_code): + for opname4, oparg4, offset4 in _parse_instructions(code.co_code, 3): if opname1 in ('LOAD_GLOBAL', 'LOAD_NAME'): if opname2 == 'LOAD_ATTR' and opname3 == 'POP_TOP': name = code.co_names[oparg1] @@ -194,16 +207,19 @@ def _find_labels_and_gotos(code): block_counter += 1 block_stack.append((opname1, block_counter)) block_exits.append(offset1 + oparg1) - elif opname1 == 'POP_BLOCK' and block_stack: - block_stack.pop() - elif block_exits and offset1 == block_exits[-1] and block_stack: - block_stack.pop() + elif opname1 == 'POP_BLOCK': + pop_block() + elif block_exits and offset1 == block_exits[-1]: + pop_block() block_exits.pop() opname1, oparg1, offset1 = opname2, oparg2, offset2 opname2, oparg2, offset2 = opname3, oparg3, offset3 opname3, oparg3, offset3 = opname4, oparg4, offset4 + if block_stack: + _warn_bug("block stack not empty") + return labels, gotos @@ -239,7 +255,10 @@ def _patch_code(code): ops.append('POP_TOP') else: ops.append('POP_BLOCK') - if __pypy__ and block == 'SETUP_FINALLY': # pypy 3.6 keep a block around until END_FINALLY; python 3.8 reuses SETUP_FINALLY for SETUP_EXCEPT. What will pypy 3.8 do? + if block in ('SETUP_WITH', 'SETUP_ASYNC_WITH'): + ops.append('POP_TOP') + # pypy 3.6 keeps a block around until END_FINALLY; python 3.8 reuses SETUP_FINALLY for SETUP_EXCEPT (where END_FINALLY is not accepted). What will pypy 3.8 do? + if __pypy__ and block in ('SETUP_FINALLY', 'SETUP_WITH', 'SETUP_ASYNC_WITH'): ops.append(('LOAD_CONST', code.co_consts.index(None))) ops.append('END_FINALLY') ops.append(('JUMP_ABSOLUTE', target // _BYTECODE.jump_unit)) diff --git a/test_goto.py b/test_goto.py index 3d5dd76..303ae48 100644 --- a/test_goto.py +++ b/test_goto.py @@ -75,6 +75,19 @@ def func(): assert func() == (9, 0) +def test_jump_out_of_loop_and_live(): + @with_goto + def func(): + for i in range(10): + for j in range(10): + for k in range(10): + goto .end + label .end + return (i, j, k) + + assert func() == (9, 0, 0) + + def test_jump_into_loop(): def func(): for i in range(10): @@ -157,6 +170,41 @@ def func(): pytest.raises(SyntaxError, with_goto, func) +class Context: + def __init__(self): + self.enters = 0 + self.exits = 0 + def __enter__(self): + self.enters += 1 + return self + def __exit__(self, a, b, c): + self.exits += 1 + return self + def data(self): + return (self.enters, self.exits) + +def test_jump_out_of_with_block(): + @with_goto + def func(): + with Context() as c: + goto .out + label .out + return c.data() + + assert func()== (1, 0) + +def test_jump_out_of_with_block_and_live(): + @with_goto + def func(): + c = Context() + for i in range(3): + for j in range(3): + with c: + goto .out + label .out + return (i, j, c.data()) + + assert func() == (2, 0, (3, 0)) def test_jump_out_of_try_block(): @with_goto From 26efae66d9619ab401aee31f57cad4cd6da1a97c Mon Sep 17 00:00:00 2001 From: condut Date: Fri, 13 Dec 2019 18:27:17 +0200 Subject: [PATCH 31/48] Support jumping out of except & finally blocks! --- goto.py | 68 +++++++++++++++++++++++++++++++++---- test_goto.py | 96 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 157 insertions(+), 7 deletions(-) diff --git a/goto.py b/goto.py index ecb4e9b..e53c6f8 100644 --- a/goto.py +++ b/goto.py @@ -41,6 +41,9 @@ def __init__(self): self.jump_unit = 1 self.has_loop_blocks = 'SETUP_LOOP' in dis.opmap + self.has_pop_except = 'POP_EXCEPT' in dis.opmap + self.has_setup_with = 'SETUP_WITH' in dis.opmap + self.has_setup_except = 'SETUP_EXCEPT' in dis.opmap @property def argument_bits(self): @@ -170,19 +173,44 @@ def _find_labels_and_gotos(code): block_stack = [] block_counter = 0 - block_exits = [] + for_exits = [] + excepts = [] + finallies = [] - opname1 = oparg1 = offset1 = None + opname0 = oparg0 = offset0 = None + opname1 = oparg1 = offset1 = None # the main one we're looking at each loop iteration opname2 = oparg2 = offset2 = None opname3 = oparg3 = offset3 = None + def replace_block_in_stack(stack, old_block, new_block): + for i, block in enumerate(stack): + if block == old_block: + stack[i] = new_block + + def replace_block(old_block, new_block): + replace_block_in_stack(block_stack, old_block, new_block) + for label in labels: + replace_block_in_stack(labels[label][2], old_block, new_block) + for goto in gotos: + replace_block_in_stack(goto[3], old_block, new_block) + def pop_block(): if block_stack: block_stack.pop() else: _warn_bug("can't pop block") + + def pop_block_of_type(type): + if block_stack and block_stack[-1][0] != type: + # in 3.8, only finally blocks are supported, so we must determine the except/finally nature ourselves, and replace the block afterwards + if not _BYTECODE.has_setup_except and type == "" and block_stack[-1][0] == '': + replace_block(block_stack[-1], (type, block_stack[-1][1])) + else: + _warn_bug("mismatched block type") + pop_block() for opname4, oparg4, offset4 in _parse_instructions(code.co_code, 3): + # check for special opcodes if opname1 in ('LOAD_GLOBAL', 'LOAD_NAME'): if opname2 == 'LOAD_ATTR' and opname3 == 'POP_TOP': name = code.co_names[oparg1] @@ -192,27 +220,49 @@ def pop_block(): raise SyntaxError('Ambiguous label {0!r}'.format(co_name)) labels[oparg2] = (offset1, offset4, - tuple(block_stack)) + list(block_stack)) elif name == 'goto': gotos.append((offset1, offset4, oparg2, - tuple(block_stack))) + list(block_stack))) elif opname1 in ('SETUP_LOOP', 'SETUP_EXCEPT', 'SETUP_FINALLY', 'SETUP_WITH', 'SETUP_ASYNC_WITH'): block_counter += 1 block_stack.append((opname1, block_counter)) + if opname1 == 'SETUP_EXCEPT' and _BYTECODE.has_pop_except: + excepts.append(offset1 + oparg1) + elif opname1 == 'SETUP_FINALLY': + finallies.append(offset1 + oparg1) elif not _BYTECODE.has_loop_blocks and opname1 == 'FOR_ITER': block_counter += 1 block_stack.append((opname1, block_counter)) - block_exits.append(offset1 + oparg1) + for_exits.append(offset1 + oparg1) elif opname1 == 'POP_BLOCK': pop_block() - elif block_exits and offset1 == block_exits[-1]: + elif opname1 == 'POP_EXCEPT': + pop_block_of_type('') + elif opname1 == 'END_FINALLY': + if opname0 != 'JUMP_FORWARD': # hack for dummy end-finally in except block (correct fix would be a jump-aware reading of instructions!) + pop_block_of_type('') + elif opname1 in ('WITH_CLEANUP', 'WITH_CLEANUP_START') and _BYTECODE.has_setup_with: + block_stack.append(('', -1)) # temporary block to match END_FINALLY + + # check for special offsets + if for_exits and offset1 == for_exits[-1]: pop_block() - block_exits.pop() + for_exits.pop() + if excepts and offset1 == excepts[-1]: + block_counter += 1 + block_stack.append(('', block_counter)) + excepts.pop() + if finallies and offset1 == finallies[-1]: + block_counter += 1 + block_stack.append(('', block_counter)) + finallies.pop() + opname0, oparg0, offset0 = opname1, oparg1, offset1 opname1, oparg1, offset1 = opname2, oparg2, offset2 opname2, oparg2, offset2 = opname3, oparg3, offset3 opname3, oparg3, offset3 = opname4, oparg4, offset4 @@ -253,6 +303,10 @@ def _patch_code(code): for block, _ in reversed(origin_stack[target_depth:]): if block == 'FOR_ITER': ops.append('POP_TOP') + elif block == '': + ops.append('POP_EXCEPT') + elif block == '': + ops.append('END_FINALLY') else: ops.append('POP_BLOCK') if block in ('SETUP_WITH', 'SETUP_ASYNC_WITH'): diff --git a/test_goto.py b/test_goto.py index 303ae48..2503c74 100644 --- a/test_goto.py +++ b/test_goto.py @@ -205,6 +205,28 @@ def func(): return (i, j, c.data()) assert func() == (2, 0, (3, 0)) + +def test_jump_into_with_block(): + def func(): + with Context() as c: + label .block + goto .block + + pytest.raises(SyntaxError, with_goto, func) + +def test_generator(): + @with_goto + def func(): + yield 0 + yield 1 + goto .x + yield 2 + yield 3 + label .x + yield 4 + yield 5 + + assert tuple(func()) == (0, 1, 4, 5) def test_jump_out_of_try_block(): @with_goto @@ -265,6 +287,80 @@ def func(): pytest.raises(SyntaxError, with_goto, func) +def test_jump_out_of_except_block(): + @with_goto + def func(): + try: + rv = 1 / 0 + except: + rv = 'except' + goto .end + finally: + rv = 'finally' + label .end + return rv + + assert func() == 'except' + +def test_jump_out_of_except_block_and_live(): + @with_goto + def func(): + for i in range(3): + for j in range(3): + try: + rv = 1 / 0 + except: + rv = 'except' + goto .end + finally: + rv = 'finally' + label .end + return (i, j, rv) + + assert func() == (2, 0, 'except') + +"""def test_jump_into_except_block(): + def func(): + try: + pass + except: + label .block + pass + goto .block + + pytest.raises(SyntaxError, with_goto, func)""" + +def test_jump_out_of_finally_block(): + @with_goto + def func(): + try: + rv = None + finally: + rv = 'finally' + goto .end + rv = 'end' + label .end + return rv + + assert func() == 'finally' + +def test_jump_out_of_finally_block_and_live(): + @with_goto + def func(): + for i in range(3): + for j in range(3): + try: + rv = None + finally: + rv = 'finally' + goto .end + rv = 'end' + label .end + return i, j, rv + + assert func() == (2, 0, 'finally') + + def test_jump_to_unknown_label(): def func(): goto .unknown From 26742550a8092e4c7aac47ffecc4c55d7a9731bf Mon Sep 17 00:00:00 2001 From: condut Date: Fri, 13 Dec 2019 18:34:47 +0200 Subject: [PATCH 32/48] Add combined try/catch/finally test and remove unneccessary op (later fix could go into fix branch as well) --- test_goto.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/test_goto.py b/test_goto.py index 2503c74..8534759 100644 --- a/test_goto.py +++ b/test_goto.py @@ -360,6 +360,32 @@ def func(): assert func() == (2, 0, 'finally') +def test_jump_out_of_try_in_except_in_finally_and_live(): + @with_goto + def func(): + for i in range(3): + for j in range(3): + try: + rv = None + finally: + rv = 'finally' + try: + rv = 1 / 0 + except: + rv = 'except' + try: + rv = 'try' + goto .end + except: + rv = 'except2' + finally: + rv = 'finally2' + rv = 'end' + label .end + return i, j, rv + + assert func() == (2, 0, 'try') + def test_jump_to_unknown_label(): def func(): From 28a3d87fc5d4ddcb1139f80d4310db8b1658016d Mon Sep 17 00:00:00 2001 From: condut Date: Fri, 13 Dec 2019 22:24:08 +0200 Subject: [PATCH 33/48] Fixed two dumb bugs that cancelled each other out in most cases (No testcase in this branch, as it'd be tricky to think of one. master branch has testcases becaues it does dis.findlabels) --- goto.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/goto.py b/goto.py index e53c6f8..ff33510 100644 --- a/goto.py +++ b/goto.py @@ -210,6 +210,21 @@ def pop_block_of_type(type): pop_block() for opname4, oparg4, offset4 in _parse_instructions(code.co_code, 3): + endoffset1 = offset2 + + # check for special offsets + if for_exits and offset1 == for_exits[-1]: + pop_block() + for_exits.pop() + if excepts and offset1 == excepts[-1]: + block_counter += 1 + block_stack.append(('', block_counter)) + excepts.pop() + if finallies and offset1 == finallies[-1]: + block_counter += 1 + block_stack.append(('', block_counter)) + finallies.pop() + # check for special opcodes if opname1 in ('LOAD_GLOBAL', 'LOAD_NAME'): if opname2 == 'LOAD_ATTR' and opname3 == 'POP_TOP': @@ -232,13 +247,13 @@ def pop_block_of_type(type): block_counter += 1 block_stack.append((opname1, block_counter)) if opname1 == 'SETUP_EXCEPT' and _BYTECODE.has_pop_except: - excepts.append(offset1 + oparg1) + excepts.append(endoffset1 + oparg1) elif opname1 == 'SETUP_FINALLY': - finallies.append(offset1 + oparg1) + finallies.append(endoffset1 + oparg1) elif not _BYTECODE.has_loop_blocks and opname1 == 'FOR_ITER': block_counter += 1 block_stack.append((opname1, block_counter)) - for_exits.append(offset1 + oparg1) + for_exits.append(endoffset1 + oparg1) elif opname1 == 'POP_BLOCK': pop_block() elif opname1 == 'POP_EXCEPT': @@ -248,19 +263,6 @@ def pop_block_of_type(type): pop_block_of_type('') elif opname1 in ('WITH_CLEANUP', 'WITH_CLEANUP_START') and _BYTECODE.has_setup_with: block_stack.append(('', -1)) # temporary block to match END_FINALLY - - # check for special offsets - if for_exits and offset1 == for_exits[-1]: - pop_block() - for_exits.pop() - if excepts and offset1 == excepts[-1]: - block_counter += 1 - block_stack.append(('', block_counter)) - excepts.pop() - if finallies and offset1 == finallies[-1]: - block_counter += 1 - block_stack.append(('', block_counter)) - finallies.pop() opname0, oparg0, offset0 = opname1, oparg1, offset1 opname1, oparg1, offset1 = opname2, oparg2, offset2 From 0419965603b6c640df22d31e670fef33d6a86ead Mon Sep 17 00:00:00 2001 From: condut Date: Fri, 13 Dec 2019 23:05:30 +0200 Subject: [PATCH 34/48] Fix jumping out of with blocks in py26 (and added test) Plus added extra test from master --- goto.py | 20 ++++++++++++-------- test_goto.py | 45 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 8 deletions(-) diff --git a/goto.py b/goto.py index ff33510..8c3932c 100644 --- a/goto.py +++ b/goto.py @@ -176,6 +176,7 @@ def _find_labels_and_gotos(code): for_exits = [] excepts = [] finallies = [] + last_block = None opname0 = oparg0 = offset0 = None opname1 = oparg1 = offset1 = None # the main one we're looking at each loop iteration @@ -196,7 +197,7 @@ def replace_block(old_block, new_block): def pop_block(): if block_stack: - block_stack.pop() + return block_stack.pop() else: _warn_bug("can't pop block") @@ -207,14 +208,14 @@ def pop_block_of_type(type): replace_block(block_stack[-1], (type, block_stack[-1][1])) else: _warn_bug("mismatched block type") - pop_block() + return pop_block() for opname4, oparg4, offset4 in _parse_instructions(code.co_code, 3): endoffset1 = offset2 # check for special offsets if for_exits and offset1 == for_exits[-1]: - pop_block() + last_block = pop_block() for_exits.pop() if excepts and offset1 == excepts[-1]: block_counter += 1 @@ -255,14 +256,17 @@ def pop_block_of_type(type): block_stack.append((opname1, block_counter)) for_exits.append(endoffset1 + oparg1) elif opname1 == 'POP_BLOCK': - pop_block() + last_block = pop_block() elif opname1 == 'POP_EXCEPT': - pop_block_of_type('') + last_block = pop_block_of_type('') elif opname1 == 'END_FINALLY': if opname0 != 'JUMP_FORWARD': # hack for dummy end-finally in except block (correct fix would be a jump-aware reading of instructions!) - pop_block_of_type('') - elif opname1 in ('WITH_CLEANUP', 'WITH_CLEANUP_START') and _BYTECODE.has_setup_with: - block_stack.append(('', -1)) # temporary block to match END_FINALLY + last_block = pop_block_of_type('') + elif opname1 in ('WITH_CLEANUP', 'WITH_CLEANUP_START'): + if _BYTECODE.has_setup_with: + block_stack.append(('', -1)) # temporary block to match END_FINALLY + else: + replace_block(last_block, ('SETUP_WITH',) + last_block[1:]) # python 2.6 - finally was actually with opname0, oparg0, offset0 = opname1, oparg1, offset1 opname1, oparg1, offset1 = opname2, oparg2, offset2 diff --git a/test_goto.py b/test_goto.py index 8534759..dfa971c 100644 --- a/test_goto.py +++ b/test_goto.py @@ -127,6 +127,39 @@ def func(): assert func() == (1, 0, 0, 0, 0) +def test_large_jumps_in_diff_orders(): + @with_goto + def func(): + goto .start + + if NonConstFalse: + label .finalle + return (i, j, k, m, n, i1, j1, k1, m1, n1, i2, j2, k2, m2, n2) + + label .start + for i in range(2): + for j in range(2): + for k in range(2): + for m in range(2): + for n in range(2): + goto .end + label .end + for i1 in range(2): + for j1 in range(2): + for k1 in range(2): + for m1 in range(2): + for n1 in range(2): + goto .end2 + label .end2 + for i2 in range(2): + for j2 in range(2): + for k2 in range(2): + for m2 in range(2): + for n2 in range(2): + goto .finalle + + assert func() == (0,) * 15 + def test_jump_out_of_nested_11_loops(): @with_goto def func(): @@ -193,6 +226,18 @@ def func(): assert func()== (1, 0) +def test_jump_out_of_with_block_and_survive(): + @with_goto + def func(): + c = Context() + for i in range(3): + with c: + goto .out + label .out + return (i, c.data()) + + assert func() == (2, (3, 0)) + def test_jump_out_of_with_block_and_live(): @with_goto def func(): From ac926dc71fc2505b681a4a12aa1b09dfd0a95bbc Mon Sep 17 00:00:00 2001 From: condut Date: Sat, 14 Dec 2019 02:33:34 +0200 Subject: [PATCH 35/48] Update merge --- goto.py | 29 ++++++++++------------------- test_goto.py | 24 ++++++++++++------------ 2 files changed, 22 insertions(+), 31 deletions(-) diff --git a/goto.py b/goto.py index 8c3932c..2ed3032 100644 --- a/goto.py +++ b/goto.py @@ -104,7 +104,7 @@ def _parse_instructions(code, yield_nones_at_end=0): extended_arg = 0 extended_arg_offset = None yield (dis.opname[opcode], oparg, offset) - + for _ in range(yield_nones_at_end): yield (None, None, None) @@ -129,15 +129,6 @@ def _get_instructions_size(ops): size += _get_instruction_size(op) else: size += _get_instruction_size(*op) - return size - -def _get_instructions_size(ops): - size = 0 - for op in ops: - if isinstance(op, str): - size += _get_instruction_size(op) - else: - size += _get_instruction_size(*op) return size def _write_instruction(buf, pos, opname, oparg=0): @@ -182,28 +173,28 @@ def _find_labels_and_gotos(code): opname1 = oparg1 = offset1 = None # the main one we're looking at each loop iteration opname2 = oparg2 = offset2 = None opname3 = oparg3 = offset3 = None - + def replace_block_in_stack(stack, old_block, new_block): for i, block in enumerate(stack): if block == old_block: stack[i] = new_block - + def replace_block(old_block, new_block): replace_block_in_stack(block_stack, old_block, new_block) for label in labels: replace_block_in_stack(labels[label][2], old_block, new_block) for goto in gotos: replace_block_in_stack(goto[3], old_block, new_block) - + def pop_block(): if block_stack: return block_stack.pop() else: _warn_bug("can't pop block") - + def pop_block_of_type(type): if block_stack and block_stack[-1][0] != type: - # in 3.8, only finally blocks are supported, so we must determine the except/finally nature ourselves, and replace the block afterwards + # in 3.8, only finally blocks are supported, so we must determine the except/finally nature ourselves, and replace the block afterwards if not _BYTECODE.has_setup_except and type == "" and block_stack[-1][0] == '': replace_block(block_stack[-1], (type, block_stack[-1][1])) else: @@ -212,7 +203,7 @@ def pop_block_of_type(type): for opname4, oparg4, offset4 in _parse_instructions(code.co_code, 3): endoffset1 = offset2 - + # check for special offsets if for_exits and offset1 == for_exits[-1]: last_block = pop_block() @@ -225,7 +216,7 @@ def pop_block_of_type(type): block_counter += 1 block_stack.append(('', block_counter)) finallies.pop() - + # check for special opcodes if opname1 in ('LOAD_GLOBAL', 'LOAD_NAME'): if opname2 == 'LOAD_ATTR' and opname3 == 'POP_TOP': @@ -288,7 +279,7 @@ def _patch_code(code): new_code = _patched_code_cache.get(code) if new_code is not None: return new_code - + labels, gotos = _find_labels_and_gotos(code) buf = array.array('B', code.co_code) @@ -343,7 +334,7 @@ def _patch_code(code): _inject_nop_sled(buf, pos, end) new_code = _make_code(code, _array_to_bytes(buf)) - + _patched_code_cache[code] = new_code return new_code diff --git a/test_goto.py b/test_goto.py index dfa971c..8bbf7d3 100644 --- a/test_goto.py +++ b/test_goto.py @@ -131,11 +131,11 @@ def test_large_jumps_in_diff_orders(): @with_goto def func(): goto .start - + if NonConstFalse: label .finalle return (i, j, k, m, n, i1, j1, k1, m1, n1, i2, j2, k2, m2, n2) - + label .start for i in range(2): for j in range(2): @@ -159,7 +159,7 @@ def func(): goto .finalle assert func() == (0,) * 15 - + def test_jump_out_of_nested_11_loops(): @with_goto def func(): @@ -223,7 +223,7 @@ def func(): goto .out label .out return c.data() - + assert func()== (1, 0) def test_jump_out_of_with_block_and_survive(): @@ -235,9 +235,9 @@ def func(): goto .out label .out return (i, c.data()) - + assert func() == (2, (3, 0)) - + def test_jump_out_of_with_block_and_live(): @with_goto def func(): @@ -248,9 +248,9 @@ def func(): goto .out label .out return (i, j, c.data()) - + assert func() == (2, 0, (3, 0)) - + def test_jump_into_with_block(): def func(): with Context() as c: @@ -270,7 +270,7 @@ def func(): label .x yield 4 yield 5 - + assert tuple(func()) == (0, 1, 4, 5) def test_jump_out_of_try_block(): @@ -457,7 +457,7 @@ def func(): assert newfunc is not func assert newfunc.foo == 'bar' - + def test_code_is_not_copy(): def outer_func(): @with_goto @@ -465,7 +465,7 @@ def inner_func(): goto .test label .test return inner_func - + assert outer_func() is not outer_func() assert outer_func().__code__ is outer_func().__code__ - + From 599edb914012aa748c60b3672824bf32955f33be Mon Sep 17 00:00:00 2001 From: condut Date: Sat, 14 Dec 2019 04:09:36 +0200 Subject: [PATCH 36/48] Added while / while true tests & fix code to make them pass --- goto.py | 44 ++++++++++++++++++++----------------------- test_goto.py | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 24 deletions(-) diff --git a/goto.py b/goto.py index 2ed3032..d48af79 100644 --- a/goto.py +++ b/goto.py @@ -164,9 +164,7 @@ def _find_labels_and_gotos(code): block_stack = [] block_counter = 0 - for_exits = [] - excepts = [] - finallies = [] + block_exits = [] last_block = None opname0 = oparg0 = offset0 = None @@ -204,18 +202,20 @@ def pop_block_of_type(type): for opname4, oparg4, offset4 in _parse_instructions(code.co_code, 3): endoffset1 = offset2 - # check for special offsets - if for_exits and offset1 == for_exits[-1]: - last_block = pop_block() - for_exits.pop() - if excepts and offset1 == excepts[-1]: - block_counter += 1 - block_stack.append(('', block_counter)) - excepts.pop() - if finallies and offset1 == finallies[-1]: - block_counter += 1 - block_stack.append(('', block_counter)) - finallies.pop() + # check for block exits + while block_exits and offset1 == block_exits[-1][-1]: + block, counter, _ = block_exits.pop() + + # Necessary for FOR_ITER and sometimes needed for other blocks as well + if block_stack and (block, counter) == block_stack[-1][:2]: + last_block = pop_block() + + if block == 'SETUP_EXCEPT' and _BYTECODE.has_pop_except: + block_counter += 1 + block_stack.append(('', block_counter)) + elif block == 'SETUP_FINALLY': + block_counter += 1 + block_stack.append(('', block_counter)) # check for special opcodes if opname1 in ('LOAD_GLOBAL', 'LOAD_NAME'): @@ -235,17 +235,11 @@ def pop_block_of_type(type): list(block_stack))) elif opname1 in ('SETUP_LOOP', 'SETUP_EXCEPT', 'SETUP_FINALLY', - 'SETUP_WITH', 'SETUP_ASYNC_WITH'): - block_counter += 1 - block_stack.append((opname1, block_counter)) - if opname1 == 'SETUP_EXCEPT' and _BYTECODE.has_pop_except: - excepts.append(endoffset1 + oparg1) - elif opname1 == 'SETUP_FINALLY': - finallies.append(endoffset1 + oparg1) - elif not _BYTECODE.has_loop_blocks and opname1 == 'FOR_ITER': + 'SETUP_WITH', 'SETUP_ASYNC_WITH') or \ + (not _BYTECODE.has_loop_blocks and opname1 == 'FOR_ITER'): block_counter += 1 block_stack.append((opname1, block_counter)) - for_exits.append(endoffset1 + oparg1) + block_exits.append((opname1, block_counter, endoffset1 + oparg1)) elif opname1 == 'POP_BLOCK': last_block = pop_block() elif opname1 == 'POP_EXCEPT': @@ -266,6 +260,8 @@ def pop_block_of_type(type): if block_stack: _warn_bug("block stack not empty") + if block_exits: + _warn_bug("block exits not empty") return labels, gotos diff --git a/test_goto.py b/test_goto.py index 8bbf7d3..c738748 100644 --- a/test_goto.py +++ b/test_goto.py @@ -203,6 +203,59 @@ def func(): pytest.raises(SyntaxError, with_goto, func) +def test_jump_out_of_while_true_loop(): + @with_goto + def func(): + i = 0 + while True: + i += 1 + goto .out + label .out + return i + + assert func() == 1 + +def test_jump_out_of_while_true_loop_and_survive(): + @with_goto + def func(): + j = 0 + for i in range(10): + while True: + j += 1 + goto .out + label .out + return i, j + + assert func() == (9, 10) + +def test_jump_out_of_while_true_loop_and_live(): + @with_goto + def func(): + k = 0 + for i in range(10): + for j in range(10): + while True: + k += 1 + goto .out + label .out + return i, j, k + + assert func() == (9, 0, 10) + +def test_jump_out_of_while_loop_and_live(): + @with_goto + def func(): + k = 0 + for i in range(10): + for j in range(4): + while k < 5: + k += 1 + goto .out + label .out + return i, j, k + + assert func() == (9, 3, 5) + class Context: def __init__(self): self.enters = 0 From 65eed29399c55afb96b6d4c11bf7738594179a14 Mon Sep 17 00:00:00 2001 From: condut Date: Sat, 14 Dec 2019 05:10:21 +0200 Subject: [PATCH 37/48] Comment new tests back in - they already pass --- test_goto.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test_goto.py b/test_goto.py index 8795c47..3bb42a1 100644 --- a/test_goto.py +++ b/test_goto.py @@ -357,7 +357,7 @@ def func(): assert func() == (9, 0) -"""def test_jump_out_of_while_true_loop(): +def test_jump_out_of_while_true_loop(): @with_goto def func(): i = 0 @@ -408,7 +408,7 @@ def func(): label .out return i, j, k - assert func() == (9, 3, 5)""" + assert func() == (9, 3, 5) class Context: def __init__(self): @@ -911,7 +911,7 @@ def func(): assert func() == 0 -"""def test_jump_with_while_true_break(): # to see it doesn't confuse parser +def test_jump_with_while_true_break(): # to see it doesn't confuse parser @with_goto def func(): i = 0 @@ -922,7 +922,7 @@ def func(): label .x return i - assert func() == 1""" + assert func() == 1 def test_function_is_copy(): From 356199755e28e6e38e6f6fba51744cc6852fce85 Mon Sep 17 00:00:00 2001 From: condut Date: Sat, 14 Dec 2019 08:24:38 +0200 Subject: [PATCH 38/48] Fix jump into while loop (& with), simplify code Note that with this commit, we no longer look at POP_BLOCK at all, using the 'target offset' of the blocks instead (that always seemed a bit more reliable, and with the block_exits/block_stack simplification, there's no more reason to look at POP_BLOCK) (Turns out jumping into a 'with' loop previously travelled out of the bytecode and into a nearby function, and this wasn't caught by any of the tests in any of the pythons...) --- goto.py | 61 ++++++++++++++++++++++++---------------------------- test_goto.py | 56 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 33 deletions(-) diff --git a/goto.py b/goto.py index 6ab6e56..3bba32b 100644 --- a/goto.py +++ b/goto.py @@ -177,7 +177,6 @@ def _find_labels_and_gotos(code): block_stack = [] block_counter = 0 - block_exits = [] last_block = None opname1 = oparg1 = offset1 = None @@ -196,9 +195,9 @@ def replace_block(old_block, new_block): for goto in gotos: replace_block_in_stack(goto[3], old_block, new_block) - def push_block(opname, oparg=0): + def push_block(opname, target_offset=None): new_counter = block_counter + 1 - block_stack.append((opname, oparg, new_counter)) + block_stack.append((opname, target_offset, new_counter)) return new_counter # to be assigned to block_counter def pop_block(): @@ -229,16 +228,12 @@ def pop_block_of_type(type): dead = False # check for block exits - while block_exits and offset1 == block_exits[-1][-1]: - exit, exitarg, exitcounter, _ = block_exits.pop() + while block_stack and offset1 == block_stack[-1][1]: + exitname, _, _ = last_block = pop_block() - # Necessary for FOR_ITER and sometimes needed for other blocks as well - if block_stack and (exit, exitarg, exitcounter) == block_stack[-1]: - last_block = pop_block() - - if exit == 'SETUP_EXCEPT' and _BYTECODE.has_pop_except: + if exitname == 'SETUP_EXCEPT' and _BYTECODE.has_pop_except: block_counter = push_block('') - elif exit == 'SETUP_FINALLY': + elif exitname == 'SETUP_FINALLY': block_counter = push_block('') # check for special opcodes @@ -267,15 +262,10 @@ def pop_block_of_type(type): list(block_stack), code.co_names[oparg2])) - elif opname1 in ('SETUP_LOOP', + elif opname1 in ('SETUP_LOOP', 'FOR_ITER', 'SETUP_EXCEPT', 'SETUP_FINALLY', - 'SETUP_WITH', 'SETUP_ASYNC_WITH') or \ - (not _BYTECODE.has_loop_blocks and opname1 == 'FOR_ITER'): - block_counter = push_block(opname1, oparg1) - block_exits.append((opname1, oparg1, block_counter, endoffset1 + oparg1)) - - elif opname1 == 'POP_BLOCK': - last_block = pop_block() + 'SETUP_WITH', 'SETUP_ASYNC_WITH'): + block_counter = push_block(opname1, endoffset1 + oparg1) elif opname1 == 'POP_EXCEPT': last_block = pop_block_of_type('') @@ -290,8 +280,7 @@ def pop_block_of_type(type): block_counter = push_block('') else: # python 2.6 - finally was actually with - replace_block(last_block, - ('SETUP_WITH',) + last_block[1:]) + replace_block(last_block, ('SETUP_WITH',) + last_block[1:]) if opname1 in ('JUMP_ABSOLUTE', 'JUMP_FORWARD'): dead = True @@ -302,8 +291,6 @@ def pop_block_of_type(type): if block_stack: _warn_bug("block stack not empty") - if block_exits: - _warn_bug("block exits not empty") return labels, gotos @@ -385,6 +372,7 @@ def _patch_code(code): ops = [] + # prepare common_depth = min(len(origin_stack), len(target_stack)) for i in _range(common_depth): if origin_stack[i] != target_stack[i]: @@ -399,9 +387,11 @@ def _patch_code(code): ops.append(('STORE_FAST', temp_var)) many_params = (params != 'param') + # pop blocks for block, _, _ in reversed(origin_stack[common_depth:]): if block == 'FOR_ITER': - ops.append('POP_TOP') + if not _BYTECODE.has_loop_blocks: + ops.append('POP_TOP') elif block == '': ops.append('POP_EXCEPT') elif block == '': @@ -423,23 +413,28 @@ def _patch_code(code): ops.append(('LOAD_CONST', data.get_const(None))) ops.append('END_FINALLY') + # push blocks + def setup_block_absolute(block, block_end): + # there's no SETUP_*_ABSOLUTE, so we setup forward to an JUMP_ABSOLUTE + jump_abs_op = ('JUMP_ABSOLUTE', block_end) + skip_jump_op = ('JUMP_FORWARD', _get_instruction_size(*jump_abs_op)) + setup_block_op = (block, _get_instruction_size(*skip_jump_op)) + ops.extend((setup_block_op, skip_jump_op, jump_abs_op)) + tuple_i = 0 - for block, blockarg, _ in target_stack[common_depth:]: - if block in ('SETUP_LOOP', 'FOR_ITER', - 'SETUP_WITH', 'SETUP_ASYNC_WITH'): + for block, block_target, _ in target_stack[common_depth:]: + if block in ('FOR_ITER', 'SETUP_WITH', 'SETUP_ASYNC_WITH'): if not params: raise SyntaxError( 'Jump into block without the necessary params') - if block == 'SETUP_LOOP': - ops.append((block, blockarg)) ops.append(('LOAD_FAST', temp_var)) if many_params: ops.append(('LOAD_CONST', data.get_const(tuple_i))) ops.append('BINARY_SUBSCR') tuple_i += 1 - if block in ('SETUP_LOOP', 'FOR_ITER'): + if block == 'FOR_ITER': # this both converts iterables to iterators for # convenience, and prevents FOR_ITER from crashing # on non-iter objects. (this is a no-op for iterators) @@ -450,10 +445,10 @@ def _patch_code(code): # inappropriate # (a goto must bypass any and all side-effects) ops.append(('LOAD_ATTR', data.get_name('__exit__'))) - ops.append(('SETUP_FINALLY', blockarg)) + setup_block_absolute('SETUP_FINALLY', block_target) - elif block in ('SETUP_EXCEPT', 'SETUP_FINALLY'): - ops.append((block, blockarg)) + elif block in ('SETUP_LOOP', 'SETUP_EXCEPT', 'SETUP_FINALLY'): + setup_block_absolute(block, block_target) elif block == '': if _BYTECODE.pypy_finally_semantics: diff --git a/test_goto.py b/test_goto.py index 3bb42a1..ae04a83 100644 --- a/test_goto.py +++ b/test_goto.py @@ -410,6 +410,62 @@ def func(): assert func() == (9, 3, 5) +def test_jump_into_while_true_loop(): + @with_goto + def func(): + x = 1 + goto .inside + x += 1 + while True: + x += 1 + label .inside + if x == 1: + break + return x + + assert func() == 1 + +def test_jump_into_while_true_loop_and_survive(): + @with_goto + def func(): + x = 0 + for i in range(10): + goto .inside + while True: + x += 1 + label .inside + break + return i, x + + assert func() == (9, 0) + +def test_jump_into_while_loop(): + @with_goto + def func(): + c, x = 0, 0 + goto .inside + while x < 10: + x += 1 + label .inside + c += 1 + return c, x + + assert func() == (11, 10) + +def test_jump_into_while_loop_and_survive(): + @with_goto + def func(): + c, x = 0, 0 + for i in range(5): + goto .inside + while x < 5: + x += 1 + label .inside + c += 1 + return i, c, x + + assert func() == (4, 10, 5) + class Context: def __init__(self): self.enters = 0 From a980b6c7f0b6f28dbc75d6b48d65ee96dddb72d5 Mon Sep 17 00:00:00 2001 From: condut Date: Sat, 14 Dec 2019 08:28:37 +0200 Subject: [PATCH 39/48] elif for consistentcy --- goto.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/goto.py b/goto.py index 3bba32b..cba7c2a 100644 --- a/goto.py +++ b/goto.py @@ -282,7 +282,7 @@ def pop_block_of_type(type): # python 2.6 - finally was actually with replace_block(last_block, ('SETUP_WITH',) + last_block[1:]) - if opname1 in ('JUMP_ABSOLUTE', 'JUMP_FORWARD'): + elif opname1 in ('JUMP_ABSOLUTE', 'JUMP_FORWARD'): dead = True opname1, oparg1, offset1 = opname2, oparg2, offset2 From cf0bccfca6eea833e60533aee853901a4052b899 Mon Sep 17 00:00:00 2001 From: condut Date: Sun, 15 Dec 2019 21:58:46 +0200 Subject: [PATCH 40/48] Style fixes --- goto.py | 101 +++++++++++++++++++++++---------------------------- test_goto.py | 51 ++++++++++++++++++++------ 2 files changed, 84 insertions(+), 68 deletions(-) diff --git a/goto.py b/goto.py index 08016ee..ac192d9 100644 --- a/goto.py +++ b/goto.py @@ -6,11 +6,6 @@ import weakref import warnings -try: - import __pypy__ -except: - __pypy__ = None - try: _array_to_bytes = array.array.tobytes except AttributeError: @@ -45,6 +40,12 @@ def __init__(self): self.has_setup_with = 'SETUP_WITH' in dis.opmap self.has_setup_except = 'SETUP_EXCEPT' in dis.opmap + try: + import __pypy__ # noqa: F401 + self.pypy_finally_semantics = True + except ImportError: + self.pypy_finally_semantics = False + @property def argument_bits(self): return self.argument.size * 8 @@ -52,16 +53,19 @@ def argument_bits(self): _BYTECODE = _Bytecode() -_patched_code_cache = weakref.WeakKeyDictionary() # use a weak dictionary in case code objects can be garbage-collected + +# use a weak dictionary in case code objects can be garbage-collected +_patched_code_cache = weakref.WeakKeyDictionary() try: _patched_code_cache[_Bytecode.__init__.__code__] = None except TypeError: - _patched_code_cache = {} # ...unless not supported + _patched_code_cache = {} # ...unless not supported + def _make_code(code, codestring): try: - return code.replace(co_code=codestring) # new in 3.8+ - except: + return code.replace(co_code=codestring) # new in 3.8+ + except AttributeError: args = [ code.co_argcount, code.co_nlocals, code.co_stacksize, code.co_flags, codestring, code.co_consts, @@ -108,28 +112,6 @@ def _parse_instructions(code, yield_nones_at_end=0): for _ in range(yield_nones_at_end): yield (None, None, None) -def _get_instruction_size(opname, oparg=0): - size = 1 - - extended_arg = oparg >> _BYTECODE.argument_bits - if extended_arg != 0: - size += _get_instruction_size('EXTENDED_ARG', extended_arg) - oparg &= (1 << _BYTECODE.argument_bits) - 1 - - opcode = dis.opmap[opname] - if opcode >= _BYTECODE.have_argument: - size += _BYTECODE.argument.size - - return size - -def _get_instructions_size(ops): - size = 0 - for op in ops: - if isinstance(op, str): - size += _get_instruction_size(op) - else: - size += _get_instruction_size(*op) - return size def _get_instruction_size(opname, oparg=0): size = 1 @@ -172,6 +154,7 @@ def _write_instruction(buf, pos, opname, oparg=0): return pos + def _write_instructions(buf, pos, ops): for op in ops: if isinstance(op, str): @@ -180,16 +163,10 @@ def _write_instructions(buf, pos, ops): pos = _write_instruction(buf, pos, *op) return pos -def _warn_bug(msg): - warnings.warn("Internal error detected - result of with_goto may be incorrect. (%s)" % msg) -def _write_instructions(buf, pos, ops): - for op in ops: - if isinstance(op, str): - pos = _write_instruction(buf, pos, op) - else: - pos = _write_instruction(buf, pos, *op) - return pos +def _warn_bug(msg): + warnings.warn("Internal error detected" + + " - result of with_goto may be incorrect. (%s)" % msg) def _find_labels_and_gotos(code): @@ -201,8 +178,8 @@ def _find_labels_and_gotos(code): block_exits = [] last_block = None - opname0 = oparg0 = offset0 = None - opname1 = oparg1 = offset1 = None # the main one we're looking at each loop iteration + opname0 = None + opname1 = oparg1 = offset1 = None # the one looked at each iteration opname2 = oparg2 = offset2 = None opname3 = oparg3 = offset3 = None @@ -226,8 +203,10 @@ def pop_block(): def pop_block_of_type(type): if block_stack and block_stack[-1][0] != type: - # in 3.8, only finally blocks are supported, so we must determine the except/finally nature ourselves, and replace the block afterwards - if not _BYTECODE.has_setup_except and type == "" and block_stack[-1][0] == '': + # in 3.8, only finally blocks are supported, so we must determine + # whether it's except/finally ourselves + if not _BYTECODE.has_setup_except and \ + type == "" and block_stack[-1][0] == '': replace_block(block_stack[-1], (type, block_stack[-1][1])) else: _warn_bug("mismatched block type") @@ -239,11 +218,12 @@ def pop_block_of_type(type): # check for block exits while block_exits and offset1 == block_exits[-1][-1]: block, counter, _ = block_exits.pop() - - # Necessary for FOR_ITER and sometimes needed for other blocks as well + + # Necessary for FOR_ITER and sometimes needed for + # other blocks as well if block_stack and (block, counter) == block_stack[-1][:2]: last_block = pop_block() - + if block == 'SETUP_EXCEPT' and _BYTECODE.has_pop_except: block_counter += 1 block_stack.append(('', block_counter)) @@ -268,9 +248,9 @@ def pop_block_of_type(type): offset4, oparg2, list(block_stack))) - elif opname1 in ('SETUP_LOOP', - 'SETUP_EXCEPT', 'SETUP_FINALLY', - 'SETUP_WITH', 'SETUP_ASYNC_WITH') or \ + elif (opname1 in ('SETUP_LOOP', + 'SETUP_EXCEPT', 'SETUP_FINALLY', + 'SETUP_WITH', 'SETUP_ASYNC_WITH')) or \ (not _BYTECODE.has_loop_blocks and opname1 == 'FOR_ITER'): block_counter += 1 block_stack.append((opname1, block_counter)) @@ -280,15 +260,19 @@ def pop_block_of_type(type): elif opname1 == 'POP_EXCEPT': last_block = pop_block_of_type('') elif opname1 == 'END_FINALLY': - if opname0 != 'JUMP_FORWARD': # hack for dummy end-finally in except block (correct fix would be a jump-aware reading of instructions!) + # hack for dummy end-finally in except block + # (correct fix would be a jump-aware reading of instructions!) + if opname0 != 'JUMP_FORWARD': last_block = pop_block_of_type('') elif opname1 in ('WITH_CLEANUP', 'WITH_CLEANUP_START'): if _BYTECODE.has_setup_with: - block_stack.append(('', -1)) # temporary block to match END_FINALLY + # temporary block to match END_FINALLY + block_stack.append(('', -1)) else: - replace_block(last_block, ('SETUP_WITH',) + last_block[1:]) # python 2.6 - finally was actually with + # python 2.6 - finally was actually with + replace_block(last_block, ('SETUP_WITH',) + last_block[1:]) - opname0, oparg0, offset0 = opname1, oparg1, offset1 + opname0 = opname1 opname1, oparg1, offset1 = opname2, oparg2, offset2 opname2, oparg2, offset2 = opname3, oparg3, offset3 opname3, oparg3, offset3 = opname4, oparg4, offset4 @@ -341,8 +325,13 @@ def _patch_code(code): ops.append('POP_BLOCK') if block in ('SETUP_WITH', 'SETUP_ASYNC_WITH'): ops.append('POP_TOP') - # pypy 3.6 keeps a block around until END_FINALLY; python 3.8 reuses SETUP_FINALLY for SETUP_EXCEPT (where END_FINALLY is not accepted). What will pypy 3.8 do? - if __pypy__ and block in ('SETUP_FINALLY', 'SETUP_WITH', 'SETUP_ASYNC_WITH'): + # pypy 3.6 keeps a block around until END_FINALLY + # python 3.8 reuses SETUP_FINALLY for SETUP_EXCEPT + # (where END_FINALLY is not accepted). + # What will pypy 3.8 do? + if _BYTECODE.pypy_finally_semantics and \ + block in ('SETUP_FINALLY', 'SETUP_WITH', + 'SETUP_ASYNC_WITH'): ops.append(('LOAD_CONST', code.co_consts.index(None))) ops.append('END_FINALLY') ops.append(('JUMP_ABSOLUTE', target // _BYTECODE.jump_unit)) diff --git a/test_goto.py b/test_goto.py index c8a3412..b160068 100644 --- a/test_goto.py +++ b/test_goto.py @@ -113,6 +113,7 @@ def func(): assert func() == (0, 0) + def test_jump_out_of_nested_4_loops_and_survive(): @with_goto def func(): @@ -127,6 +128,7 @@ def func(): assert func() == (1, 0, 0, 0, 0) + def test_large_jumps_in_diff_orders(): @with_goto def func(): @@ -160,6 +162,7 @@ def func(): assert func() == (0,) * 15 + def test_jump_out_of_nested_11_loops(): @with_goto def func(): @@ -189,6 +192,7 @@ def func(): assert func() == (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) + def test_jump_across_loops(): def func(): for i in range(10): @@ -199,6 +203,7 @@ def func(): pytest.raises(SyntaxError, with_goto, func) + def test_jump_out_of_while_true_loop(): @with_goto def func(): @@ -208,9 +213,10 @@ def func(): goto .out label .out return i - + assert func() == 1 + def test_jump_out_of_while_true_loop_and_survive(): @with_goto def func(): @@ -221,9 +227,10 @@ def func(): goto .out label .out return i, j - + assert func() == (9, 10) + def test_jump_out_of_while_true_loop_and_live(): @with_goto def func(): @@ -235,9 +242,10 @@ def func(): goto .out label .out return i, j, k - + assert func() == (9, 0, 10) + def test_jump_out_of_while_loop_and_live(): @with_goto def func(): @@ -249,22 +257,27 @@ def func(): goto .out label .out return i, j, k - + assert func() == (9, 3, 5) + class Context: def __init__(self): self.enters = 0 self.exits = 0 + def __enter__(self): self.enters += 1 return self + def __exit__(self, a, b, c): self.exits += 1 return self + def data(self): return (self.enters, self.exits) + def test_jump_out_of_with_block(): @with_goto def func(): @@ -273,7 +286,8 @@ def func(): label .out return c.data() - assert func()== (1, 0) + assert func() == (1, 0) + def test_jump_out_of_with_block_and_survive(): @with_goto @@ -287,6 +301,7 @@ def func(): assert func() == (2, (3, 0)) + def test_jump_out_of_with_block_and_live(): @with_goto def func(): @@ -300,14 +315,17 @@ def func(): assert func() == (2, 0, (3, 0)) + def test_jump_into_with_block(): def func(): with Context() as c: label .block goto .block + return c pytest.raises(SyntaxError, with_goto, func) + def test_generator(): @with_goto def func(): @@ -322,6 +340,7 @@ def func(): assert tuple(func()) == (0, 1, 4, 5) + def test_jump_out_of_try_block(): @with_goto def func(): @@ -337,6 +356,7 @@ def func(): assert func() is None + def test_jump_out_of_try_block_and_survive(): @with_goto def func(): @@ -344,7 +364,7 @@ def func(): try: rv = None goto .end - except: + except Exception: rv = 'except' finally: rv = 'finally' @@ -353,6 +373,7 @@ def func(): assert func() == (9, None) + def test_jump_out_of_try_block_and_live(): @with_goto def func(): @@ -361,7 +382,7 @@ def func(): try: rv = None goto .end - except: + except Exception: rv = 'except' finally: rv = 'finally' @@ -370,6 +391,7 @@ def func(): assert func() == (2, 2, None) + def test_jump_into_try_block(): def func(): try: @@ -386,7 +408,7 @@ def test_jump_out_of_except_block(): def func(): try: rv = 1 / 0 - except: + except Exception: rv = 'except' goto .end finally: @@ -396,6 +418,7 @@ def func(): assert func() == 'except' + def test_jump_out_of_except_block_and_live(): @with_goto def func(): @@ -403,7 +426,7 @@ def func(): for j in range(3): try: rv = 1 / 0 - except: + except Exception: rv = 'except' goto .end finally: @@ -413,6 +436,7 @@ def func(): assert func() == (2, 0, 'except') + """def test_jump_into_except_block(): def func(): try: @@ -424,6 +448,7 @@ def func(): pytest.raises(SyntaxError, with_goto, func)""" + def test_jump_out_of_finally_block(): @with_goto def func(): @@ -438,6 +463,7 @@ def func(): assert func() == 'finally' + def test_jump_out_of_finally_block_and_live(): @with_goto def func(): @@ -454,6 +480,7 @@ def func(): assert func() == (2, 0, 'finally') + def test_jump_out_of_try_in_except_in_finally_and_live(): @with_goto def func(): @@ -465,12 +492,12 @@ def func(): rv = 'finally' try: rv = 1 / 0 - except: + except Exception: rv = 'except' try: rv = 'try' goto .end - except: + except Exception: rv = 'except2' finally: rv = 'finally2' @@ -507,6 +534,7 @@ def func(): assert newfunc is not func assert newfunc.foo == 'bar' + def test_code_is_not_copy(): def outer_func(): @with_goto @@ -517,4 +545,3 @@ def inner_func(): assert outer_func() is not outer_func() assert outer_func().__code__ is outer_func().__code__ - From e3c23ce9f35cc9e0902b72bcf8087e0354000978 Mon Sep 17 00:00:00 2001 From: condut Date: Sun, 15 Dec 2019 22:29:46 +0200 Subject: [PATCH 41/48] Added tests that show remaining issues, first part of fixing them. --- goto.py | 16 +++--- test_goto.py | 157 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 165 insertions(+), 8 deletions(-) diff --git a/goto.py b/goto.py index ac192d9..b2120e5 100644 --- a/goto.py +++ b/goto.py @@ -178,7 +178,6 @@ def _find_labels_and_gotos(code): block_exits = [] last_block = None - opname0 = None opname1 = oparg1 = offset1 = None # the one looked at each iteration opname2 = oparg2 = offset2 = None opname3 = oparg3 = offset3 = None @@ -203,13 +202,18 @@ def pop_block(): def pop_block_of_type(type): if block_stack and block_stack[-1][0] != type: - # in 3.8, only finally blocks are supported, so we must determine - # whether it's except/finally ourselves if not _BYTECODE.has_setup_except and \ type == "" and block_stack[-1][0] == '': + # in 3.8, only finally blocks are supported, so we must + # determine whether it's except/finally ourselves replace_block(block_stack[-1], (type, block_stack[-1][1])) + elif type == "": + # Python puts END_FINALLY at the very end of except + # clauses, so we must ignore it. + return else: _warn_bug("mismatched block type") + return # better not to pop (a tiny bit) return pop_block() for opname4, oparg4, offset4 in _parse_instructions(code.co_code, 3): @@ -260,10 +264,7 @@ def pop_block_of_type(type): elif opname1 == 'POP_EXCEPT': last_block = pop_block_of_type('') elif opname1 == 'END_FINALLY': - # hack for dummy end-finally in except block - # (correct fix would be a jump-aware reading of instructions!) - if opname0 != 'JUMP_FORWARD': - last_block = pop_block_of_type('') + last_block = pop_block_of_type('') elif opname1 in ('WITH_CLEANUP', 'WITH_CLEANUP_START'): if _BYTECODE.has_setup_with: # temporary block to match END_FINALLY @@ -272,7 +273,6 @@ def pop_block_of_type(type): # python 2.6 - finally was actually with replace_block(last_block, ('SETUP_WITH',) + last_block[1:]) - opname0 = opname1 opname1, oparg1, offset1 = opname2, oparg2, offset2 opname2, oparg2, offset2 = opname3, oparg3, offset3 opname3, oparg3, offset3 = opname4, oparg4, offset4 diff --git a/test_goto.py b/test_goto.py index b160068..4251100 100644 --- a/test_goto.py +++ b/test_goto.py @@ -326,6 +326,19 @@ def func(): pytest.raises(SyntaxError, with_goto, func) +def test_generator(): + @with_goto + def func(): + yield 0 + yield 1 + goto .x + yield 2 + yield 3 + label .x + yield 4 + yield 5 + + assert tuple(func()) == (0, 1, 4, 5) def test_generator(): @with_goto def func(): @@ -341,6 +354,66 @@ def func(): assert tuple(func()) == (0, 1, 4, 5) +def test_jump_out_of_try_except_block(): + @with_goto + def func(): + try: + rv = None + goto .end + except Exception: + rv = 'except' + label .end + return rv + + assert func() == None + + +def test_jump_out_of_try_except_block_and_live(): + @with_goto + def func(): + for i in range(3): + for j in range(3): + try: + rv = None + goto .end + except Exception: + rv = 'except' + label .end + return (i, j, rv) + + assert func() == (2, 2, None) + + +def test_jump_out_of_try_finally_block(): + @with_goto + def func(): + try: + rv = None + goto .end + finally: + rv = 'finally' + label .end + return rv + + assert func() == None + + +def test_jump_out_of_try_finally_block_and_live(): + @with_goto + def func(): + for i in range(3): + for j in range(3): + try: + rv = None + goto .end + finally: + rv = 'finally' + label .end + return (i, j, rv) + + assert func() == (2, 2, None) + + def test_jump_out_of_try_block(): @with_goto def func(): @@ -403,6 +476,36 @@ def func(): pytest.raises(SyntaxError, with_goto, func) +def test_jump_out_of_except_block_wo_finally(): + @with_goto + def func(): + try: + rv = 1 / 0 + except Exception: + rv = 'except' + goto .end + label .end + return rv + + assert func() == 'except' + + +def test_jump_out_of_except_block_wo_finally_and_live(): + @with_goto + def func(): + for i in range(3): + for j in range(3): + try: + rv = 1 / 0 + except Exception: + rv = 'except' + goto .end + label .end + return (i, j, rv) + + assert func() == (2, 0, 'except') + + def test_jump_out_of_except_block(): @with_goto def func(): @@ -437,6 +540,24 @@ def func(): assert func() == (2, 0, 'except') +def test_jump_out_of_bare_except_block_and_live(): + @with_goto + def func(): + for i in range(3): + for j in range(3): + try: + rv = 1 / 0 + except: + rv = 'except' + goto .end + finally: + rv = 'finally' + label .end + return (i, j, rv) + + assert func() == (2, 0, 'except') + + """def test_jump_into_except_block(): def func(): try: @@ -449,6 +570,42 @@ def func(): pytest.raises(SyntaxError, with_goto, func)""" +def test_jump_out_of_finally_block_w_except(): + @with_goto + def func(): + try: + rv = None + except Exception: + rv = 'except' + finally: + rv = 'finally' + goto .end + rv = 'end' + label .end + return rv + + assert func() == 'finally' + + +def test_jump_out_of_finally_block_w_except_and_live(): + @with_goto + def func(): + for i in range(3): + for j in range(3): + try: + rv = None + except Exception: + rv = 'except' + finally: + rv = 'finally' + goto .end + rv = 'end' + label .end + return i, j, rv + + assert func() == (2, 0, 'finally') + + def test_jump_out_of_finally_block(): @with_goto def func(): From 08359582bf5715c4b25335dd51e165e4831ca13d Mon Sep 17 00:00:00 2001 From: condut Date: Sun, 15 Dec 2019 22:52:23 +0200 Subject: [PATCH 42/48] Backport change from 'iter' needed for the newly added tests (Change makes it so POP_BLOCK is no longer looked at (as it's a poor indicator in some cases), instead - we look at the target of each block. Also includes a simplification removing block_exits) --- goto.py | 58 ++++++++++++++++++++++------------------------------ test_goto.py | 19 +++-------------- 2 files changed, 28 insertions(+), 49 deletions(-) diff --git a/goto.py b/goto.py index b2120e5..27da450 100644 --- a/goto.py +++ b/goto.py @@ -175,10 +175,9 @@ def _find_labels_and_gotos(code): block_stack = [] block_counter = 0 - block_exits = [] last_block = None - opname1 = oparg1 = offset1 = None # the one looked at each iteration + opname1 = oparg1 = offset1 = None opname2 = oparg2 = offset2 = None opname3 = oparg3 = offset3 = None @@ -190,9 +189,16 @@ def replace_block_in_stack(stack, old_block, new_block): def replace_block(old_block, new_block): replace_block_in_stack(block_stack, old_block, new_block) for label in labels: - replace_block_in_stack(labels[label][2], old_block, new_block) + _, _, label_blocks = labels[label] + replace_block_in_stack(label_blocks, old_block, new_block) for goto in gotos: - replace_block_in_stack(goto[3], old_block, new_block) + _, _, _, goto_blocks = goto + replace_block_in_stack(goto_blocks, old_block, new_block) + + def push_block(opname, target_offset=None): + new_counter = block_counter + 1 + block_stack.append((opname, target_offset, new_counter)) + return new_counter # to be assigned to block_counter def pop_block(): if block_stack: @@ -206,34 +212,23 @@ def pop_block_of_type(type): type == "" and block_stack[-1][0] == '': # in 3.8, only finally blocks are supported, so we must # determine whether it's except/finally ourselves - replace_block(block_stack[-1], (type, block_stack[-1][1])) - elif type == "": - # Python puts END_FINALLY at the very end of except - # clauses, so we must ignore it. - return + replace_block(block_stack[-1], (type,) + block_stack[-1][1:]) else: _warn_bug("mismatched block type") - return # better not to pop (a tiny bit) + return last_block # better not to pop (a tiny bit) return pop_block() for opname4, oparg4, offset4 in _parse_instructions(code.co_code, 3): endoffset1 = offset2 # check for block exits - while block_exits and offset1 == block_exits[-1][-1]: - block, counter, _ = block_exits.pop() - - # Necessary for FOR_ITER and sometimes needed for - # other blocks as well - if block_stack and (block, counter) == block_stack[-1][:2]: - last_block = pop_block() + while block_stack and offset1 == block_stack[-1][1]: + exitname, _, _ = last_block = pop_block() - if block == 'SETUP_EXCEPT' and _BYTECODE.has_pop_except: - block_counter += 1 - block_stack.append(('', block_counter)) - elif block == 'SETUP_FINALLY': - block_counter += 1 - block_stack.append(('', block_counter)) + if exitname == 'SETUP_EXCEPT' and _BYTECODE.has_pop_except: + block_counter = push_block('') + elif exitname == 'SETUP_FINALLY': + block_counter = push_block('') # check for special opcodes if opname1 in ('LOAD_GLOBAL', 'LOAD_NAME'): @@ -256,19 +251,18 @@ def pop_block_of_type(type): 'SETUP_EXCEPT', 'SETUP_FINALLY', 'SETUP_WITH', 'SETUP_ASYNC_WITH')) or \ (not _BYTECODE.has_loop_blocks and opname1 == 'FOR_ITER'): - block_counter += 1 - block_stack.append((opname1, block_counter)) - block_exits.append((opname1, block_counter, endoffset1 + oparg1)) - elif opname1 == 'POP_BLOCK': - last_block = pop_block() + block_counter = push_block(opname1, endoffset1 + oparg1) elif opname1 == 'POP_EXCEPT': last_block = pop_block_of_type('') elif opname1 == 'END_FINALLY': - last_block = pop_block_of_type('') + # Python puts END_FINALLY at the very end of except + # clauses, so we must ignore it in the wrong place. + if block_stack and block_stack[-1][0] == '': + last_block = pop_block_of_type('') elif opname1 in ('WITH_CLEANUP', 'WITH_CLEANUP_START'): if _BYTECODE.has_setup_with: # temporary block to match END_FINALLY - block_stack.append(('', -1)) + block_counter = push_block('') else: # python 2.6 - finally was actually with replace_block(last_block, ('SETUP_WITH',) + last_block[1:]) @@ -279,8 +273,6 @@ def pop_block_of_type(type): if block_stack: _warn_bug("block stack not empty") - if block_exits: - _warn_bug("block exits not empty") return labels, gotos @@ -314,7 +306,7 @@ def _patch_code(code): raise SyntaxError('Jump into different block') ops = [] - for block, _ in reversed(origin_stack[target_depth:]): + for block, _, _ in reversed(origin_stack[target_depth:]): if block == 'FOR_ITER': ops.append('POP_TOP') elif block == '': diff --git a/test_goto.py b/test_goto.py index 4251100..c07af24 100644 --- a/test_goto.py +++ b/test_goto.py @@ -326,19 +326,6 @@ def func(): pytest.raises(SyntaxError, with_goto, func) -def test_generator(): - @with_goto - def func(): - yield 0 - yield 1 - goto .x - yield 2 - yield 3 - label .x - yield 4 - yield 5 - - assert tuple(func()) == (0, 1, 4, 5) def test_generator(): @with_goto def func(): @@ -365,7 +352,7 @@ def func(): label .end return rv - assert func() == None + assert func() is None def test_jump_out_of_try_except_block_and_live(): @@ -395,7 +382,7 @@ def func(): label .end return rv - assert func() == None + assert func() is None def test_jump_out_of_try_finally_block_and_live(): @@ -547,7 +534,7 @@ def func(): for j in range(3): try: rv = 1 / 0 - except: + except: # noqa: E722 rv = 'except' goto .end finally: From 7d4829249c02f46f99b838a2626a9184e42cc22a Mon Sep 17 00:00:00 2001 From: condut Date: Sun, 15 Dec 2019 23:35:53 +0200 Subject: [PATCH 43/48] Refactor _find_labels_and_gotos to use a class instead of nested funcs --- goto.py | 120 +++++++++++++++++++++++++++++++++----------------------- 1 file changed, 70 insertions(+), 50 deletions(-) diff --git a/goto.py b/goto.py index 27da450..a46a0d6 100644 --- a/goto.py +++ b/goto.py @@ -169,66 +169,78 @@ def _warn_bug(msg): " - result of with_goto may be incorrect. (%s)" % msg) -def _find_labels_and_gotos(code): - labels = {} - gotos = [] - - block_stack = [] - block_counter = 0 - last_block = None - - opname1 = oparg1 = offset1 = None - opname2 = oparg2 = offset2 = None - opname3 = oparg3 = offset3 = None - - def replace_block_in_stack(stack, old_block, new_block): +class _BlockStack(object): + def __init__(self, labels, gotos): + self.stack = [] + self.block_counter = 0 + self.last_block = None + self.labels = labels + self.gotos = gotos + + def _replace_in_stack(self, stack, old_block, new_block): for i, block in enumerate(stack): if block == old_block: stack[i] = new_block - def replace_block(old_block, new_block): - replace_block_in_stack(block_stack, old_block, new_block) - for label in labels: - _, _, label_blocks = labels[label] - replace_block_in_stack(label_blocks, old_block, new_block) - for goto in gotos: + def replace(self, old_block, new_block): + self._replace_in_stack(self.stack, old_block, new_block) + + for label in self.labels: + _, _, label_blocks = self.labels[label] + self._replace_in_stack(label_blocks, old_block, new_block) + + for goto in self.gotos: _, _, _, goto_blocks = goto - replace_block_in_stack(goto_blocks, old_block, new_block) + self._replace_in_stack(goto_blocks, old_block, new_block) - def push_block(opname, target_offset=None): - new_counter = block_counter + 1 - block_stack.append((opname, target_offset, new_counter)) - return new_counter # to be assigned to block_counter + def push(self, opname, target_offset=None): + self.block_counter += 1 + self.stack.append((opname, target_offset, self.block_counter)) - def pop_block(): - if block_stack: - return block_stack.pop() + def pop(self): + if self.stack: + self.last_block = self.stack.pop() + return self.last_block else: _warn_bug("can't pop block") - def pop_block_of_type(type): - if block_stack and block_stack[-1][0] != type: - if not _BYTECODE.has_setup_except and \ - type == "" and block_stack[-1][0] == '': - # in 3.8, only finally blocks are supported, so we must - # determine whether it's except/finally ourselves - replace_block(block_stack[-1], (type,) + block_stack[-1][1:]) - else: - _warn_bug("mismatched block type") - return last_block # better not to pop (a tiny bit) - return pop_block() + def pop_of_type(self, type): + if self.stack and self.top()[0] != type: + _warn_bug("mismatched block type") + else: + return self.pop() + + def copy_to_list(self): + return list(self.stack) + + def top(self): + return self.stack[-1] if self.stack else None + + def __len__(self): + return len(self.stack) + + +def _find_labels_and_gotos(code): + labels = {} + gotos = [] + + block_stack = _BlockStack(labels, gotos) + + opname1 = oparg1 = offset1 = None + opname2 = oparg2 = offset2 = None + opname3 = oparg3 = offset3 = None for opname4, oparg4, offset4 in _parse_instructions(code.co_code, 3): endoffset1 = offset2 # check for block exits - while block_stack and offset1 == block_stack[-1][1]: - exitname, _, _ = last_block = pop_block() + while block_stack and offset1 == block_stack.top()[1]: + exitname, _, _ = block_stack.pop() if exitname == 'SETUP_EXCEPT' and _BYTECODE.has_pop_except: - block_counter = push_block('') + block_stack.push('') elif exitname == 'SETUP_FINALLY': - block_counter = push_block('') + block_stack.push('') # check for special opcodes if opname1 in ('LOAD_GLOBAL', 'LOAD_NAME'): @@ -241,31 +253,39 @@ def pop_block_of_type(type): )) labels[oparg2] = (offset1, offset4, - list(block_stack)) + block_stack.copy_to_list()) elif name == 'goto': gotos.append((offset1, offset4, oparg2, - list(block_stack))) + block_stack.copy_to_list())) elif (opname1 in ('SETUP_LOOP', 'SETUP_EXCEPT', 'SETUP_FINALLY', 'SETUP_WITH', 'SETUP_ASYNC_WITH')) or \ (not _BYTECODE.has_loop_blocks and opname1 == 'FOR_ITER'): - block_counter = push_block(opname1, endoffset1 + oparg1) + block_stack.push(opname1, endoffset1 + oparg1) elif opname1 == 'POP_EXCEPT': - last_block = pop_block_of_type('') + top_block = block_stack.top() + if not _BYTECODE.has_setup_except and \ + top_block and top_block[0] == '': + # in 3.8, only finally blocks are supported, so we must + # determine whether it's except/finally ourselves + block_stack.replace(top_block, ('',) + top_block[1:]) + block_stack.pop_of_type('') elif opname1 == 'END_FINALLY': # Python puts END_FINALLY at the very end of except # clauses, so we must ignore it in the wrong place. - if block_stack and block_stack[-1][0] == '': - last_block = pop_block_of_type('') + if block_stack and block_stack.top()[0] == '': + block_stack.pop_of_type('') elif opname1 in ('WITH_CLEANUP', 'WITH_CLEANUP_START'): if _BYTECODE.has_setup_with: # temporary block to match END_FINALLY - block_counter = push_block('') + block_stack.push('') else: # python 2.6 - finally was actually with - replace_block(last_block, ('SETUP_WITH',) + last_block[1:]) + last_block = block_stack.last_block + block_stack.replace(last_block, + ('SETUP_WITH',) + last_block[1:]) opname1, oparg1, offset1 = opname2, oparg2, offset2 opname2, oparg2, offset2 = opname3, oparg3, offset3 From c9fe97f6ee0a1d6165898ab844fc3bd4f2f475c0 Mon Sep 17 00:00:00 2001 From: condut Date: Sun, 15 Dec 2019 23:48:02 +0200 Subject: [PATCH 44/48] Avoid __pypy__ import and instead always call END_FINALLY when possible (Not possible in 3.8 and as comment says - it's not clear how pypy and py38 will reconcile their differences there) --- goto.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/goto.py b/goto.py index a46a0d6..22157ce 100644 --- a/goto.py +++ b/goto.py @@ -40,12 +40,6 @@ def __init__(self): self.has_setup_with = 'SETUP_WITH' in dis.opmap self.has_setup_except = 'SETUP_EXCEPT' in dis.opmap - try: - import __pypy__ # noqa: F401 - self.pypy_finally_semantics = True - except ImportError: - self.pypy_finally_semantics = False - @property def argument_bits(self): return self.argument.size * 8 @@ -337,11 +331,11 @@ def _patch_code(code): ops.append('POP_BLOCK') if block in ('SETUP_WITH', 'SETUP_ASYNC_WITH'): ops.append('POP_TOP') - # pypy 3.6 keeps a block around until END_FINALLY - # python 3.8 reuses SETUP_FINALLY for SETUP_EXCEPT - # (where END_FINALLY is not accepted). - # What will pypy 3.8 do? - if _BYTECODE.pypy_finally_semantics and \ + # END_FINALLY is required for pypy. (To pop special block) + # It is not clear at this point what will happen once + # pypy moves to py38, where there's no distinction between + # except and finally setups. + if _BYTECODE.has_setup_except and \ block in ('SETUP_FINALLY', 'SETUP_WITH', 'SETUP_ASYNC_WITH'): ops.append(('LOAD_CONST', code.co_consts.index(None))) From 27a27ba86fa72d4930167094a01827532bbc3fd9 Mon Sep 17 00:00:00 2001 From: condut Date: Mon, 16 Dec 2019 00:02:43 +0200 Subject: [PATCH 45/48] Go a step further and always call END_FINALLY - make it possible in 3.8 While this isn't strictly required, making it happen requires fully handling the has_setup_except variation (replacing SETUP_FINALLY with SETUP_EXCEPT where appropriate) so it's a good thing to have in the repo for the future --- goto.py | 43 +++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/goto.py b/goto.py index 22157ce..39f147e 100644 --- a/goto.py +++ b/goto.py @@ -39,6 +39,7 @@ def __init__(self): self.has_pop_except = 'POP_EXCEPT' in dis.opmap self.has_setup_with = 'SETUP_WITH' in dis.opmap self.has_setup_except = 'SETUP_EXCEPT' in dis.opmap + self.has_begin_finally = 'BEGIN_FINALLY' in dis.opmap @property def argument_bits(self): @@ -187,9 +188,9 @@ def replace(self, old_block, new_block): _, _, _, goto_blocks = goto self._replace_in_stack(goto_blocks, old_block, new_block) - def push(self, opname, target_offset=None): + def push(self, opname, target_offset=None, previous=None): self.block_counter += 1 - self.stack.append((opname, target_offset, self.block_counter)) + self.stack.append((opname, target_offset, previous, self.block_counter)) def pop(self): if self.stack: @@ -229,12 +230,13 @@ def _find_labels_and_gotos(code): # check for block exits while block_stack and offset1 == block_stack.top()[1]: - exitname, _, _ = block_stack.pop() + exit_block = block_stack.pop() + exit_name = exit_block[0] - if exitname == 'SETUP_EXCEPT' and _BYTECODE.has_pop_except: - block_stack.push('') - elif exitname == 'SETUP_FINALLY': - block_stack.push('') + if exit_name == 'SETUP_EXCEPT' and _BYTECODE.has_pop_except: + block_stack.push('', previous=exit_block) + elif exit_name == 'SETUP_FINALLY': + block_stack.push('', previous=exit_block) # check for special opcodes if opname1 in ('LOAD_GLOBAL', 'LOAD_NAME'): @@ -264,7 +266,11 @@ def _find_labels_and_gotos(code): top_block and top_block[0] == '': # in 3.8, only finally blocks are supported, so we must # determine whether it's except/finally ourselves - block_stack.replace(top_block, ('',) + top_block[1:]) + block_stack.replace(top_block, + ('',) + top_block[1:]) + _, _, setup_block, _ = top_block + block_stack.replace(setup_block, + ('SETUP_EXCEPT',) + setup_block[1:]) block_stack.pop_of_type('') elif opname1 == 'END_FINALLY': # Python puts END_FINALLY at the very end of except @@ -320,7 +326,7 @@ def _patch_code(code): raise SyntaxError('Jump into different block') ops = [] - for block, _, _ in reversed(origin_stack[target_depth:]): + for block, _, _, _ in reversed(origin_stack[target_depth:]): if block == 'FOR_ITER': ops.append('POP_TOP') elif block == '': @@ -331,15 +337,16 @@ def _patch_code(code): ops.append('POP_BLOCK') if block in ('SETUP_WITH', 'SETUP_ASYNC_WITH'): ops.append('POP_TOP') - # END_FINALLY is required for pypy. (To pop special block) - # It is not clear at this point what will happen once - # pypy moves to py38, where there's no distinction between - # except and finally setups. - if _BYTECODE.has_setup_except and \ - block in ('SETUP_FINALLY', 'SETUP_WITH', - 'SETUP_ASYNC_WITH'): - ops.append(('LOAD_CONST', code.co_consts.index(None))) - ops.append('END_FINALLY') + # END_FINALLY is required for pypy. (To pop special block) + # It is not clear at this point what will happen once + # pypy moves to py38, where there's no distinction between + # except and finally setups. + if block in ('SETUP_FINALLY', 'SETUP_WITH', + 'SETUP_ASYNC_WITH'): + ops.append('BEGIN_FINALLY' if \ + _BYTECODE.has_begin_finally else \ + ('LOAD_CONST', code.co_consts.index(None))) + ops.append('END_FINALLY') ops.append(('JUMP_ABSOLUTE', target // _BYTECODE.jump_unit)) if pos + _get_instructions_size(ops) > end: From bf3076f8a63e343a7b07bd43c60d69cca1296d6b Mon Sep 17 00:00:00 2001 From: condut Date: Mon, 16 Dec 2019 00:04:08 +0200 Subject: [PATCH 46/48] flakate --- goto.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/goto.py b/goto.py index 39f147e..700db3a 100644 --- a/goto.py +++ b/goto.py @@ -39,7 +39,7 @@ def __init__(self): self.has_pop_except = 'POP_EXCEPT' in dis.opmap self.has_setup_with = 'SETUP_WITH' in dis.opmap self.has_setup_except = 'SETUP_EXCEPT' in dis.opmap - self.has_begin_finally = 'BEGIN_FINALLY' in dis.opmap + self.has_begin_finally = 'BEGIN_FINALLY' in dis.opmap @property def argument_bits(self): @@ -190,7 +190,8 @@ def replace(self, old_block, new_block): def push(self, opname, target_offset=None, previous=None): self.block_counter += 1 - self.stack.append((opname, target_offset, previous, self.block_counter)) + self.stack.append((opname, target_offset, + previous, self.block_counter)) def pop(self): if self.stack: @@ -343,8 +344,8 @@ def _patch_code(code): # except and finally setups. if block in ('SETUP_FINALLY', 'SETUP_WITH', 'SETUP_ASYNC_WITH'): - ops.append('BEGIN_FINALLY' if \ - _BYTECODE.has_begin_finally else \ + ops.append('BEGIN_FINALLY' if + _BYTECODE.has_begin_finally else ('LOAD_CONST', code.co_consts.index(None))) ops.append('END_FINALLY') ops.append(('JUMP_ABSOLUTE', target // _BYTECODE.jump_unit)) From e9480ffb493dd0c80865a6ca565a08d89c9ed415 Mon Sep 17 00:00:00 2001 From: condut Date: Mon, 16 Dec 2019 00:15:17 +0200 Subject: [PATCH 47/48] Update comment --- goto.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/goto.py b/goto.py index 700db3a..d1ac90c 100644 --- a/goto.py +++ b/goto.py @@ -338,10 +338,7 @@ def _patch_code(code): ops.append('POP_BLOCK') if block in ('SETUP_WITH', 'SETUP_ASYNC_WITH'): ops.append('POP_TOP') - # END_FINALLY is required for pypy. (To pop special block) - # It is not clear at this point what will happen once - # pypy moves to py38, where there's no distinction between - # except and finally setups. + # END_FINALLY is needed only in pypy, but seems logical everywhere if block in ('SETUP_FINALLY', 'SETUP_WITH', 'SETUP_ASYNC_WITH'): ops.append('BEGIN_FINALLY' if From bc40d4ef1409bbe4e024a64710fcd35a373c82d2 Mon Sep 17 00:00:00 2001 From: condut Date: Mon, 16 Dec 2019 00:56:56 +0200 Subject: [PATCH 48/48] Merge... --- goto.py | 63 +++++++++++++++++++++++---------------------- test_goto.py | 72 ++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 86 insertions(+), 49 deletions(-) diff --git a/goto.py b/goto.py index 2aad02d..eb8cbf6 100644 --- a/goto.py +++ b/goto.py @@ -16,6 +16,7 @@ except NameError: _range = range + class _Bytecode: def __init__(self): code = (lambda: x if x else y).__code__.co_code @@ -45,12 +46,6 @@ def __init__(self): self.has_setup_except = 'SETUP_EXCEPT' in dis.opmap self.has_begin_finally = 'BEGIN_FINALLY' in dis.opmap - try: - import __pypy__ - self.pypy_finally_semantics = True - except: - self.pypy_finally_semantics = False - @property def argument_bits(self): return self.argument.size * 8 @@ -200,7 +195,7 @@ def replace(self, old_block, new_block): self._replace_in_stack(label_blocks, old_block, new_block) for goto in self.gotos: - _, _, _, goto_blocks = goto + _, _, _, goto_blocks, _ = goto self._replace_in_stack(goto_blocks, old_block, new_block) def push(self, opname, target_offset=None, previous=None): @@ -254,7 +249,6 @@ def _find_labels_and_gotos(code): elif exit_name == 'SETUP_FINALLY': block_stack.push('', previous=exit_block) - # check for special opcodes if opname1 in ('LOAD_GLOBAL', 'LOAD_NAME'): if opname2 == 'LOAD_ATTR' and opname3 == 'POP_TOP': @@ -282,7 +276,7 @@ def _find_labels_and_gotos(code): block_stack.copy_to_list(), code.co_names[oparg2])) elif opname1 in ('SETUP_LOOP', 'FOR_ITER', - 'SETUP_EXCEPT', 'SETUP_FINALLY', + 'SETUP_EXCEPT', 'SETUP_FINALLY', 'SETUP_WITH', 'SETUP_ASYNC_WITH'): block_stack.push(opname1, endoffset1 + oparg1) elif opname1 == 'POP_EXCEPT': @@ -297,15 +291,15 @@ def _find_labels_and_gotos(code): block_stack.replace(setup_block, ('SETUP_EXCEPT',) + setup_block[1:]) block_stack.pop_of_type('') - elif opname1 == 'END_FINALLY' and not dead: + elif opname1 == 'END_FINALLY': # Python puts END_FINALLY at the very end of except # clauses, so we must ignore it in the wrong place. if block_stack and block_stack.top()[0] == '': - last_block = pop_block_of_type('') + block_stack.pop_of_type('') elif opname1 in ('WITH_CLEANUP', 'WITH_CLEANUP_START'): if _BYTECODE.has_setup_with: # temporary block to match END_FINALLY - block_stack.append(('', -1)) # temporary block to match END_FINALLY + block_stack.push('') else: # python 2.6 - finally was actually with last_block = block_stack.last_block @@ -326,6 +320,7 @@ def _inject_nop_sled(buf, pos, end): while pos < end: pos = _write_instruction(buf, pos, 'NOP') + def _inject_ops(buf, pos, end, ops): size = _get_instructions_size(ops) @@ -348,6 +343,7 @@ def _inject_ops(buf, pos, end, ops): pos = _write_instructions(buf, pos, ops) _inject_nop_sled(buf, pos, end) + class _CodeData: def __init__(self, code): self.nlocals = code.co_nlocals @@ -377,6 +373,7 @@ def add_var(self, name): self.nlocals += 1 return idx + def _patch_code(code): new_code = _patched_code_cache.get(code) if new_code is not None: @@ -429,22 +426,25 @@ def _patch_code(code): ops.append('POP_BLOCK') if block in ('SETUP_WITH', 'SETUP_ASYNC_WITH'): ops.append('POP_TOP') - # END_FINALLY is needed only in pypy, but seems logical everywhere - if block in ('SETUP_FINALLY', + # END_FINALLY is needed only in pypy, + # but seems logical everywhere + if block in ('SETUP_FINALLY', 'SETUP_WITH', 'SETUP_ASYNC_WITH'): - ops.append('BEGIN_FINALLY' if - _BYTECODE.has_begin_finally else - ('LOAD_CONST', code.co_consts.index(None))) - ops.append('END_FINALLY') + ops.append('BEGIN_FINALLY' if + _BYTECODE.has_begin_finally else + ('LOAD_CONST', code.co_consts.index(None))) + ops.append('END_FINALLY') # push blocks def setup_block_absolute(block, block_end): - # there's no SETUP_*_ABSOLUTE, so we setup forward to an JUMP_ABSOLUTE + # there's no SETUP_*_ABSOLUTE, + # so we setup forward to an JUMP_ABSOLUTE jump_abs_op = ('JUMP_ABSOLUTE', block_end) - skip_jump_op = ('JUMP_FORWARD', _get_instruction_size(*jump_abs_op)) + skip_jump_op = ('JUMP_FORWARD', + _get_instruction_size(*jump_abs_op)) setup_block_op = (block, _get_instruction_size(*skip_jump_op)) ops.extend((setup_block_op, skip_jump_op, jump_abs_op)) - + tuple_i = 0 for block, block_target, _, _ in target_stack[common_depth:]: if block in ('FOR_ITER', 'SETUP_WITH', 'SETUP_ASYNC_WITH'): @@ -472,24 +472,27 @@ def setup_block_absolute(block, block_end): setup_block_absolute('SETUP_FINALLY', block_target) elif block in ('SETUP_LOOP', 'SETUP_EXCEPT', 'SETUP_FINALLY'): + if block == 'SETUP_EXCEPT' and not _BYTECODE.has_setup_except: + block = 'SETUP_FINALLY' setup_block_absolute(block, block_target) elif block == '': - if _BYTECODE.pypy_finally_semantics: - ops.append('SETUP_FINALLY') - ops.append('POP_BLOCK') - if _BYTECODE.has_begin_finally: - ops.append('BEGIN_FINALLY') - else: - ops.append(('LOAD_CONST', data.get_const(None))) + # the following two opcodes needed just for pypy, + # but seem logical elsewhere too (enter/exit 'try') + ops.append('SETUP_FINALLY') + ops.append('POP_BLOCK') + ops.append('BEGIN_FINALLY' if + _BYTECODE.has_begin_finally else + ('LOAD_CONST', data.get_const(None))) elif block == '': # we raise an exception to get the right block pushed raise_ops = [('LOAD_CONST', data.get_const(None)), ('RAISE_VARARGS', 1)] - setup_except = 'SETUP_EXCEPT' if _BYTECODE.has_setup_except else \ - 'SETUP_FINALLY' + setup_except = ('SETUP_EXCEPT' if + _BYTECODE.has_setup_except else + 'SETUP_FINALLY') ops.append((setup_except, _get_instructions_size(raise_ops))) ops += raise_ops for _ in _range(3): diff --git a/test_goto.py b/test_goto.py index 84294b5..fa18686 100644 --- a/test_goto.py +++ b/test_goto.py @@ -109,6 +109,7 @@ def func(): assert func() == (4, 0) + def test_jump_into_loop_iter_params(): @with_goto def func(): @@ -120,6 +121,7 @@ def func(): assert func() == (4, 0) + def test_jump_into_loop_iterable_param(): @with_goto def func(): @@ -130,6 +132,7 @@ def func(): assert func() == 4 + def test_jump_into_loop_bad_params(): @with_goto def func(): @@ -140,6 +143,7 @@ def func(): pytest.raises(TypeError, func) + def test_jump_into_loop_params_not_seq(): @with_goto def func(): @@ -150,6 +154,7 @@ def func(): pytest.raises(TypeError, func) + def test_jump_into_loop_param_with_index(): @with_goto def func(): @@ -163,6 +168,7 @@ def func(): assert func() == [-1, 0, 1, 2, 3, 4] + def test_jump_into_loop_param_without_index(): @with_goto def func(): @@ -175,6 +181,7 @@ def func(): pytest.raises(UnboundLocalError, func) + def test_jump_into_2_loops_and_live(): @with_goto def func(): @@ -189,6 +196,7 @@ def func(): assert func() == (2, 11 + 6) + def test_jump_out_then_back_in_for_loop_and_survive(): @with_goto def func(): @@ -331,6 +339,7 @@ def func(): assert func() == 2 + def test_jump_across_loops_with_param_and_live(): @with_goto def func(): @@ -346,6 +355,7 @@ def func(): assert func() == (4, 2) + def test_jump_into_with_unneeded_params_and_live(): @with_goto def func(): @@ -358,6 +368,7 @@ def func(): assert func() == (9, 0) + def test_jump_out_of_while_true_loop(): @with_goto def func(): @@ -413,7 +424,8 @@ def func(): return i, j, k assert func() == (9, 3, 5) - + + def test_jump_into_while_true_loop(): @with_goto def func(): @@ -426,9 +438,10 @@ def func(): if x == 1: break return x - + assert func() == 1 - + + def test_jump_into_while_true_loop_and_survive(): @with_goto def func(): @@ -440,9 +453,10 @@ def func(): label .inside break return i, x - + assert func() == (9, 0) - + + def test_jump_into_while_loop(): @with_goto def func(): @@ -453,9 +467,10 @@ def func(): label .inside c += 1 return c, x - + assert func() == (11, 10) - + + def test_jump_into_while_loop_and_survive(): @with_goto def func(): @@ -467,9 +482,10 @@ def func(): label .inside c += 1 return i, c, x - + assert func() == (4, 10, 5) - + + class Context: def __init__(self): self.enters = 0 @@ -534,6 +550,7 @@ def func(): pytest.raises(SyntaxError, with_goto, func) + def test_jump_into_with_block_with_param(): @with_goto def func(): @@ -545,6 +562,7 @@ def func(): assert func() == (0, 1) + def test_jump_into_with_block_with_params(): @with_goto def func(): @@ -556,6 +574,7 @@ def func(): assert func() == (0, 1) + def test_jump_into_with_block_and_survive(): @with_goto def func(): @@ -568,15 +587,18 @@ def func(): assert func() == (9, (0, 10)) + def test_jump_into_with_block_with_bad_params(): @with_goto def func(): with Context() as c: label .block goto.param .block = 123 + return c pytest.raises(AttributeError, func) + def test_jump_into_with_block_with_bad_exit_params(): class BadAttr: __exit__ = 123 @@ -586,9 +608,11 @@ def func(): with Context() as c: label .block goto.param .block = BadAttr + return c pytest.raises(TypeError, func) + def test_jump_out_then_in_with_block_and_survive(): @with_goto def func(): @@ -609,6 +633,7 @@ def func(): assert func() == (9, 10, (10, 10)) + def test_jump_out_then_in_2_nested_with_blocks_and_survive(): @with_goto def func(): @@ -632,6 +657,7 @@ def func(): assert func() == (10, 10, (11, 11), (10, 10)) + def test_generator(): @with_goto def func(): @@ -774,6 +800,7 @@ def func(): assert func() == 3 + def test_jump_into_try_except_block_and_survive(): @with_goto def func(): @@ -783,12 +810,13 @@ def func(): try: rv = 1 label .block - except: + except Exception: rv = 2 return i, rv assert func() == (9, 0) + def test_jump_into_try_finally_block_and_survive(): @with_goto def func(): @@ -804,6 +832,7 @@ def func(): assert func() == (9, 0, 1) + def test_jump_into_try_block_and_survive(): @with_goto def func(): @@ -813,7 +842,7 @@ def func(): try: rv = 1 label .block - except: + except Exception: rv = 2 finally: fv = 1 @@ -911,13 +940,14 @@ def func(): goto .block try: i = 2 - except: + except Exception: label .block i = 3 return i assert func() == 3 + def test_jump_into_except_block_and_live(): @with_goto def func(): @@ -926,13 +956,12 @@ def func(): goto .block try: j = 2 - except: + except Exception: label .block j = 3 return i, j - def test_jump_out_of_finally_block_w_except(): @with_goto def func(): @@ -967,7 +996,6 @@ def func(): return i, j, rv - def test_jump_out_of_finally_block(): @with_goto def func(): @@ -1026,6 +1054,7 @@ def func(): assert func() == (2, 0, 'try') + def test_jump_into_finally_block(): @with_goto def func(): @@ -1040,6 +1069,7 @@ def func(): assert func() == 2 + def test_jump_into_finally_block_and_live(): @with_goto def func(): @@ -1055,6 +1085,7 @@ def func(): assert func() == (2, 2) + def test_jump_to_unknown_label(): def func(): goto .unknown @@ -1071,7 +1102,7 @@ def func(): pytest.raises(SyntaxError, with_goto, func) -def test_jump_with_for_break(): # to see it doesn't confuse parser +def test_jump_with_for_break(): # to see it doesn't confuse parser @with_goto def func(): for i in range(4): @@ -1082,7 +1113,8 @@ def func(): assert func() == 0 -def test_jump_with_for_continue(): # to see it doesn't confuse parser + +def test_jump_with_for_continue(): # to see it doesn't confuse parser @with_goto def func(): for i in range(4): @@ -1093,7 +1125,8 @@ def func(): assert func() == 0 -def test_jump_with_for_return(): # to see it doesn't confuse parser + +def test_jump_with_for_return(): # to see it doesn't confuse parser @with_goto def func(): for i in range(4): @@ -1104,7 +1137,8 @@ def func(): assert func() == 0 -def test_jump_with_while_true_break(): # to see it doesn't confuse parser + +def test_jump_with_while_true_break(): # to see it doesn't confuse parser @with_goto def func(): i = 0