From fb88ac3da22c2d34e03d30521f22f24a5441129d Mon Sep 17 00:00:00 2001 From: danceratopz Date: Tue, 24 Oct 2023 23:13:19 +0100 Subject: [PATCH 01/34] refactor(fw): use click to wrap the pytest fill & tf entry points --- setup.cfg | 5 ++- src/entry_points/cli.py | 85 ++++++++++++++++++++++++++++++++++++++++ src/entry_points/fill.py | 15 ------- src/entry_points/tf.py | 19 --------- whitelist.txt | 1 + 5 files changed, 89 insertions(+), 36 deletions(-) create mode 100644 src/entry_points/cli.py delete mode 100644 src/entry_points/fill.py delete mode 100644 src/entry_points/tf.py diff --git a/setup.cfg b/setup.cfg index 91b27644896..8633ae27a61 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,6 +23,7 @@ package_dir = python_requires = >=3.10 install_requires = + click>=8.1.0,<9 ethereum@git+https://github.com/ethereum/execution-specs.git setuptools types-setuptools @@ -44,8 +45,8 @@ evm_transition_tool = [options.entry_points] console_scripts = - fill = entry_points.fill:main - tf = entry_points.tf:main + fill = entry_points.cli:fill + tf = entry_points.cli:tf order_fixtures = entry_points.order_fixtures:main pyspelling_soft_fail = entry_points.pyspelling_soft_fail:main markdownlintcli2_soft_fail = entry_points.markdownlintcli2_soft_fail:main diff --git a/src/entry_points/cli.py b/src/entry_points/cli.py new file mode 100644 index 00000000000..99aed0d92d0 --- /dev/null +++ b/src/entry_points/cli.py @@ -0,0 +1,85 @@ +""" +CLI entry points for the main commands provided by execution-spec-tests. + +These can be directly accessed in a prompt if the user has directly installed +the package via: + +``` +python -m venv venv +source venv/bin/activate +pip install -e .[doc,lint,test] +# or, more minimally: +pip install -e . +``` + +Then, the entry points can be executed via: + +``` +fill --help +# for example, or +fill --collect-only +``` + +They can also be executed (and debugged) directly in an interactive python +shell: + +``` +from src.entry_points.cli import fill +from click.testing import CliRunner + +runner = CliRunner() +result = runner.invoke(fill, ["--help"]) +print(result.output) +``` +""" +import sys + +import click +import pytest + + +@click.command(context_settings=dict(ignore_unknown_options=True)) +def tf(): # noqa: D103 + """ + The `tf` command, deprecated as of 2023-06. + """ + print( + "The `tf` command-line tool has been superseded by `fill`. Try:\n\n" + "fill --help\n\n" + "or see the online docs:\n" + "https://ethereum.github.io/execution-spec-tests/getting_started/executing_tests_command_line/" # noqa: E501 + ) + sys.exit(1) + + +@click.command(context_settings=dict(ignore_unknown_options=True)) +@click.option( + "-h", + "--help", + "help_flag", + is_flag=True, + default=False, + expose_value=True, + help="Show pytest's help message.", +) +@click.option( + "--pytest-help", + "pytest_help_flag", + is_flag=True, + default=False, + expose_value=True, + help="Show pytest's help message.", +) +@click.argument("pytest_args", nargs=-1, type=click.UNPROCESSED) +def fill(pytest_args, help_flag, pytest_help_flag): + """ + Entry point for the fill command. + """ + if help_flag: + pytest_args = ["--test-help"] + elif pytest_help_flag: + pytest_args = ["--help"] + else: + pytest_args = list(pytest_args) + + pytest.main(pytest_args) diff --git a/src/entry_points/fill.py b/src/entry_points/fill.py deleted file mode 100644 index 423f52558e4..00000000000 --- a/src/entry_points/fill.py +++ /dev/null @@ -1,15 +0,0 @@ -""" -Define an entry point wrapper for pytest. -""" - -import sys - -import pytest - - -def main(): # noqa: D103 - pytest.main(sys.argv[1:]) - - -if __name__ == "__main__": - main() diff --git a/src/entry_points/tf.py b/src/entry_points/tf.py deleted file mode 100644 index 1e893615606..00000000000 --- a/src/entry_points/tf.py +++ /dev/null @@ -1,19 +0,0 @@ -""" -Define an entry point wrapper for the now-deprecated tf command-line tool that -advises users to use the new `fill` tool. -""" - -import sys - - -def main(): # noqa: D103 - print( - "The `tf` command-line tool has been superseded by `fill`, please " - "see the docs for help running `fill`:\n" - "https://ethereum.github.io/execution-spec-tests/getting_started/executing_tests_command_line/" # noqa: E501 - ) - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/whitelist.txt b/whitelist.txt index 0c10ada66f1..722a8b95943 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -336,6 +336,7 @@ subclasses subcommand substring substrings +tf tryfirst trylast usefixtures From 9d8ed97710ebfd6876ea1ea900b39a95fc224a0b Mon Sep 17 00:00:00 2001 From: danceratopz Date: Wed, 25 Oct 2023 13:20:55 +0100 Subject: [PATCH 02/34] feat(pytest): add a skeleton consume cli command & pytest plugin --- consume/consume_fixtures.py | 8 ++ pytest-consume.ini | 9 +++ setup.cfg | 1 + src/entry_points/cli.py | 78 +++++++++++++------ .../fixture_consumer/fixture_consumer.py | 18 +++++ src/pytest_plugins/test_help/test_help.py | 31 +++++--- whitelist.txt | 1 + 7 files changed, 112 insertions(+), 34 deletions(-) create mode 100644 consume/consume_fixtures.py create mode 100644 pytest-consume.ini create mode 100644 src/pytest_plugins/fixture_consumer/fixture_consumer.py diff --git a/consume/consume_fixtures.py b/consume/consume_fixtures.py new file mode 100644 index 00000000000..23e23cc0ef1 --- /dev/null +++ b/consume/consume_fixtures.py @@ -0,0 +1,8 @@ +""" +Test module that generates parametrized test cases for each JSON fixture file +located in the fixtures directory. +""" + + +def test_fixtures(): # noqa: D103 + pass diff --git a/pytest-consume.ini b/pytest-consume.ini new file mode 100644 index 00000000000..10909dd7d5b --- /dev/null +++ b/pytest-consume.ini @@ -0,0 +1,9 @@ +[pytest] +console_output_style = count +minversion = 7.0 +python_files = *.py +testpaths = consume/consume_fixtures.py +addopts = + -p pytest_plugins.fixture_consumer.fixture_consumer + -p pytest_plugins.test_help.test_help + --dist loadscope diff --git a/setup.cfg b/setup.cfg index 8633ae27a61..97278ad29b9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -47,6 +47,7 @@ evm_transition_tool = console_scripts = fill = entry_points.cli:fill tf = entry_points.cli:tf + consume = entry_points.cli:consume order_fixtures = entry_points.order_fixtures:main pyspelling_soft_fail = entry_points.pyspelling_soft_fail:main markdownlintcli2_soft_fail = entry_points.markdownlintcli2_soft_fail:main diff --git a/src/entry_points/cli.py b/src/entry_points/cli.py index 99aed0d92d0..8ba0e88683b 100644 --- a/src/entry_points/cli.py +++ b/src/entry_points/cli.py @@ -52,34 +52,62 @@ def tf(): # noqa: D103 sys.exit(1) -@click.command(context_settings=dict(ignore_unknown_options=True)) -@click.option( - "-h", - "--help", - "help_flag", - is_flag=True, - default=False, - expose_value=True, - help="Show pytest's help message.", -) -@click.option( - "--pytest-help", - "pytest_help_flag", - is_flag=True, - default=False, - expose_value=True, - help="Show pytest's help message.", -) -@click.argument("pytest_args", nargs=-1, type=click.UNPROCESSED) -def fill(pytest_args, help_flag, pytest_help_flag): +def common_options(func): """ - Entry point for the fill command. + Common options for both the fill and consume commands. + """ + func = click.option( + "-h", + "--help", + "help_flag", + is_flag=True, + default=False, + expose_value=True, + help="Show pytest's help message.", + )(func) + + func = click.option( + "--pytest-help", + "pytest_help_flag", + is_flag=True, + default=False, + expose_value=True, + help="Show pytest's help message.", + )(func) + + func = click.argument("pytest_args", nargs=-1, type=click.UNPROCESSED)(func) + + return func + + +def handle_help_flags(pytest_args, help_flag, pytest_help_flag): + """ + Modify pytest arguments based on the provided help flags. """ if help_flag: - pytest_args = ["--test-help"] + return ["--test-help"] elif pytest_help_flag: - pytest_args = ["--help"] + return ["--help"] else: - pytest_args = list(pytest_args) + return list(pytest_args) + - pytest.main(pytest_args) +@click.command(context_settings=dict(ignore_unknown_options=True)) +@common_options +def fill(pytest_args, help_flag, pytest_help_flag): + """ + Entry point for the fill command. + """ + args = handle_help_flags(pytest_args, help_flag, pytest_help_flag) + pytest.main(args) + + +@click.command(context_settings=dict(ignore_unknown_options=True)) +@common_options +def consume(pytest_args, help_flag, pytest_help_flag): + """ + Entry point for the consume command. + """ + args = handle_help_flags(pytest_args, help_flag, pytest_help_flag) + args += ["-c", "pytest-consume.ini"] + pytest.main(args) diff --git a/src/pytest_plugins/fixture_consumer/fixture_consumer.py b/src/pytest_plugins/fixture_consumer/fixture_consumer.py new file mode 100644 index 00000000000..6a4b0c4e295 --- /dev/null +++ b/src/pytest_plugins/fixture_consumer/fixture_consumer.py @@ -0,0 +1,18 @@ +""" +A pytest plugin to execute the blocktest on the specified fixture directory. +""" + +from pathlib import Path + + +def pytest_addoption(parser): # noqa: D103 + consume_group = parser.getgroup( + "consume", "Arguments related to consuming fixtures via a client" + ) + consume_group.addoption( + "--fixture-directory", + type=Path, + action="store", + default="fixtures", + help="Specify the fixture directory to execute tests on.", + ) diff --git a/src/pytest_plugins/test_help/test_help.py b/src/pytest_plugins/test_help/test_help.py index 660f259bcb0..ffedd7f16ef 100644 --- a/src/pytest_plugins/test_help/test_help.py +++ b/src/pytest_plugins/test_help/test_help.py @@ -4,6 +4,7 @@ """ import argparse +from pathlib import Path import pytest @@ -18,7 +19,10 @@ def pytest_addoption(parser): action="store_true", dest="show_test_help", default=False, - help="Only show help options specific to execution-spec-tests and exit.", + help=( + "Only show help options specific to a specific execution-spec-tests command and " + "exit." + ), ) @@ -37,14 +41,23 @@ def show_test_help(config): that group is specific to execution-spec-tests command-line arguments. """ - test_group_substrings = [ - "execution-spec-tests", - "evm", - "solc", - "fork range", - "filler location", - "defining debug", # the "debug" group in test_filler plugin. - ] + pytest_ini = Path(config.inifile) + if pytest_ini.name == "pytest.ini": + test_group_substrings = [ + "execution-spec-tests", + "evm", + "solc", + "fork range", + "filler location", + "defining debug", # the "debug" group in test_filler plugin. + ] + elif pytest_ini.name == "pytest-consume.ini": + test_group_substrings = [ + "execution-spec-tests", + "consume", + ] + else: + raise ValueError("Unexpected pytest.ini file option generating test help.") test_parser = argparse.ArgumentParser() for group in config._parser.optparser._action_groups: diff --git a/whitelist.txt b/whitelist.txt index 722a8b95943..e314e396d00 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -307,6 +307,7 @@ hookimpl hookwrapper IEXEC IGNORECASE +inifile iterdir ljust makepyfile From 35b208c7c0f74c231ee8c744a7ee09716c8777b5 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Wed, 25 Oct 2023 20:05:25 +0100 Subject: [PATCH 03/34] feat(pytest): add initial implementation of the consume plugin --- consume/consume_fixtures.py | 8 -- pytest-consume.ini | 2 +- .../fixture_consumer/fixture_consumer.py | 105 +++++++++++++++++- 3 files changed, 104 insertions(+), 11 deletions(-) delete mode 100644 consume/consume_fixtures.py diff --git a/consume/consume_fixtures.py b/consume/consume_fixtures.py deleted file mode 100644 index 23e23cc0ef1..00000000000 --- a/consume/consume_fixtures.py +++ /dev/null @@ -1,8 +0,0 @@ -""" -Test module that generates parametrized test cases for each JSON fixture file -located in the fixtures directory. -""" - - -def test_fixtures(): # noqa: D103 - pass diff --git a/pytest-consume.ini b/pytest-consume.ini index 10909dd7d5b..85febf37437 100644 --- a/pytest-consume.ini +++ b/pytest-consume.ini @@ -2,7 +2,7 @@ console_output_style = count minversion = 7.0 python_files = *.py -testpaths = consume/consume_fixtures.py +testpaths = src/pytest_plugins/fixture_consumer/consume_fixtures.py addopts = -p pytest_plugins.fixture_consumer.fixture_consumer -p pytest_plugins.test_help.test_help diff --git a/src/pytest_plugins/fixture_consumer/fixture_consumer.py b/src/pytest_plugins/fixture_consumer/fixture_consumer.py index 6a4b0c4e295..44e3cc85fcc 100644 --- a/src/pytest_plugins/fixture_consumer/fixture_consumer.py +++ b/src/pytest_plugins/fixture_consumer/fixture_consumer.py @@ -1,8 +1,21 @@ """ A pytest plugin to execute the blocktest on the specified fixture directory. """ - +import json +from dataclasses import dataclass from pathlib import Path +from typing import Generator + +import pytest + +from evm_transition_tool import FixtureFormats, TransitionTool + + +@dataclass +class FixtureData: # noqa: D101 + fixture_name: str + fixture_format: FixtureFormats + json_file_path: Path def pytest_addoption(parser): # noqa: D103 @@ -14,5 +27,93 @@ def pytest_addoption(parser): # noqa: D103 type=Path, action="store", default="fixtures", - help="Specify the fixture directory to execute tests on.", + help="Specify the fixture directory to execute tests on. Default: 'fixtures'.", + ) + consume_group.addoption( + "--multiple-tests-per-file", + action="store_true", + dest="do_multiple_tests_per_file", + default=False, + help="Execute one test per fixture in each json file within the fixtures directory.", + ) + consume_group.addoption( + "--evm-bin", + action="store", + dest="evm_bin", + type=Path, + default=None, + help=( + "Path to an evm executable that provides `blocktest`. Default: First 'evm' entry in " + "PATH." + ), + ) + consume_group.addoption( + "--traces", + action="store_true", + dest="evm_collect_traces", + default=False, + help="Collect traces of the execution information from the transition tool.", ) + + +@pytest.fixture(autouse=True, scope="session") +def evm(request) -> Generator[TransitionTool, None, None]: + """ + Returns the interface to the evm binary that will consume tests. + """ + evm = TransitionTool.from_binary_path( + binary_path=request.config.getoption("evm_bin"), + # TODO: The verify_fixture() method doesn't currently use this option. + trace=request.config.getoption("evm_collect_traces"), + ) + yield evm + evm.shutdown() + + +@pytest.fixture(scope="function") +def json_fixture_path(fixture_data): + """ + Provide the path to the current JSON fixture file. + """ + return fixture_data.json_file_path + + +@pytest.fixture(scope="function") +def fixture_name(fixture_data): + """ + The name of the current fixture. + """ + return fixture_data.fixture_name + + +def pytest_generate_tests(metafunc): + """ + Generate test cases for every fixture in all JSON fixture files within the + fixtures directory. + """ + if "fixture_name" in metafunc.fixturenames: + fixtures_directory = metafunc.config.getoption("fixture_directory") + + fixture_data = [] + fixture_ids = [] + for json_file in fixtures_directory.glob("**/*.json"): + with json_file.open() as f: + data = json.load(f) + if metafunc.config.getoption("do_multiple_tests_per_file"): + for fixture_name in data.keys(): + fixture_data.append( + FixtureData(fixture_name, FixtureFormats.BLOCKCHAIN_TEST, json_file) + ) + fixture_ids.append(f"{json_file.name}_{fixture_name}") + else: + fixture_data.append( + FixtureData(json_file.name, FixtureFormats.BLOCKCHAIN_TEST, json_file) + ) + fixture_ids.append(f"{json_file.name}") + + metafunc.parametrize("fixture_data", fixture_data, ids=fixture_ids) + + +def test_fixtures(evm: TransitionTool, json_fixture_path: Path, fixture_name: str): # noqa: D103 + test_dump_dir = None + evm.verify_fixture(FixtureFormats.BLOCKCHAIN_TEST, json_fixture_path, test_dump_dir) From e0ab7b8eba06574ebd4f992c44911840e9dc1333 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Thu, 26 Oct 2023 07:36:32 +0100 Subject: [PATCH 04/34] fix(pytest): remove duplicated test function from plugin module --- src/pytest_plugins/fixture_consumer/fixture_consumer.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/pytest_plugins/fixture_consumer/fixture_consumer.py b/src/pytest_plugins/fixture_consumer/fixture_consumer.py index 44e3cc85fcc..541572e991a 100644 --- a/src/pytest_plugins/fixture_consumer/fixture_consumer.py +++ b/src/pytest_plugins/fixture_consumer/fixture_consumer.py @@ -112,8 +112,3 @@ def pytest_generate_tests(metafunc): fixture_ids.append(f"{json_file.name}") metafunc.parametrize("fixture_data", fixture_data, ids=fixture_ids) - - -def test_fixtures(evm: TransitionTool, json_fixture_path: Path, fixture_name: str): # noqa: D103 - test_dump_dir = None - evm.verify_fixture(FixtureFormats.BLOCKCHAIN_TEST, json_fixture_path, test_dump_dir) From 0a22cbfb56dc85176d5c4a4a7fab3800446ac55d Mon Sep 17 00:00:00 2001 From: danceratopz Date: Thu, 26 Oct 2023 13:38:29 +0100 Subject: [PATCH 05/34] feat(pytest): run blocktest with --single-test option; add evm_dump_dir --- src/evm_transition_tool/geth.py | 14 +++++++--- src/evm_transition_tool/transition_tool.py | 6 ++++- .../fixture_consumer/fixture_consumer.py | 27 ++++++++++++++++++- src/pytest_plugins/test_filler/test_filler.py | 2 +- 4 files changed, 42 insertions(+), 7 deletions(-) diff --git a/src/evm_transition_tool/geth.py b/src/evm_transition_tool/geth.py index 06dfe3fe23e..282639cec06 100644 --- a/src/evm_transition_tool/geth.py +++ b/src/evm_transition_tool/geth.py @@ -54,7 +54,11 @@ def is_fork_supported(self, fork: Fork) -> bool: return fork.fork() in self.help_string def verify_fixture( - self, fixture_format: FixtureFormats, fixture_path: Path, debug_output_path: Optional[Path] + self, + fixture_format: FixtureFormats, + fixture_path: Path, + fixture_name: Optional[str], + debug_output_path: Optional[Path], ): """ Executes `evm [state|block]test` to verify the fixture at `fixture_path`. @@ -73,6 +77,9 @@ def verify_fixture( else: raise Exception(f"Invalid test fixture format: {fixture_format}") + if fixture_name: + command.append("--single-test") + command.append(fixture_name) command.append(str(fixture_path)) result = subprocess.run( @@ -83,7 +90,6 @@ def verify_fixture( if debug_output_path: debug_fixture_path = debug_output_path / "fixtures.json" - shutil.copyfile(fixture_path, debug_fixture_path) # Use the local copy of the fixture in the debug directory verify_fixtures_call = " ".join(command[:-1]) + f" {debug_fixture_path}" verify_fixtures_script = textwrap.dedent( @@ -102,9 +108,9 @@ def verify_fixture( "verify_fixtures.sh+x": verify_fixtures_script, }, ) + shutil.copyfile(fixture_path, debug_fixture_path) if result.returncode != 0: raise Exception( - f"Failed to verify fixture via: '{' '.join(command)}'. " - f"Error: '{result.stderr.decode()}'" + f"EVM test failed.\n{' '.join(command)}\n\n Error:\n{result.stderr.decode()}" ) diff --git a/src/evm_transition_tool/transition_tool.py b/src/evm_transition_tool/transition_tool.py index ae727a0aa0d..4bf75b749a8 100644 --- a/src/evm_transition_tool/transition_tool.py +++ b/src/evm_transition_tool/transition_tool.py @@ -439,7 +439,11 @@ def calc_state_root( return new_alloc, bytes.fromhex(state_root[2:]) def verify_fixture( - self, fixture_format: FixtureFormats, fixture_path: Path, debug_output_path: Optional[Path] + self, + fixture_format: FixtureFormats, + fixture_path: Path, + fixture_name: Optional[str], + debug_output_path: Optional[Path], ): """ Executes `evm [state|block]test` to verify the fixture at `fixture_path`. diff --git a/src/pytest_plugins/fixture_consumer/fixture_consumer.py b/src/pytest_plugins/fixture_consumer/fixture_consumer.py index 541572e991a..825e7db69dc 100644 --- a/src/pytest_plugins/fixture_consumer/fixture_consumer.py +++ b/src/pytest_plugins/fixture_consumer/fixture_consumer.py @@ -4,7 +4,7 @@ import json from dataclasses import dataclass from pathlib import Path -from typing import Generator +from typing import Generator, Optional import pytest @@ -54,6 +54,15 @@ def pytest_addoption(parser): # noqa: D103 default=False, help="Collect traces of the execution information from the transition tool.", ) + debug_group = parser.getgroup("debug", "Arguments defining debug behavior") + debug_group.addoption( + "--evm-dump-dir", + action="store", + dest="base_dump_dir", + type=Path, + default=None, + help="Path to dump the transition tool debug output.", + ) @pytest.fixture(autouse=True, scope="session") @@ -70,6 +79,22 @@ def evm(request) -> Generator[TransitionTool, None, None]: evm.shutdown() +@pytest.fixture(scope="function") +def test_dump_dir(request, json_fixture_path: Path, fixture_name: str) -> Optional[Path]: + """ + The directory to write evm debug output to. + """ + base_dump_dir = request.config.getoption("base_dump_dir") + if not base_dump_dir: + return None + if request.config.getoption("do_multiple_tests_per_file"): + if len(fixture_name) > 142: + # ensure file name is not too long for eCryptFS + fixture_name = fixture_name[:70] + "..." + fixture_name[-70:] + return base_dump_dir / json_fixture_path.stem / fixture_name + return base_dump_dir / json_fixture_path.stem + + @pytest.fixture(scope="function") def json_fixture_path(fixture_data): """ diff --git a/src/pytest_plugins/test_filler/test_filler.py b/src/pytest_plugins/test_filler/test_filler.py index 943b214f01c..43e25503ae7 100644 --- a/src/pytest_plugins/test_filler/test_filler.py +++ b/src/pytest_plugins/test_filler/test_filler.py @@ -472,7 +472,7 @@ def verify_fixture_files(self, evm_fixture_verification: TransitionTool) -> None item = self.json_path_to_test_item[fixture_path] verify_fixtures_dump_dir = self._get_verify_fixtures_dump_dir(item) evm_fixture_verification.verify_fixture( - fixture_format, fixture_path, verify_fixtures_dump_dir + fixture_format, fixture_path, "", verify_fixtures_dump_dir ) def _get_verify_fixtures_dump_dir( From ee66a76e4a16a22095e42fef0e28dfa4a67decbe Mon Sep 17 00:00:00 2001 From: danceratopz Date: Thu, 26 Oct 2023 15:17:53 +0100 Subject: [PATCH 06/34] fix(pytest): update help groups for consume command --- src/pytest_plugins/test_help/test_help.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pytest_plugins/test_help/test_help.py b/src/pytest_plugins/test_help/test_help.py index ffedd7f16ef..c6f5b8d30ea 100644 --- a/src/pytest_plugins/test_help/test_help.py +++ b/src/pytest_plugins/test_help/test_help.py @@ -49,12 +49,13 @@ def show_test_help(config): "solc", "fork range", "filler location", - "defining debug", # the "debug" group in test_filler plugin. + "defining debug", ] elif pytest_ini.name == "pytest-consume.ini": test_group_substrings = [ "execution-spec-tests", - "consume", + "consuming", + "defining debug", ] else: raise ValueError("Unexpected pytest.ini file option generating test help.") From cac60776010d2f9abb864fe20b074a5f6b303906 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Thu, 26 Oct 2023 16:42:30 +0100 Subject: [PATCH 07/34] feat(pytest): detect blocktest and --single-test availability If blocktest is unavailable exit pytest before the test session starts. If --single-test is supported test individual fixtures using that, otherwise test per fixture file, not per single fixture. --- src/evm_transition_tool/execution_specs.py | 28 +++++++++++ src/evm_transition_tool/geth.py | 17 ++++++- src/evm_transition_tool/transition_tool.py | 11 ++++- .../fixture_consumer/fixture_consumer.py | 47 ++++++++++++------- src/pytest_plugins/test_filler/test_filler.py | 8 +++- 5 files changed, 91 insertions(+), 20 deletions(-) diff --git a/src/evm_transition_tool/execution_specs.py b/src/evm_transition_tool/execution_specs.py index b7d06f430fb..2d492b3bdc6 100644 --- a/src/evm_transition_tool/execution_specs.py +++ b/src/evm_transition_tool/execution_specs.py @@ -11,6 +11,7 @@ from ethereum_test_forks import Constantinople, ConstantinopleFix, Fork from .geth import GethTransitionTool +from .transition_tool import FixtureFormats UNSUPPORTED_FORKS = ( Constantinople, @@ -99,3 +100,30 @@ def is_fork_supported(self, fork: Fork) -> bool: Currently, ethereum-spec-evm provides no way to determine supported forks. """ return fork not in UNSUPPORTED_FORKS + + def get_blocktest_help(self) -> str: + """ + Return the help string for the blocktest subcommand. + """ + raise NotImplementedError( + "The `blocktest` command is not supported by the ethereum-spec-evm. " + "Use geth's evm tool." + ) + + def verify_fixture( + self, + fixture_format: FixtureFormats, + fixture_path: Path, + use_evm_single_test: bool, + fixture_name: Optional[str], + debug_output_path: Optional[Path], + ): + """ + Executes `evm [state|block]test` to verify the fixture at `fixture_path`. + + Currently only implemented by geth's evm. + """ + raise NotImplementedError( + "The `verify_fixture()` function is not supported by the ethereum-spec-evm. " + "Use geth's evm tool." + ) diff --git a/src/evm_transition_tool/geth.py b/src/evm_transition_tool/geth.py index 282639cec06..3ca205a716d 100644 --- a/src/evm_transition_tool/geth.py +++ b/src/evm_transition_tool/geth.py @@ -53,10 +53,24 @@ def is_fork_supported(self, fork: Fork) -> bool: """ return fork.fork() in self.help_string + def get_blocktest_help(self) -> str: + """ + Return the help string for the blocktest subcommand. + """ + args = [str(self.binary), "blocktest", "--help"] + try: + result = subprocess.run(args, capture_output=True, text=True) + except subprocess.CalledProcessError as e: + raise Exception("evm process unexpectedly returned a non-zero status code: " f"{e}.") + except Exception as e: + raise Exception(f"Unexpected exception calling evm tool: {e}.") + return result.stdout + def verify_fixture( self, fixture_format: FixtureFormats, fixture_path: Path, + use_evm_single_test: bool, fixture_name: Optional[str], debug_output_path: Optional[Path], ): @@ -77,7 +91,8 @@ def verify_fixture( else: raise Exception(f"Invalid test fixture format: {fixture_format}") - if fixture_name: + if use_evm_single_test: + assert isinstance(fixture_name, str), "fixture_name must be a string" command.append("--single-test") command.append(fixture_name) command.append(str(fixture_path)) diff --git a/src/evm_transition_tool/transition_tool.py b/src/evm_transition_tool/transition_tool.py index 4bf75b749a8..c7aa6ff434a 100644 --- a/src/evm_transition_tool/transition_tool.py +++ b/src/evm_transition_tool/transition_tool.py @@ -438,10 +438,19 @@ def calc_state_root( raise Exception("Unable to calculate state root") return new_alloc, bytes.fromhex(state_root[2:]) + def get_blocktest_help(self) -> str: + """ + Return the help string for the blocktest subcommand. + """ + raise NotImplementedError( + "The `blocktest` command is not supported by this tool. Use geth's evm tool." + ) + def verify_fixture( self, fixture_format: FixtureFormats, fixture_path: Path, + use_evm_single_test: bool, fixture_name: Optional[str], debug_output_path: Optional[Path], ): @@ -450,6 +459,6 @@ def verify_fixture( Currently only implemented by geth's evm. """ - raise Exception( + raise NotImplementedError( "The `verify_fixture()` function is not supported by this tool. Use geth's evm tool." ) diff --git a/src/pytest_plugins/fixture_consumer/fixture_consumer.py b/src/pytest_plugins/fixture_consumer/fixture_consumer.py index 825e7db69dc..637b1462d7d 100644 --- a/src/pytest_plugins/fixture_consumer/fixture_consumer.py +++ b/src/pytest_plugins/fixture_consumer/fixture_consumer.py @@ -29,13 +29,6 @@ def pytest_addoption(parser): # noqa: D103 default="fixtures", help="Specify the fixture directory to execute tests on. Default: 'fixtures'.", ) - consume_group.addoption( - "--multiple-tests-per-file", - action="store_true", - dest="do_multiple_tests_per_file", - default=False, - help="Execute one test per fixture in each json file within the fixtures directory.", - ) consume_group.addoption( "--evm-bin", action="store", @@ -65,29 +58,48 @@ def pytest_addoption(parser): # noqa: D103 ) +def pytest_configure(config): # noqa: D103 + evm = TransitionTool.from_binary_path( + binary_path=config.getoption("evm_bin"), + # TODO: The verify_fixture() method doesn't currently use this option. + trace=config.getoption("evm_collect_traces"), + ) + try: + blocktest_help_string = evm.get_blocktest_help() + except NotImplementedError as e: + pytest.exit(str(e)) + config.evm = evm + config.evm_use_single_test = "--single-test" in blocktest_help_string + + @pytest.fixture(autouse=True, scope="session") def evm(request) -> Generator[TransitionTool, None, None]: """ Returns the interface to the evm binary that will consume tests. """ - evm = TransitionTool.from_binary_path( - binary_path=request.config.getoption("evm_bin"), - # TODO: The verify_fixture() method doesn't currently use this option. - trace=request.config.getoption("evm_collect_traces"), - ) - yield evm - evm.shutdown() + yield request.config.evm + request.config.evm.shutdown() + + +@pytest.fixture(scope="session") +def evm_use_single_test(request) -> bool: + """ + Helper specifying whether to execute one test per fixture in each json file. + """ + return request.config.evm_use_single_test @pytest.fixture(scope="function") -def test_dump_dir(request, json_fixture_path: Path, fixture_name: str) -> Optional[Path]: +def test_dump_dir( + request, json_fixture_path: Path, fixture_name: str, evm_use_single_test: bool +) -> Optional[Path]: """ The directory to write evm debug output to. """ base_dump_dir = request.config.getoption("base_dump_dir") if not base_dump_dir: return None - if request.config.getoption("do_multiple_tests_per_file"): + if evm_use_single_test: if len(fixture_name) > 142: # ensure file name is not too long for eCryptFS fixture_name = fixture_name[:70] + "..." + fixture_name[-70:] @@ -124,13 +136,14 @@ def pytest_generate_tests(metafunc): for json_file in fixtures_directory.glob("**/*.json"): with json_file.open() as f: data = json.load(f) - if metafunc.config.getoption("do_multiple_tests_per_file"): + if metafunc.config.evm_use_single_test: for fixture_name in data.keys(): fixture_data.append( FixtureData(fixture_name, FixtureFormats.BLOCKCHAIN_TEST, json_file) ) fixture_ids.append(f"{json_file.name}_{fixture_name}") else: + # evm bin does not support --single-test fixture_data.append( FixtureData(json_file.name, FixtureFormats.BLOCKCHAIN_TEST, json_file) ) diff --git a/src/pytest_plugins/test_filler/test_filler.py b/src/pytest_plugins/test_filler/test_filler.py index 43e25503ae7..02d279d31b0 100644 --- a/src/pytest_plugins/test_filler/test_filler.py +++ b/src/pytest_plugins/test_filler/test_filler.py @@ -471,8 +471,14 @@ def verify_fixture_files(self, evm_fixture_verification: TransitionTool) -> None for fixture_path, fixture_format in self.json_path_to_fixture_type.items(): item = self.json_path_to_test_item[fixture_path] verify_fixtures_dump_dir = self._get_verify_fixtures_dump_dir(item) + use_single_test = False + fixture_name = "" evm_fixture_verification.verify_fixture( - fixture_format, fixture_path, "", verify_fixtures_dump_dir + fixture_format, + fixture_path, + use_single_test, + fixture_name, + verify_fixtures_dump_dir, ) def _get_verify_fixtures_dump_dir( From da07ba2aa6180edc0dff6ceeaa97a8172b8c5360 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Thu, 2 Nov 2023 15:58:31 +0100 Subject: [PATCH 08/34] feat(pytest): re-add test_fixtures test; change --single-test to --run --- src/evm_transition_tool/geth.py | 2 +- .../fixture_consumer/consume_fixtures.py | 23 +++++++++++++++++++ .../fixture_consumer/fixture_consumer.py | 14 ++++++++--- 3 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 src/pytest_plugins/fixture_consumer/consume_fixtures.py diff --git a/src/evm_transition_tool/geth.py b/src/evm_transition_tool/geth.py index 3ca205a716d..a2a14d19e84 100644 --- a/src/evm_transition_tool/geth.py +++ b/src/evm_transition_tool/geth.py @@ -93,7 +93,7 @@ def verify_fixture( if use_evm_single_test: assert isinstance(fixture_name, str), "fixture_name must be a string" - command.append("--single-test") + command.append("--run") command.append(fixture_name) command.append(str(fixture_path)) diff --git a/src/pytest_plugins/fixture_consumer/consume_fixtures.py b/src/pytest_plugins/fixture_consumer/consume_fixtures.py new file mode 100644 index 00000000000..a70947ef6f5 --- /dev/null +++ b/src/pytest_plugins/fixture_consumer/consume_fixtures.py @@ -0,0 +1,23 @@ +""" +Test module that defines a test to execute a fixture against an EVM blocktest-like command. +""" +from pathlib import Path +from typing import Optional + +from evm_transition_tool import FixtureFormats, TransitionTool + + +def test_fixtures( # noqa: D103 + evm: TransitionTool, + json_fixture_path: Path, + evm_use_single_test: bool, + fixture_name: str, + test_dump_dir: Optional[Path], +): + evm.verify_fixture( + FixtureFormats.BLOCKCHAIN_TEST, + json_fixture_path, + evm_use_single_test, + fixture_name, + test_dump_dir, + ) diff --git a/src/pytest_plugins/fixture_consumer/fixture_consumer.py b/src/pytest_plugins/fixture_consumer/fixture_consumer.py index 637b1462d7d..8b85bb894fb 100644 --- a/src/pytest_plugins/fixture_consumer/fixture_consumer.py +++ b/src/pytest_plugins/fixture_consumer/fixture_consumer.py @@ -69,7 +69,7 @@ def pytest_configure(config): # noqa: D103 except NotImplementedError as e: pytest.exit(str(e)) config.evm = evm - config.evm_use_single_test = "--single-test" in blocktest_help_string + config.evm_use_single_test = "--run" in blocktest_help_string @pytest.fixture(autouse=True, scope="session") @@ -108,7 +108,7 @@ def test_dump_dir( @pytest.fixture(scope="function") -def json_fixture_path(fixture_data): +def json_fixture_path(fixture_data: FixtureData): """ Provide the path to the current JSON fixture file. """ @@ -116,7 +116,15 @@ def json_fixture_path(fixture_data): @pytest.fixture(scope="function") -def fixture_name(fixture_data): +def fixture_format(fixture_data: FixtureData): + """ + The format of the current fixture. + """ + return fixture_data.fixture_format + + +@pytest.fixture(scope="function") +def fixture_name(fixture_data: FixtureData): """ The name of the current fixture. """ From ce20ba0204d351bd5ecb3ea4015658b59d135a7f Mon Sep 17 00:00:00 2001 From: danceratopz Date: Thu, 2 Nov 2023 16:00:47 +0100 Subject: [PATCH 09/34] chore(pytest): remove restrictive & unnecessary xdist config for consume command --- pytest-consume.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/pytest-consume.ini b/pytest-consume.ini index 85febf37437..61ccee5a27c 100644 --- a/pytest-consume.ini +++ b/pytest-consume.ini @@ -6,4 +6,3 @@ testpaths = src/pytest_plugins/fixture_consumer/consume_fixtures.py addopts = -p pytest_plugins.fixture_consumer.fixture_consumer -p pytest_plugins.test_help.test_help - --dist loadscope From a6b149c2f96ee295649a0ecf4834dfce5ddba5f3 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Fri, 17 Nov 2023 07:25:42 +0300 Subject: [PATCH 10/34] feat(pytest): add a friendlier command alias for fill --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 97278ad29b9..1040a4f0552 100644 --- a/setup.cfg +++ b/setup.cfg @@ -46,6 +46,7 @@ evm_transition_tool = [options.entry_points] console_scripts = fill = entry_points.cli:fill + phil = entry_points.cli:fill tf = entry_points.cli:tf consume = entry_points.cli:consume order_fixtures = entry_points.order_fixtures:main From 863c095d7e5af9659457933cf298594f2daaee07 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Sat, 25 Nov 2023 23:00:35 +0100 Subject: [PATCH 11/34] feat(fw): add a re-write of the hive consensus simulator --- .gitignore | 3 + pytest-consume-direct.ini | 11 + pytest-consume-via-engine-api.ini | 11 + pytest-consume-via-rlp.ini | 11 + pytest-consume.ini | 8 - setup.cfg | 4 +- src/entry_points/cli.py | 69 +++- src/ethereum_test_tools/common/types.py | 27 ++ src/pytest_plugins/consume/consume.py | 87 +++++ .../consume_direct.py} | 28 +- .../consume_via_engine_api.py | 5 + .../consume_via_rlp/__init__.py | 4 + .../consume_via_rlp/consume_via_rlp.py | 123 +++++++ .../consume_via_rlp/network_ruleset_hive.py | 309 ++++++++++++++++++ src/pytest_plugins/pytest_hive/pytest_hive.py | 73 +++++ src/pytest_plugins/test_filler/test_filler.py | 2 +- src/pytest_plugins/test_help/test_help.py | 6 +- tests/cancun/eip4788_beacon_root/conftest.py | 4 +- .../test_point_evaluation_precompile.py | 4 +- .../test_point_evaluation_precompile_gas.py | 4 +- .../test_direct.py | 3 +- tests_consume/test_via_engine_api.py | 2 + tests_consume/test_via_rlp.py | 244 ++++++++++++++ whitelist.txt | 14 +- 24 files changed, 1016 insertions(+), 40 deletions(-) create mode 100644 pytest-consume-direct.ini create mode 100644 pytest-consume-via-engine-api.ini create mode 100644 pytest-consume-via-rlp.ini delete mode 100644 pytest-consume.ini create mode 100644 src/pytest_plugins/consume/consume.py rename src/pytest_plugins/{fixture_consumer/fixture_consumer.py => consume_direct/consume_direct.py} (83%) create mode 100644 src/pytest_plugins/consume_via_engine_api/consume_via_engine_api.py create mode 100644 src/pytest_plugins/consume_via_rlp/__init__.py create mode 100644 src/pytest_plugins/consume_via_rlp/consume_via_rlp.py create mode 100644 src/pytest_plugins/consume_via_rlp/network_ruleset_hive.py create mode 100644 src/pytest_plugins/pytest_hive/pytest_hive.py rename src/pytest_plugins/fixture_consumer/consume_fixtures.py => tests_consume/test_direct.py (78%) create mode 100644 tests_consume/test_via_engine_api.py create mode 100644 tests_consume/test_via_rlp.py diff --git a/.gitignore b/.gitignore index 4a40fef8857..3c31e0b2bb0 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,6 @@ _readthedocs site venv-docs/ .pyspelling_en.dict + +# cached fixture downloads (consume) +cached_downloads/ \ No newline at end of file diff --git a/pytest-consume-direct.ini b/pytest-consume-direct.ini new file mode 100644 index 00000000000..1e671a04894 --- /dev/null +++ b/pytest-consume-direct.ini @@ -0,0 +1,11 @@ +[pytest] +console_output_style = count +minversion = 7.0 +python_files = test_direct.py +testpaths = tests_consume/test_direct.py +addopts = + -rxXs + -p pytest_plugins.consume.consume + -p pytest_plugins.pytest_hive.pytest_hive + -p pytest_plugins.consume_direct.consume_direct + -p pytest_plugins.test_help.test_help diff --git a/pytest-consume-via-engine-api.ini b/pytest-consume-via-engine-api.ini new file mode 100644 index 00000000000..1fe64f137d7 --- /dev/null +++ b/pytest-consume-via-engine-api.ini @@ -0,0 +1,11 @@ +[pytest] +console_output_style = count +minversion = 7.0 +python_files = test_via_engine_api.py +testpaths = tests_consume/test_via_engine_api.py +addopts = + -rxXs + -p pytest_plugins.consume.consume + -p pytest_plugins.pytest_hive.pytest_hive + -p pytest_plugins.consume_via_engine_api.consume_via_engine_api + -p pytest_plugins.test_help.test_help diff --git a/pytest-consume-via-rlp.ini b/pytest-consume-via-rlp.ini new file mode 100644 index 00000000000..08f150f498c --- /dev/null +++ b/pytest-consume-via-rlp.ini @@ -0,0 +1,11 @@ +[pytest] +console_output_style = count +minversion = 7.0 +python_files = test_via_rlp.py +testpaths = tests_consume/test_via_rlp.py +addopts = + -rxXs + -p pytest_plugins.consume.consume + -p pytest_plugins.pytest_hive.pytest_hive + -p pytest_plugins.consume_via_rlp.consume_via_rlp + -p pytest_plugins.test_help.test_help diff --git a/pytest-consume.ini b/pytest-consume.ini deleted file mode 100644 index 61ccee5a27c..00000000000 --- a/pytest-consume.ini +++ /dev/null @@ -1,8 +0,0 @@ -[pytest] -console_output_style = count -minversion = 7.0 -python_files = *.py -testpaths = src/pytest_plugins/fixture_consumer/consume_fixtures.py -addopts = - -p pytest_plugins.fixture_consumer.fixture_consumer - -p pytest_plugins.test_help.test_help diff --git a/setup.cfg b/setup.cfg index 1040a4f0552..9aed2df0fee 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,6 +25,7 @@ python_requires = >=3.10 install_requires = click>=8.1.0,<9 ethereum@git+https://github.com/ethereum/execution-specs.git + hive.py@git+https://github.com/danceratopz/hive.py@fix/client/files-def-in-post-with-files setuptools types-setuptools requests>=2.31.0 @@ -32,6 +33,7 @@ install_requires = pytest==7.3.2 pytest-xdist>=3.3.1,<4 coincurve>=18.0.0,<19 + tenacity>8.2.0,<9 trie==2.1.1 semver==3.0.1 @@ -61,7 +63,7 @@ test = lint = isort>=5.8,<6 - mypy==0.982; implementation_name == "cpython" + mypy>=1.4.0,<2; implementation_name == "cpython" types-requests black==22.3.0; implementation_name == "cpython" flake8-spellcheck>=0.24,<0.25 diff --git a/src/entry_points/cli.py b/src/entry_points/cli.py index 8ba0e88683b..fea6dafba75 100644 --- a/src/entry_points/cli.py +++ b/src/entry_points/cli.py @@ -32,7 +32,9 @@ print(result.output) ``` """ +import os import sys +import warnings import click import pytest @@ -92,6 +94,30 @@ def handle_help_flags(pytest_args, help_flag, pytest_help_flag): return list(pytest_args) +def get_hive_flags_from_env(): + """ + Read simulator flags from environment variables and convert them, as best as + possible, into pytest flags. + """ + pytest_args = [] + xdist_workers = os.getenv("HIVE_PARALLELISM") + if xdist_workers is not None: + pytest_args.extend("-n", xdist_workers) + test_pattern = os.getenv("HIVE_TEST_PATTERN") + if test_pattern is not None: + # TODO: Check that the regex is a valid pytest -k "test expression" + pytest_args.extend("-k", test_pattern) + random_seed = os.getenv("HIVE_RANDOM_SEED") + if random_seed is not None: + # TODO: implement random seed + warnings.warning("HIVE_RANDOM_SEED is not yet supported.") + log_level = os.getenv("HIVE_LOGLEVEL") + if log_level is not None: + # TODO add logging within simulators and implement log level via cli + warnings.warning("HIVE_LOG_LEVEL is not yet supported.") + return pytest_args + + @click.command(context_settings=dict(ignore_unknown_options=True)) @common_options def fill(pytest_args, help_flag, pytest_help_flag): @@ -102,12 +128,49 @@ def fill(pytest_args, help_flag, pytest_help_flag): pytest.main(args) +@click.group() +def consume(): + """ + Help clients consume JSON test fixtures. + """ + pass + + @click.command(context_settings=dict(ignore_unknown_options=True)) @common_options -def consume(pytest_args, help_flag, pytest_help_flag): +def consume_direct(pytest_args, help_flag, pytest_help_flag): """ - Entry point for the consume command. + Clients consume directly via the `blocktest` interface. """ args = handle_help_flags(pytest_args, help_flag, pytest_help_flag) - args += ["-c", "pytest-consume.ini"] + args += ["-c", "pytest-consume-direct.ini"] pytest.main(args) + + +@click.command(context_settings=dict(ignore_unknown_options=True)) +@common_options +def consume_via_rlp(pytest_args, help_flag, pytest_help_flag): + """ + Clients consume RLP-encoded blocks on startup. + """ + args = handle_help_flags(pytest_args, help_flag, pytest_help_flag) + args += ["-c", "pytest-consume-via-rlp.ini"] + pytest.main(args) + + +@click.command(context_settings=dict(ignore_unknown_options=True)) +@common_options +def consume_via_engine_api(pytest_args, help_flag, pytest_help_flag): + """ + Clients consume via the Engine API. + """ + args = handle_help_flags(pytest_args, help_flag, pytest_help_flag) + args += ["-c", "pytest-consume-via-engine-api.ini"] + args += get_hive_flags_from_env() + raise NotImplementedError("Consume via Engine API simulator is not implemented yet.") + # pytest.main(args) + + +consume.add_command(consume_direct, name="direct") +consume.add_command(consume_via_rlp, name="rlp") +consume.add_command(consume_via_engine_api, name="engine") diff --git a/src/ethereum_test_tools/common/types.py b/src/ethereum_test_tools/common/types.py index 344b9be37e1..f9ab5766bd5 100644 --- a/src/ethereum_test_tools/common/types.py +++ b/src/ethereum_test_tools/common/types.py @@ -45,6 +45,33 @@ from .json import JSONEncoder, SupportsJSON, field, to_json +def load_dataclass_from_json(dataclass_type, json_as_dict: dict): + """ + Loads a dataclass from a JSON object. This could be as simple as, for example, + ``` + fixture = Fixture(**json_as_dict) + ``` + but as we name our dataclass fields differently than those we write to json, + we need to do a bit more work. + """ + init_args = {} + for dataclass_field in fields(dataclass_type): + # Retrieve the JSONEncoder.Field instance from metadata + json_encoder_field = dataclass_field.metadata.get("json_encoder") + + if json_encoder_field is None or json_encoder_field.skip: + continue + + json_key = json_encoder_field.name or dataclass_field.name + if json_key in json_as_dict: + value = json_as_dict[json_key] + if json_encoder_field.cast_type: + value = json_encoder_field.cast_type(value) + init_args[dataclass_field.name] = value + + return dataclass_type(**init_args) + + # Sentinel classes class Removable: """ diff --git a/src/pytest_plugins/consume/consume.py b/src/pytest_plugins/consume/consume.py new file mode 100644 index 00000000000..8bbf5f5456c --- /dev/null +++ b/src/pytest_plugins/consume/consume.py @@ -0,0 +1,87 @@ +""" +A pytest plugin providing common functionality for consuming test fixtures. +""" +import tarfile +from pathlib import Path +from urllib.parse import urlparse + +import pytest +import requests + +cached_downloads_directory = Path("./cached_downloads") + + +def is_url(string: str) -> bool: + """ + Check if a string is a remote URL. + """ + result = urlparse(string) + return all([result.scheme, result.netloc]) + + +def download_and_extract(url: str, base_directory: Path) -> Path: + """ + Download the URL and extract it locally if it hasn't already been downloaded. + """ + parsed_url = urlparse(url) + # Extract filename and version from URL + filename = Path(parsed_url.path).name + version = Path(parsed_url.path).parts[-2] + + # Create unique directory path for this version + extract_to = base_directory / version / filename.removesuffix(".tar.gz") + + if extract_to.exists(): + return extract_to + + extract_to.mkdir(parents=True, exist_ok=False) + + # Download and extract the archive + response = requests.get(url) + response.raise_for_status() + + archive_path = extract_to / filename + with open(archive_path, "wb") as file: + file.write(response.content) + + with tarfile.open(archive_path, "r:gz") as tar: + tar.extractall(path=extract_to) + + return extract_to + + +def pytest_addoption(parser): # noqa: D103 + consume_group = parser.getgroup( + "consume", "Arguments related to consuming fixtures via a client" + ) + consume_group.addoption( + "--input", + action="store", + dest="fixture_directory", + default="fixtures", + help="A URL or local directory specifying the JSON test fixtures. Default: './fixtures'.", + ) + + +def pytest_configure(config): # noqa: D103 + input_source = config.getoption("fixture_directory") + download_directory = cached_downloads_directory + + if is_url(input_source): + download_directory.mkdir(parents=True, exist_ok=True) + input_source = download_and_extract(input_source, download_directory) + + input_source = Path(input_source) + if not input_source.exists(): + pytest.exit(f"Specified fixture directory '{input_source}' does not exist.") + if not any(input_source.glob("**/*.json")): + pytest.exit( + f"Specified fixture directory '{input_source}' does not contain any JSON files." + ) + + config.option.fixture_directory = input_source + + +def pytest_report_header(config): # noqa: D103 + input_source = config.getoption("fixture_directory") + return f"fixtures: {input_source}" diff --git a/src/pytest_plugins/fixture_consumer/fixture_consumer.py b/src/pytest_plugins/consume_direct/consume_direct.py similarity index 83% rename from src/pytest_plugins/fixture_consumer/fixture_consumer.py rename to src/pytest_plugins/consume_direct/consume_direct.py index 8b85bb894fb..7b9efdeb6a5 100644 --- a/src/pytest_plugins/fixture_consumer/fixture_consumer.py +++ b/src/pytest_plugins/consume_direct/consume_direct.py @@ -12,7 +12,7 @@ @dataclass -class FixtureData: # noqa: D101 +class TestCase: # noqa: D101 fixture_name: str fixture_format: FixtureFormats json_file_path: Path @@ -22,13 +22,7 @@ def pytest_addoption(parser): # noqa: D103 consume_group = parser.getgroup( "consume", "Arguments related to consuming fixtures via a client" ) - consume_group.addoption( - "--fixture-directory", - type=Path, - action="store", - default="fixtures", - help="Specify the fixture directory to execute tests on. Default: 'fixtures'.", - ) + consume_group.addoption( "--evm-bin", action="store", @@ -108,7 +102,7 @@ def test_dump_dir( @pytest.fixture(scope="function") -def json_fixture_path(fixture_data: FixtureData): +def json_fixture_path(fixture_data: TestCase): """ Provide the path to the current JSON fixture file. """ @@ -116,7 +110,7 @@ def json_fixture_path(fixture_data: FixtureData): @pytest.fixture(scope="function") -def fixture_format(fixture_data: FixtureData): +def fixture_format(fixture_data: TestCase): """ The format of the current fixture. """ @@ -124,7 +118,7 @@ def fixture_format(fixture_data: FixtureData): @pytest.fixture(scope="function") -def fixture_name(fixture_data: FixtureData): +def fixture_name(fixture_data: TestCase): """ The name of the current fixture. """ @@ -140,21 +134,21 @@ def pytest_generate_tests(metafunc): fixtures_directory = metafunc.config.getoption("fixture_directory") fixture_data = [] - fixture_ids = [] + test_case_ids = [] for json_file in fixtures_directory.glob("**/*.json"): with json_file.open() as f: data = json.load(f) if metafunc.config.evm_use_single_test: for fixture_name in data.keys(): fixture_data.append( - FixtureData(fixture_name, FixtureFormats.BLOCKCHAIN_TEST, json_file) + TestCase(fixture_name, FixtureFormats.BLOCKCHAIN_TEST, json_file) ) - fixture_ids.append(f"{json_file.name}_{fixture_name}") + test_case_ids.append(f"{json_file.name}_{fixture_name}") else: # evm bin does not support --single-test fixture_data.append( - FixtureData(json_file.name, FixtureFormats.BLOCKCHAIN_TEST, json_file) + TestCase(json_file.name, FixtureFormats.BLOCKCHAIN_TEST, json_file) ) - fixture_ids.append(f"{json_file.name}") + test_case_ids.append(f"{json_file.name}") - metafunc.parametrize("fixture_data", fixture_data, ids=fixture_ids) + metafunc.parametrize("fixture_data", fixture_data, ids=test_case_ids) diff --git a/src/pytest_plugins/consume_via_engine_api/consume_via_engine_api.py b/src/pytest_plugins/consume_via_engine_api/consume_via_engine_api.py new file mode 100644 index 00000000000..801d4cd12d7 --- /dev/null +++ b/src/pytest_plugins/consume_via_engine_api/consume_via_engine_api.py @@ -0,0 +1,5 @@ +""" +A hive simulator that feeds blocks from test fixtures to clients via the Engine API. + +Implemented using the pytest framework as a pytest plugin. +""" diff --git a/src/pytest_plugins/consume_via_rlp/__init__.py b/src/pytest_plugins/consume_via_rlp/__init__.py new file mode 100644 index 00000000000..e1c9121ef92 --- /dev/null +++ b/src/pytest_plugins/consume_via_rlp/__init__.py @@ -0,0 +1,4 @@ +""" +A hive simulator that feeds blocks defined in Blockchain tests to clients as +RLP upon start-up. +""" diff --git a/src/pytest_plugins/consume_via_rlp/consume_via_rlp.py b/src/pytest_plugins/consume_via_rlp/consume_via_rlp.py new file mode 100644 index 00000000000..5b8ecfc71a9 --- /dev/null +++ b/src/pytest_plugins/consume_via_rlp/consume_via_rlp.py @@ -0,0 +1,123 @@ +""" +A hive simulator that feeds test fixtures to clients as RLP-encoded blocks +upon start-up. + +Implemented using the pytest framework as a pytest plugin. +""" +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, List, Optional, Tuple + +import pytest + +from ethereum_test_tools.common.types import Fixture, load_dataclass_from_json + +from .network_ruleset_hive import ruleset + + +@pytest.fixture(scope="session") +def test_suite_name() -> str: + """ + The name of the hive test suite used in this simulator. + """ + return "EEST Consume Blocks via RLP" + + +@pytest.fixture(scope="session") +def test_suite_description() -> str: + """ + The description of the hive test suite used in this simulator. + """ + return "Execute blockchain tests by providing RLP-encoded blocks to a client upon start-up." + + +@dataclass +class TestCase: # noqa: D101 + """ + Define the test case data associated a JSON test fixture in blockchain test + format. + """ + + fixture_name: str + json_file_path: Path + json_as_dict: dict + fixture: Optional[Fixture] = None + marks: List[pytest.MarkDecorator] = field(default_factory=list) + __test__ = False # stop pytest from collecting this dataclass as a test + + def __post_init__(self): + """ + Sanity check the loaded test-case and add pytest marks. + + Marks can be applied based on any issues detected with the fixture. In + the future, we can apply marks that were written into the json fixture + file from `fill`. + """ + if any(mark is pytest.mark.xfail for mark in self.marks): + return # no point continuing + if not all("blockHeader" in block for block in self.fixture.blocks): + print("Skipping fixture with missing block header", self.fixture_name) + self.marks.append(pytest.mark.xfail(reason="Missing block header", run=False)) + if self.fixture.fork not in ruleset: + self.marks.append( + pytest.mark.xfail(reason=f"Unsupported network '{self.fixture.fork}'", run=False) + ) + + +def create_test_cases_from_json(json_file_path: Path) -> Tuple[List[Any], List[str]]: + """ + Extract blockchain test cases from a JSON fixture file. + """ + test_cases = [] + test_case_ids = [] + # TODO: Consider try-except block here + with open(json_file_path, "r") as file: + json_data = json.load(file) + + for fixture_name, fixture_data in json_data.items(): + fixture = None + marks = [] + + try: + # TODO: here we validate fixture.blocks, for example, but not nested fields. Can we? + # Or should we? (it'll be brittle). + fixture = load_dataclass_from_json(Fixture, fixture_data) + except Exception as e: + reason = f"Error creating test case {fixture_name} from {json_file_path}: {e}" + # TODO: Add logger.error() entry here + marks.append(pytest.mark.xfail(reason=reason, run=False)) + + test_case = TestCase( + json_file_path=json_file_path, + json_as_dict=fixture_data, + fixture_name=fixture_name, + fixture=fixture, + marks=marks, + ) + test_cases.append(pytest.param(test_case, marks=test_case.marks)) + + if "::.py" in fixture_name: # new format; fixture name if fill pytest node id + test_case_ids.append(str(fixture_name)) + else: # old format, pre v1.0.7 + test_case_ids.append(f"{json_file_path.name}_{str(fixture_name)}") + + return test_cases, test_case_ids + + +def pytest_generate_tests(metafunc): + """ + Generate test cases for every test fixture in all the JSON fixture files + within the specified fixtures directory. + """ + fixtures_directory = metafunc.config.getoption("fixture_directory") + test_cases: List[TestCase] = [] + test_case_ids: List[str] = [] + for json_file in fixtures_directory.glob("**/*.json"): + cases, ids = create_test_cases_from_json(json_file) + test_cases.extend(cases) + test_case_ids.extend(ids) + metafunc.parametrize("test_case", test_cases, ids=test_case_ids) + if "client_type" in metafunc.fixturenames: + client_ids = [client.name for client in metafunc.config.hive_execution_clients] + metafunc.parametrize("client_type", metafunc.config.hive_execution_clients, ids=client_ids) diff --git a/src/pytest_plugins/consume_via_rlp/network_ruleset_hive.py b/src/pytest_plugins/consume_via_rlp/network_ruleset_hive.py new file mode 100644 index 00000000000..2f3af7ff4cd --- /dev/null +++ b/src/pytest_plugins/consume_via_rlp/network_ruleset_hive.py @@ -0,0 +1,309 @@ +""" +Network/fork rules for Hive, taken verbatim from the consensus simulator. +""" + +ruleset = { + "Frontier": { + "HIVE_FORK_HOMESTEAD": 2000, + "HIVE_FORK_DAO_BLOCK": 2000, + "HIVE_FORK_TANGERINE": 2000, + "HIVE_FORK_SPURIOUS": 2000, + "HIVE_FORK_BYZANTIUM": 2000, + "HIVE_FORK_CONSTANTINOPLE": 2000, + "HIVE_FORK_PETERSBURG": 2000, + "HIVE_FORK_ISTANBUL": 2000, + "HIVE_FORK_BERLIN": 2000, + "HIVE_FORK_LONDON": 2000, + }, + "Homestead": { + "HIVE_FORK_HOMESTEAD": 0, + "HIVE_FORK_DAO_BLOCK": 2000, + "HIVE_FORK_TANGERINE": 2000, + "HIVE_FORK_SPURIOUS": 2000, + "HIVE_FORK_BYZANTIUM": 2000, + "HIVE_FORK_CONSTANTINOPLE": 2000, + "HIVE_FORK_PETERSBURG": 2000, + "HIVE_FORK_ISTANBUL": 2000, + "HIVE_FORK_BERLIN": 2000, + "HIVE_FORK_LONDON": 2000, + }, + "EIP150": { + "HIVE_FORK_HOMESTEAD": 0, + "HIVE_FORK_TANGERINE": 0, + "HIVE_FORK_SPURIOUS": 2000, + "HIVE_FORK_BYZANTIUM": 2000, + "HIVE_FORK_CONSTANTINOPLE": 2000, + "HIVE_FORK_PETERSBURG": 2000, + "HIVE_FORK_ISTANBUL": 2000, + "HIVE_FORK_BERLIN": 2000, + "HIVE_FORK_LONDON": 2000, + }, + "EIP158": { + "HIVE_FORK_HOMESTEAD": 0, + "HIVE_FORK_TANGERINE": 0, + "HIVE_FORK_SPURIOUS": 0, + "HIVE_FORK_BYZANTIUM": 2000, + "HIVE_FORK_CONSTANTINOPLE": 2000, + "HIVE_FORK_PETERSBURG": 2000, + "HIVE_FORK_ISTANBUL": 2000, + "HIVE_FORK_BERLIN": 2000, + "HIVE_FORK_LONDON": 2000, + }, + "Byzantium": { + "HIVE_FORK_HOMESTEAD": 0, + "HIVE_FORK_TANGERINE": 0, + "HIVE_FORK_SPURIOUS": 0, + "HIVE_FORK_BYZANTIUM": 0, + "HIVE_FORK_CONSTANTINOPLE": 2000, + "HIVE_FORK_PETERSBURG": 2000, + "HIVE_FORK_ISTANBUL": 2000, + "HIVE_FORK_BERLIN": 2000, + "HIVE_FORK_LONDON": 2000, + }, + "Constantinople": { + "HIVE_FORK_HOMESTEAD": 0, + "HIVE_FORK_TANGERINE": 0, + "HIVE_FORK_SPURIOUS": 0, + "HIVE_FORK_BYZANTIUM": 0, + "HIVE_FORK_CONSTANTINOPLE": 0, + "HIVE_FORK_PETERSBURG": 2000, + "HIVE_FORK_ISTANBUL": 2000, + "HIVE_FORK_BERLIN": 2000, + "HIVE_FORK_LONDON": 2000, + }, + "ConstantinopleFix": { + "HIVE_FORK_HOMESTEAD": 0, + "HIVE_FORK_TANGERINE": 0, + "HIVE_FORK_SPURIOUS": 0, + "HIVE_FORK_BYZANTIUM": 0, + "HIVE_FORK_CONSTANTINOPLE": 0, + "HIVE_FORK_PETERSBURG": 0, + "HIVE_FORK_ISTANBUL": 2000, + "HIVE_FORK_BERLIN": 2000, + "HIVE_FORK_LONDON": 2000, + }, + "Istanbul": { + "HIVE_FORK_HOMESTEAD": 0, + "HIVE_FORK_TANGERINE": 0, + "HIVE_FORK_SPURIOUS": 0, + "HIVE_FORK_BYZANTIUM": 0, + "HIVE_FORK_CONSTANTINOPLE": 0, + "HIVE_FORK_PETERSBURG": 0, + "HIVE_FORK_ISTANBUL": 0, + "HIVE_FORK_BERLIN": 2000, + "HIVE_FORK_LONDON": 2000, + }, + "Berlin": { + "HIVE_FORK_HOMESTEAD": 0, + "HIVE_FORK_TANGERINE": 0, + "HIVE_FORK_SPURIOUS": 0, + "HIVE_FORK_BYZANTIUM": 0, + "HIVE_FORK_CONSTANTINOPLE": 0, + "HIVE_FORK_PETERSBURG": 0, + "HIVE_FORK_ISTANBUL": 0, + "HIVE_FORK_BERLIN": 0, + "HIVE_FORK_LONDON": 2000, + }, + "FrontierToHomesteadAt5": { + "HIVE_FORK_HOMESTEAD": 5, + "HIVE_FORK_DAO_BLOCK": 2000, + "HIVE_FORK_TANGERINE": 2000, + "HIVE_FORK_SPURIOUS": 2000, + "HIVE_FORK_BYZANTIUM": 2000, + "HIVE_FORK_CONSTANTINOPLE": 2000, + "HIVE_FORK_PETERSBURG": 2000, + "HIVE_FORK_ISTANBUL": 2000, + "HIVE_FORK_BERLIN": 2000, + "HIVE_FORK_LONDON": 2000, + }, + "HomesteadToEIP150At5": { + "HIVE_FORK_HOMESTEAD": 0, + # "HIVE_FORK_DAO_BLOCK": 2000, + "HIVE_FORK_TANGERINE": 5, + "HIVE_FORK_SPURIOUS": 2000, + "HIVE_FORK_BYZANTIUM": 2000, + "HIVE_FORK_CONSTANTINOPLE": 2000, + "HIVE_FORK_PETERSBURG": 2000, + "HIVE_FORK_ISTANBUL": 2000, + "HIVE_FORK_BERLIN": 2000, + "HIVE_FORK_LONDON": 2000, + }, + "HomesteadToDaoAt5": { + "HIVE_FORK_HOMESTEAD": 0, + "HIVE_FORK_DAO_BLOCK": 5, + "HIVE_FORK_TANGERINE": 2000, + "HIVE_FORK_SPURIOUS": 2000, + "HIVE_FORK_BYZANTIUM": 2000, + "HIVE_FORK_CONSTANTINOPLE": 2000, + "HIVE_FORK_PETERSBURG": 2000, + "HIVE_FORK_ISTANBUL": 2000, + "HIVE_FORK_BERLIN": 2000, + "HIVE_FORK_LONDON": 2000, + }, + "EIP158ToByzantiumAt5": { + "HIVE_FORK_HOMESTEAD": 0, + # "HIVE_FORK_DAO_BLOCK": 2000, + "HIVE_FORK_TANGERINE": 0, + "HIVE_FORK_SPURIOUS": 0, + "HIVE_FORK_BYZANTIUM": 5, + "HIVE_FORK_CONSTANTINOPLE": 2000, + "HIVE_FORK_PETERSBURG": 2000, + "HIVE_FORK_ISTANBUL": 2000, + "HIVE_FORK_BERLIN": 2000, + "HIVE_FORK_LONDON": 2000, + }, + "ByzantiumToConstantinopleAt5": { + "HIVE_FORK_HOMESTEAD": 0, + # "HIVE_FORK_DAO_BLOCK": 2000, + "HIVE_FORK_TANGERINE": 0, + "HIVE_FORK_SPURIOUS": 0, + "HIVE_FORK_BYZANTIUM": 0, + "HIVE_FORK_CONSTANTINOPLE": 5, + "HIVE_FORK_PETERSBURG": 2000, + "HIVE_FORK_ISTANBUL": 2000, + "HIVE_FORK_BERLIN": 2000, + "HIVE_FORK_LONDON": 2000, + }, + "ByzantiumToConstantinopleFixAt5": { + "HIVE_FORK_HOMESTEAD": 0, + # "HIVE_FORK_DAO_BLOCK": 2000, + "HIVE_FORK_TANGERINE": 0, + "HIVE_FORK_SPURIOUS": 0, + "HIVE_FORK_BYZANTIUM": 0, + "HIVE_FORK_CONSTANTINOPLE": 5, + "HIVE_FORK_PETERSBURG": 5, + "HIVE_FORK_ISTANBUL": 2000, + "HIVE_FORK_BERLIN": 2000, + "HIVE_FORK_LONDON": 2000, + }, + "ConstantinopleFixToIstanbulAt5": { + "HIVE_FORK_HOMESTEAD": 0, + # "HIVE_FORK_DAO_BLOCK": 2000, + "HIVE_FORK_TANGERINE": 0, + "HIVE_FORK_SPURIOUS": 0, + "HIVE_FORK_BYZANTIUM": 0, + "HIVE_FORK_CONSTANTINOPLE": 0, + "HIVE_FORK_PETERSBURG": 0, + "HIVE_FORK_ISTANBUL": 5, + "HIVE_FORK_BERLIN": 2000, + "HIVE_FORK_LONDON": 2000, + }, + "IstanbulToBerlinAt5": { + "HIVE_FORK_HOMESTEAD": 0, + # "HIVE_FORK_DAO_BLOCK": 2000, + "HIVE_FORK_TANGERINE": 0, + "HIVE_FORK_SPURIOUS": 0, + "HIVE_FORK_BYZANTIUM": 0, + "HIVE_FORK_CONSTANTINOPLE": 0, + "HIVE_FORK_PETERSBURG": 0, + "HIVE_FORK_ISTANBUL": 0, + "HIVE_FORK_BERLIN": 5, + "HIVE_FORK_LONDON": 2000, + }, + "BerlinToLondonAt5": { + "HIVE_FORK_HOMESTEAD": 0, + # "HIVE_FORK_DAO_BLOCK": 2000, + "HIVE_FORK_TANGERINE": 0, + "HIVE_FORK_SPURIOUS": 0, + "HIVE_FORK_BYZANTIUM": 0, + "HIVE_FORK_CONSTANTINOPLE": 0, + "HIVE_FORK_PETERSBURG": 0, + "HIVE_FORK_ISTANBUL": 0, + "HIVE_FORK_BERLIN": 0, + "HIVE_FORK_LONDON": 5, + }, + "London": { + "HIVE_FORK_HOMESTEAD": 0, + "HIVE_FORK_TANGERINE": 0, + "HIVE_FORK_SPURIOUS": 0, + "HIVE_FORK_BYZANTIUM": 0, + "HIVE_FORK_CONSTANTINOPLE": 0, + "HIVE_FORK_PETERSBURG": 0, + "HIVE_FORK_ISTANBUL": 0, + "HIVE_FORK_BERLIN": 0, + "HIVE_FORK_LONDON": 0, + }, + "ArrowGlacierToMergeAtDiffC0000": { + "HIVE_FORK_HOMESTEAD": 0, + "HIVE_FORK_TANGERINE": 0, + "HIVE_FORK_SPURIOUS": 0, + "HIVE_FORK_BYZANTIUM": 0, + "HIVE_FORK_CONSTANTINOPLE": 0, + "HIVE_FORK_PETERSBURG": 0, + "HIVE_FORK_ISTANBUL": 0, + "HIVE_FORK_BERLIN": 0, + "HIVE_FORK_LONDON": 0, + "HIVE_TERMINAL_TOTAL_DIFFICULTY": 786432, + }, + "Merge": { + "HIVE_FORK_HOMESTEAD": 0, + "HIVE_FORK_TANGERINE": 0, + "HIVE_FORK_SPURIOUS": 0, + "HIVE_FORK_BYZANTIUM": 0, + "HIVE_FORK_CONSTANTINOPLE": 0, + "HIVE_FORK_PETERSBURG": 0, + "HIVE_FORK_ISTANBUL": 0, + "HIVE_FORK_BERLIN": 0, + "HIVE_FORK_LONDON": 0, + "HIVE_FORK_MERGE": 0, + "HIVE_TERMINAL_TOTAL_DIFFICULTY": 0, + }, + "Shanghai": { + "HIVE_FORK_HOMESTEAD": 0, + "HIVE_FORK_TANGERINE": 0, + "HIVE_FORK_SPURIOUS": 0, + "HIVE_FORK_BYZANTIUM": 0, + "HIVE_FORK_CONSTANTINOPLE": 0, + "HIVE_FORK_PETERSBURG": 0, + "HIVE_FORK_ISTANBUL": 0, + "HIVE_FORK_BERLIN": 0, + "HIVE_FORK_LONDON": 0, + "HIVE_FORK_MERGE": 0, + "HIVE_TERMINAL_TOTAL_DIFFICULTY": 0, + "HIVE_SHANGHAI_TIMESTAMP": 0, + }, + "MergeToShanghaiAtTime15k": { + "HIVE_FORK_HOMESTEAD": 0, + "HIVE_FORK_TANGERINE": 0, + "HIVE_FORK_SPURIOUS": 0, + "HIVE_FORK_BYZANTIUM": 0, + "HIVE_FORK_CONSTANTINOPLE": 0, + "HIVE_FORK_PETERSBURG": 0, + "HIVE_FORK_ISTANBUL": 0, + "HIVE_FORK_BERLIN": 0, + "HIVE_FORK_LONDON": 0, + "HIVE_FORK_MERGE": 0, + "HIVE_TERMINAL_TOTAL_DIFFICULTY": 0, + "HIVE_SHANGHAI_TIMESTAMP": 15000, + }, + "Cancun": { + "HIVE_FORK_HOMESTEAD": 0, + "HIVE_FORK_TANGERINE": 0, + "HIVE_FORK_SPURIOUS": 0, + "HIVE_FORK_BYZANTIUM": 0, + "HIVE_FORK_CONSTANTINOPLE": 0, + "HIVE_FORK_PETERSBURG": 0, + "HIVE_FORK_ISTANBUL": 0, + "HIVE_FORK_BERLIN": 0, + "HIVE_FORK_LONDON": 0, + "HIVE_FORK_MERGE": 0, + "HIVE_TERMINAL_TOTAL_DIFFICULTY": 0, + "HIVE_SHANGHAI_TIMESTAMP": 0, + "HIVE_CANCUN_TIMESTAMP": 0, + }, + "ShanghaiToCancunAtTime15k": { + "HIVE_FORK_HOMESTEAD": 0, + "HIVE_FORK_TANGERINE": 0, + "HIVE_FORK_SPURIOUS": 0, + "HIVE_FORK_BYZANTIUM": 0, + "HIVE_FORK_CONSTANTINOPLE": 0, + "HIVE_FORK_PETERSBURG": 0, + "HIVE_FORK_ISTANBUL": 0, + "HIVE_FORK_BERLIN": 0, + "HIVE_FORK_LONDON": 0, + "HIVE_FORK_MERGE": 0, + "HIVE_TERMINAL_TOTAL_DIFFICULTY": 0, + "HIVE_SHANGHAI_TIMESTAMP": 0, + "HIVE_CANCUN_TIMESTAMP": 15000, + }, +} diff --git a/src/pytest_plugins/pytest_hive/pytest_hive.py b/src/pytest_plugins/pytest_hive/pytest_hive.py new file mode 100644 index 00000000000..1e08cf50160 --- /dev/null +++ b/src/pytest_plugins/pytest_hive/pytest_hive.py @@ -0,0 +1,73 @@ +""" +A pytest plugin providing common functionality for Hive simulators. + +Simulators using this plugin must define two pytest fixtures: + +1. `test_suite_name`: The name of the test suite. +2. `test_suite_description`: The description of the test suite. + +These fixtures are used when creating the hive test suite. +""" +import os + +import pytest +from hive.client import ClientRole +from hive.simulation import Simulation + + +@pytest.fixture(scope="session") +def simulator(request): # noqa: D103 + return request.config.hive_simulator + + +@pytest.fixture(scope="session") +def test_suite(request, simulator: Simulation): + """ + Defines a Hive test suite and cleans up after all tests have run. + """ + try: + test_suite_name = request.getfixturevalue("test_suite_name") + test_suite_description = request.getfixturevalue("test_suite_description") + except pytest.FixtureLookupError: + pytest.exit( + "Error: The 'test_suite_name' and 'test_suite_description' fixtures are not defined " + "by the hive simulator pytest plugin using this ('test_suite') fixture!" + ) + + suite = simulator.start_suite(name=test_suite_name, description=test_suite_description) + # TODO: Can we share this fixture across all nodes using xdist? Hive uses different suites. + yield suite + suite.end() + + +def pytest_configure(config): # noqa: D103 + if config.option.collectonly: + return + hive_simulator_url = os.environ.get("HIVE_SIMULATOR") + if hive_simulator_url is None: + pytest.exit( + "The HIVE_SIMULATOR environment variable is not set.\n\n" + "If running locally, start hive in --dev mode, for example:\n" + "./hive --dev --client go-ethereum\n\n" + "and set the HIVE_SIMULATOR to the reported URL. For example, in bash:\n" + "export HIVE_SIMULATOR=http://127.0.0.1:3000\n" + "or in fish:\n" + "set -x HIVE_SIMULATOR http://127.0.0.1:3000" + ) + # TODO: Try and get these into fixtures; this is only here due to the "dynamic" parametrization + # of client_type with hive_execution_clients. + config.hive_simulator_url = hive_simulator_url + config.hive_simulator = Simulation(url=hive_simulator_url) + config.hive_execution_clients = config.hive_simulator.client_types( + role=ClientRole.ExecutionClient + ) + + +@pytest.hookimpl(trylast=True) +def pytest_report_header(config, start_path): + """ + Add lines to pytest's console output header. + """ + if config.option.collectonly: + return + return [f"hive simulator: {config.hive_simulator_url}"] diff --git a/src/pytest_plugins/test_filler/test_filler.py b/src/pytest_plugins/test_filler/test_filler.py index 7bca5ff2cb1..9356a03d2dc 100644 --- a/src/pytest_plugins/test_filler/test_filler.py +++ b/src/pytest_plugins/test_filler/test_filler.py @@ -254,7 +254,7 @@ def do_fixture_verification(request, t8n) -> bool: @pytest.fixture(autouse=True, scope="session") def evm_fixture_verification( request, do_fixture_verification: bool, evm_bin: Path, verify_fixtures_bin: Path -) -> Optional[Generator[TransitionTool, None, None]]: +) -> Generator[Optional[TransitionTool], None, None]: """ Returns the configured evm binary for executing statetest and blocktest commands used to verify generated JSON fixtures. diff --git a/src/pytest_plugins/test_help/test_help.py b/src/pytest_plugins/test_help/test_help.py index c6f5b8d30ea..a1318077837 100644 --- a/src/pytest_plugins/test_help/test_help.py +++ b/src/pytest_plugins/test_help/test_help.py @@ -51,7 +51,11 @@ def show_test_help(config): "filler location", "defining debug", ] - elif pytest_ini.name == "pytest-consume.ini": + elif pytest_ini.name in [ + "pytest-consume-direct.ini", + "pytest-consume-via-rlp.ini", + "pytest-consume-via-engine-api.ini", + ]: test_group_substrings = [ "execution-spec-tests", "consuming", diff --git a/tests/cancun/eip4788_beacon_root/conftest.py b/tests/cancun/eip4788_beacon_root/conftest.py index b6fb6d4255e..f8849e3c8de 100644 --- a/tests/cancun/eip4788_beacon_root/conftest.py +++ b/tests/cancun/eip4788_beacon_root/conftest.py @@ -100,7 +100,7 @@ def contract_call_account(call_type: Op, call_value: int, call_gas: int) -> Acco if call_type == Op.CALL or call_type == Op.CALLCODE: contract_call_code += Op.SSTORE( 0x00, # store the result of the contract call in storage[0] - call_type( + call_type( # type: ignore # https://github.com/ethereum/execution-spec-tests/issues/348 # noqa: E501 call_gas, Spec.BEACON_ROOTS_ADDRESS, call_value, @@ -114,7 +114,7 @@ def contract_call_account(call_type: Op, call_value: int, call_gas: int) -> Acco # delegatecall and staticcall use one less argument contract_call_code += Op.SSTORE( 0x00, - call_type( + call_type( # type: ignore # https://github.com/ethereum/execution-spec-tests/issues/348 # noqa: E501 call_gas, Spec.BEACON_ROOTS_ADDRESS, args_start, diff --git a/tests/cancun/eip4844_blobs/test_point_evaluation_precompile.py b/tests/cancun/eip4844_blobs/test_point_evaluation_precompile.py index 4da73cc9e98..b54baa86d7c 100644 --- a/tests/cancun/eip4844_blobs/test_point_evaluation_precompile.py +++ b/tests/cancun/eip4844_blobs/test_point_evaluation_precompile.py @@ -114,7 +114,7 @@ def precompile_caller_account(call_type: Op, call_gas: int) -> Account: if call_type == Op.CALL or call_type == Op.CALLCODE: precompile_caller_code += Op.SSTORE( 0, - call_type( + call_type( # type: ignore # https://github.com/ethereum/execution-spec-tests/issues/348 # noqa: E501 call_gas, Spec.POINT_EVALUATION_PRECOMPILE_ADDRESS, 0x00, @@ -128,7 +128,7 @@ def precompile_caller_account(call_type: Op, call_gas: int) -> Account: # Delegatecall and staticcall use one less argument precompile_caller_code += Op.SSTORE( 0, - call_type( + call_type( # type: ignore # https://github.com/ethereum/execution-spec-tests/issues/348 # noqa: E501 call_gas, Spec.POINT_EVALUATION_PRECOMPILE_ADDRESS, 0x00, diff --git a/tests/cancun/eip4844_blobs/test_point_evaluation_precompile_gas.py b/tests/cancun/eip4844_blobs/test_point_evaluation_precompile_gas.py index efd324b3ff4..57206b930c2 100644 --- a/tests/cancun/eip4844_blobs/test_point_evaluation_precompile_gas.py +++ b/tests/cancun/eip4844_blobs/test_point_evaluation_precompile_gas.py @@ -90,7 +90,7 @@ def precompile_caller_account( + copy_opcode_cost(len(precompile_input)) ) if call_type == Op.CALL or call_type == Op.CALLCODE: - precompile_caller_code += call_type( + precompile_caller_code += call_type( # type: ignore # https://github.com/ethereum/execution-spec-tests/issues/348 # noqa: E501 call_gas, Spec.POINT_EVALUATION_PRECOMPILE_ADDRESS, 0x00, @@ -102,7 +102,7 @@ def precompile_caller_account( overhead_cost += (PUSH_OPERATIONS_COST * 6) + (CALLDATASIZE_COST * 1) elif call_type == Op.DELEGATECALL or call_type == Op.STATICCALL: # Delegatecall and staticcall use one less argument - precompile_caller_code += call_type( + precompile_caller_code += call_type( # type: ignore # https://github.com/ethereum/execution-spec-tests/issues/348 # noqa: E501 call_gas, Spec.POINT_EVALUATION_PRECOMPILE_ADDRESS, 0x00, diff --git a/src/pytest_plugins/fixture_consumer/consume_fixtures.py b/tests_consume/test_direct.py similarity index 78% rename from src/pytest_plugins/fixture_consumer/consume_fixtures.py rename to tests_consume/test_direct.py index a70947ef6f5..a2cd98e63a8 100644 --- a/src/pytest_plugins/fixture_consumer/consume_fixtures.py +++ b/tests_consume/test_direct.py @@ -1,5 +1,6 @@ """ -Test module that defines a test to execute a fixture against an EVM blocktest-like command. +Executes a JSON test fixture directly against a client using a dedicated +client interface similar to geth's EVM 'blocktest' command. """ from pathlib import Path from typing import Optional diff --git a/tests_consume/test_via_engine_api.py b/tests_consume/test_via_engine_api.py new file mode 100644 index 00000000000..5fa9035a35d --- /dev/null +++ b/tests_consume/test_via_engine_api.py @@ -0,0 +1,2 @@ +def test_via_engine_api(): + pass diff --git a/tests_consume/test_via_rlp.py b/tests_consume/test_via_rlp.py new file mode 100644 index 00000000000..ce46b9be4fb --- /dev/null +++ b/tests_consume/test_via_rlp.py @@ -0,0 +1,244 @@ +""" +Test module that defines a test to execute a fixture against an EVM blocktest-like command. +""" +import json +import tempfile +from pathlib import Path +from typing import List, Literal, Union + +import pytest +import requests +from hive.client import Client +from hive.testing import HiveTest, HiveTestResult, HiveTestSuite +from tenacity import retry, stop_after_attempt, wait_exponential + +from ethereum_test_tools.common.types import Fixture +from pytest_plugins.consume_via_rlp.consume_via_rlp import TestCase +from pytest_plugins.consume_via_rlp.network_ruleset_hive import ruleset + + +@pytest.fixture(scope="function") +def temp_dir() -> tempfile.TemporaryDirectory: + """ + Return a temporary directory to write the genesis.json and block RLP files to. + """ + return tempfile.TemporaryDirectory() + + +@pytest.fixture(scope="function") +def test_case_fixture(test_case: TestCase) -> Fixture: + """ + The test fixture as a dictionary. + """ + assert test_case.fixture is not None + return test_case.fixture + + +@pytest.fixture(scope="function") +def expected_hash(test_case_fixture: Fixture) -> str: + """ + The hash defined in the test fixture's last block header. + """ + return test_case_fixture.blocks[-1]["blockHeader"]["hash"] + + +@pytest.fixture(scope="function") +def expected_state_root(test_case: TestCase) -> str: + """ + The state root defined in the test fixture's last block header. + """ + return test_case.fixture.blocks[-1]["blockHeader"]["stateRoot"] + + +# @pytest.fixture +# def execution_client(request): +# execution_clients = request.cw +# # assert len(execution_client) == 1 +# return execution_clients[0] + + +@pytest.fixture +def hive_test(request, test_suite: HiveTestSuite): + test_parameter_string = request.node.nodeid.split("[")[-1].rstrip("]") + test: HiveTest = test_suite.start_test( + name=test_parameter_string, description="TODO: This should come from the '_info' field." + ) + yield test + test_result: HiveTestResult + try: + if hasattr(request.node, "rep_call"): + test_passed = request.node.rep_call.passed + else: + test_passed = False + test_result_details = "All good." if test_passed else "Oops, test failed." + test_result = HiveTestResult(test_pass=test_passed, details=test_result_details) + except Exception as e: + test_result = HiveTestResult(test_pass=False, details=str(e)) + test.end(result=test_result) + + +@pytest.fixture(scope="session") +def network(test_suite): + return test_suite.create_network("execution_client_network") + + +@pytest.fixture(scope="function") +def blocks_rlp(test_case_fixture: Fixture) -> List[str]: + """ + A list of RLP-encoded blocks for the current test fixture. + """ + return [block["rlp"] for block in test_case_fixture.blocks] + + +@pytest.fixture(scope="function") +def to_geth_genesis(test_case: TestCase, test_case_fixture: Fixture): + """ + Convert the genesis block header of the current test fixture to a geth genesis block. + """ + # TODO: Ask Martin why we can't just use the genesis block header as-is. + geth_genesis = { + "nonce": test_case_fixture.genesis["nonce"], + "timestamp": test_case_fixture.genesis["timestamp"], + "extraData": test_case_fixture.genesis["extraData"], + "gasLimit": test_case_fixture.genesis["gasLimit"], + "difficulty": test_case_fixture.genesis["difficulty"], + "mixhash": test_case_fixture.genesis["mixHash"], + "coinbase": test_case_fixture.genesis["coinbase"], + # TODO: retrieve pre_state from the fixture? Instead of the json + # (and potentially remove the json_as_dict field completely from TestCase) + "alloc": test_case.json_as_dict["pre"], + } + # TODO: Use ethereum_test_forks to detect new fields automatically? + for field in ["baseFeePerGas", "withdrawalsRoot", "blobFeePerGas", "blobGasUsed"]: + if field in test_case_fixture.genesis: + geth_genesis[field] = test_case_fixture.genesis[field] + return geth_genesis + + +@pytest.fixture +def genesis_file(to_geth_genesis: dict, temp_dir: tempfile.TemporaryDirectory) -> Path: + genesis_file = Path(temp_dir.name) / "genesis.json" + with open(genesis_file, "w") as f: + f.write(json.dumps(to_geth_genesis)) + return genesis_file + + +@pytest.fixture +def block_rlp_files(temp_dir, blocks_rlp, start=1) -> List[Path]: + block_rlp_files = [] + for i, block_rlp in enumerate(blocks_rlp): + blocks_rlp_file = Path(temp_dir.name) / f"{i:04d}.rlp" + with open(blocks_rlp_file, "wb") as f: + f.write(bytes.fromhex(block_rlp[2:])) + block_rlp_files.append(blocks_rlp_file) + yield block_rlp_files + + +@pytest.fixture +def environment(test_case: TestCase) -> dict: + env = { + "HIVE_FORK_DAO_VOTE": "1", + "HIVE_CHAIN_ID": "1", + } + assert test_case.fixture.fork in ruleset, "Oops, should never get here" + for k, v in ruleset[test_case.fixture.fork].items(): + env[k] = f"{v:d}" + if test_case.fixture.seal_engine == "NoProof": + env["HIVE_SKIP_POW"] = "1" + return env + + +@pytest.fixture +def files(test_case: TestCase, genesis_file: Path, block_rlp_files: list[Path]): + """ + Define the files that will be sent to the client container upon initializing + the client. + + The files are specified as a dictionary whose: + - Keys are the target file paths in the client's docker container, and, + - Values are the source file paths in the simulator container, respectively + the host (if hive is running in --dev mode). + """ + target_block_files = [Path(f"/blocks/{file.name}") for file in block_rlp_files] + target_genesis_file = Path("/genesis.json") + files = { + str(target_file): str(source_file) + for target_file, source_file in zip(target_block_files, block_rlp_files) + } + files[str(target_genesis_file)] = str(genesis_file) + return files + + +@pytest.fixture(scope="function") +def client(hive_test: HiveTest, files: dict, environment: dict, network, client_type) -> Client: + client = hive_test.start_client(client_type=client_type, environment=environment, files=files) + assert client is not None + network.connect_client(client) + yield client + network.disconnect_client(client) + client.stop() + + +BlockNumberType = Union[int, Literal["latest", "earliest", "pending"]] + + +@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, max=10)) +def get_block(client: Client, block_number: BlockNumberType) -> dict: + """ + Retrieve the i-th block from the client using the JSON-RPC API. + Retries up to two times (three attempts total) in case of an error or a timeout, + with exponential backoff. + """ + if isinstance(block_number, int): + block_number = hex(block_number) + url = f"http://{client.ip}:8545" + payload = { + "jsonrpc": "2.0", + "method": "eth_getBlockByNumber", + "params": [block_number, False], + "id": 1, + } + headers = {"Content-Type": "application/json"} + + response = requests.post(url, data=json.dumps(payload), headers=headers) + response.raise_for_status() + result = response.json().get("result") + + if result is None or "error" in result: + error_info = "result is None; and therefore contains no error info" + error_code = None + if result is not None: + error_info = result["error"] + error_code = error_info["code"] + raise Exception( + f"Error calling JSON RPC eth_getBlockByNumber, code: {error_code}, " + f"message: {error_info}" + ) + + return result + + +def test_via_rlp( + client: Client, + test_case: TestCase, + expected_hash: str, + expected_state_root: str, +): + """ + Verify that the client's state as calculated from the specified genesis state + and blocks matches those defined in the test fixture. + + Test: + + 1. The client's genesis block hash matches that of the fixture. + 2. The client's last block's hash and stateRoot` match those of the fixture. + """ + genesis_block = get_block(client, 0) + assert genesis_block["hash"] == test_case.fixture.genesis["hash"], "genesis hash mismatch" + + block = get_block(client, "latest") + assert block["number"] == hex(len(test_case.fixture.blocks)), "unexpected latest block number" + # print("\n got state root", block["stateRoot"], "hash", block["hash"]) + # print("expected state root", expected_state_root, "hash", expected_hash) + assert block["stateRoot"] == expected_state_root, "state root mismatch in last block" + assert block["hash"] == expected_hash, "hash mismatch in last block" diff --git a/whitelist.txt b/whitelist.txt index e7140b1d55f..fb404aff9c3 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -69,6 +69,7 @@ dirname discordapp docstrings dup +EEST eip eips EIPs @@ -154,7 +155,7 @@ marioevz markdownlint md metaclass -Misspelled words: +mixhash mkdocs mkdocstrings mypy @@ -210,6 +211,7 @@ repos returndatacopy returndatasize rlp +ruleset runtime sandboxed secp256k1 @@ -299,9 +301,11 @@ copytree dedent dest exc +extractall fixturenames fspath funcargs +getfixturevalue getgroup getoption groupby @@ -324,11 +328,13 @@ params parametrize parametrizer parametrizers +parametrization popen pytester pytestmark readline regexes +removesuffix reportinfo ret rglob @@ -508,4 +514,8 @@ deployer fi url -gz \ No newline at end of file +gz +tT +blobgasfee +istanbul +berlin \ No newline at end of file From 74efd5e9a2aa14ccd7afc0d85550e5695400d344 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Sat, 25 Nov 2023 23:47:34 +0100 Subject: [PATCH 12/34] chore: provide entry_points as a package --- setup.cfg | 1 + src/entry_points/__init__.py | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 src/entry_points/__init__.py diff --git a/setup.cfg b/setup.cfg index 9aed2df0fee..2f6143f9917 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,6 +16,7 @@ packages = ethereum_test_forks ethereum_test_tools pytest_plugins + entry_points package_dir = =src diff --git a/src/entry_points/__init__.py b/src/entry_points/__init__.py new file mode 100644 index 00000000000..38706e09b66 --- /dev/null +++ b/src/entry_points/__init__.py @@ -0,0 +1,3 @@ +""" +Various entry points. +""" From ddc5a5beb020661dcc8f74ba14bc2dca43eb2122 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Sun, 26 Nov 2023 00:02:39 +0100 Subject: [PATCH 13/34] Revert "chore: provide entry_points as a package" This reverts commit 74efd5e9a2aa14ccd7afc0d85550e5695400d344. --- setup.cfg | 1 - src/entry_points/__init__.py | 3 --- 2 files changed, 4 deletions(-) delete mode 100644 src/entry_points/__init__.py diff --git a/setup.cfg b/setup.cfg index 2f6143f9917..9aed2df0fee 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,7 +16,6 @@ packages = ethereum_test_forks ethereum_test_tools pytest_plugins - entry_points package_dir = =src diff --git a/src/entry_points/__init__.py b/src/entry_points/__init__.py deleted file mode 100644 index 38706e09b66..00000000000 --- a/src/entry_points/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -""" -Various entry points. -""" From 09a27f6267d142c07dc1831eb645fca2888ec717 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Wed, 29 Nov 2023 14:35:08 +0100 Subject: [PATCH 14/34] chore: change hive.py package source to marioevz main --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 9aed2df0fee..dcbec2b186a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,8 +24,8 @@ python_requires = >=3.10 install_requires = click>=8.1.0,<9 - ethereum@git+https://github.com/ethereum/execution-specs.git - hive.py@git+https://github.com/danceratopz/hive.py@fix/client/files-def-in-post-with-files + ethereum@git+https://github.com/ethereum/execution-specs + hive.py@git+https://github.com/marioevz/hive.py setuptools types-setuptools requests>=2.31.0 From 30e2242c903bdf6510272177be25d71865802816 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Wed, 29 Nov 2023 15:53:37 +0100 Subject: [PATCH 15/34] fix: fix test result propagation to hive server; move to hive plugin --- src/pytest_plugins/pytest_hive/pytest_hive.py | 51 +++++++++++++++++++ tests_consume/test_via_rlp.py | 29 +---------- 2 files changed, 52 insertions(+), 28 deletions(-) diff --git a/src/pytest_plugins/pytest_hive/pytest_hive.py b/src/pytest_plugins/pytest_hive/pytest_hive.py index 1e08cf50160..e4af5ad7596 100644 --- a/src/pytest_plugins/pytest_hive/pytest_hive.py +++ b/src/pytest_plugins/pytest_hive/pytest_hive.py @@ -13,6 +13,7 @@ import pytest from hive.client import ClientRole from hive.simulation import Simulation +from hive.testing import HiveTest, HiveTestResult, HiveTestSuite @pytest.fixture(scope="session") @@ -71,3 +72,53 @@ def pytest_report_header(config, start_path): if config.option.collectonly: return return [f"hive simulator: {config.hive_simulator_url}"] + + +@pytest.hookimpl(tryfirst=True, hookwrapper=True) +def pytest_runtest_makereport(item, call): + """ + Make the setup, call, and teardown results available in the teardown phase of + a test fixture (i.e., after yield has been called). + + This is used to get the test result and pass it to the hive test suite. + + Available as: + - result_setup - setup result + - result_call - test result + - result_teardown - teardown result + """ + outcome = yield + rep = outcome.get_result() + setattr(item, f"result_{rep.when}", rep) + + +@pytest.fixture +def hive_test(request, test_suite: HiveTestSuite): + """ + Propagate the pytest test case and its result to the hive server. + """ + test_parameter_string = request.node.nodeid.split("[")[-1].rstrip("]") # test fixture name + test: HiveTest = test_suite.start_test( + # TODO: pass test case documentation when available + name=test_parameter_string, + description="TODO: This should come from the '_info' field.", + ) + yield test + try: + # TODO: Handle xfail/skip, does this work with run=False? + if hasattr(request.node, "result_call") and request.node.result_call.passed: + test_passed = True + test_result_details = "Test passed." + elif hasattr(request.node, "result_call") and not request.node.result_call.passed: + test_passed = False + test_result_details = request.node.result_call.longreprtext + elif hasattr(request.node, "result_setup") and not request.node.result_setup.passed: + test_passed = False + test_result_details = "Test setup failed.\n" + request.node.result_call.longreprtext + else: + test_passed = False + test_result_details = "Test failed for unknown reason (setup or call status unknown)." + except Exception as e: + test_passed = False + test_result_details = f"Exception whilst processing test result: {str(e)}" + test.end(result=HiveTestResult(test_pass=test_passed, details=test_result_details)) diff --git a/tests_consume/test_via_rlp.py b/tests_consume/test_via_rlp.py index ce46b9be4fb..a3fa63b9b1e 100644 --- a/tests_consume/test_via_rlp.py +++ b/tests_consume/test_via_rlp.py @@ -9,7 +9,7 @@ import pytest import requests from hive.client import Client -from hive.testing import HiveTest, HiveTestResult, HiveTestSuite +from hive.testing import HiveTest from tenacity import retry, stop_after_attempt, wait_exponential from ethereum_test_tools.common.types import Fixture @@ -50,33 +50,6 @@ def expected_state_root(test_case: TestCase) -> str: return test_case.fixture.blocks[-1]["blockHeader"]["stateRoot"] -# @pytest.fixture -# def execution_client(request): -# execution_clients = request.cw -# # assert len(execution_client) == 1 -# return execution_clients[0] - - -@pytest.fixture -def hive_test(request, test_suite: HiveTestSuite): - test_parameter_string = request.node.nodeid.split("[")[-1].rstrip("]") - test: HiveTest = test_suite.start_test( - name=test_parameter_string, description="TODO: This should come from the '_info' field." - ) - yield test - test_result: HiveTestResult - try: - if hasattr(request.node, "rep_call"): - test_passed = request.node.rep_call.passed - else: - test_passed = False - test_result_details = "All good." if test_passed else "Oops, test failed." - test_result = HiveTestResult(test_pass=test_passed, details=test_result_details) - except Exception as e: - test_result = HiveTestResult(test_pass=False, details=str(e)) - test.end(result=test_result) - - @pytest.fixture(scope="session") def network(test_suite): return test_suite.create_network("execution_client_network") From e2daceb134d94aac843899135c71aa6819467ee8 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Wed, 29 Nov 2023 15:58:14 +0100 Subject: [PATCH 16/34] chore: update whitelist for pytest keywords --- whitelist.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/whitelist.txt b/whitelist.txt index 2cf3176adf3..fca03d7fd9c 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -317,7 +317,9 @@ IGNORECASE inifile iterdir ljust +longreprtext makepyfile +makereport metafunc modifyitems nodeid From 3766bf12ee7e492f8ba2f348e32f63b1bfd7658d Mon Sep 17 00:00:00 2001 From: danceratopz Date: Thu, 30 Nov 2023 12:12:16 +0100 Subject: [PATCH 17/34] docs: fix consume rlp module docstring --- tests_consume/test_via_rlp.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests_consume/test_via_rlp.py b/tests_consume/test_via_rlp.py index a3fa63b9b1e..5a4bb36cba6 100644 --- a/tests_consume/test_via_rlp.py +++ b/tests_consume/test_via_rlp.py @@ -1,5 +1,13 @@ """ -Test module that defines a test to execute a fixture against an EVM blocktest-like command. +Test module to test clients using RLP-encoded blocks from blockchain tests. + +The test fixtures should have the blockchain test format. The setup sends +the genesis file and RLP-encoded blocks to the client container using hive. +The client consumes these files upon start-up. + +The test verifies: +1. The client's genesis block hash matches that of the fixture. +2. The client's last block's hash and stateRoot` match those of the fixture. """ import json import tempfile From 36d1aa36d61a42556e302f8f0b4bb235517d104c Mon Sep 17 00:00:00 2001 From: danceratopz Date: Thu, 30 Nov 2023 12:20:25 +0100 Subject: [PATCH 18/34] feat: sepcify genesis & block rlp files BufferedReader objects --- tests_consume/test_via_rlp.py | 79 +++++++++++++++-------------------- 1 file changed, 33 insertions(+), 46 deletions(-) diff --git a/tests_consume/test_via_rlp.py b/tests_consume/test_via_rlp.py index 5a4bb36cba6..65196891afe 100644 --- a/tests_consume/test_via_rlp.py +++ b/tests_consume/test_via_rlp.py @@ -9,10 +9,9 @@ 1. The client's genesis block hash matches that of the fixture. 2. The client's last block's hash and stateRoot` match those of the fixture. """ +import io import json -import tempfile -from pathlib import Path -from typing import List, Literal, Union +from typing import List, Literal, Mapping, Union import pytest import requests @@ -25,18 +24,12 @@ from pytest_plugins.consume_via_rlp.network_ruleset_hive import ruleset -@pytest.fixture(scope="function") -def temp_dir() -> tempfile.TemporaryDirectory: - """ - Return a temporary directory to write the genesis.json and block RLP files to. - """ - return tempfile.TemporaryDirectory() - - @pytest.fixture(scope="function") def test_case_fixture(test_case: TestCase) -> Fixture: """ The test fixture as a dictionary. + + If we failed to parse a test case fixture, it's None: We xfail/skip the test. """ assert test_case.fixture is not None return test_case.fixture @@ -66,17 +59,16 @@ def network(test_suite): @pytest.fixture(scope="function") def blocks_rlp(test_case_fixture: Fixture) -> List[str]: """ - A list of RLP-encoded blocks for the current test fixture. + A list of RLP-encoded blocks for the current json test fixture. """ return [block["rlp"] for block in test_case_fixture.blocks] @pytest.fixture(scope="function") -def to_geth_genesis(test_case: TestCase, test_case_fixture: Fixture): +def to_geth_genesis(test_case: TestCase, test_case_fixture: Fixture) -> dict: """ Convert the genesis block header of the current test fixture to a geth genesis block. """ - # TODO: Ask Martin why we can't just use the genesis block header as-is. geth_genesis = { "nonce": test_case_fixture.genesis["nonce"], "timestamp": test_case_fixture.genesis["timestamp"], @@ -97,22 +89,38 @@ def to_geth_genesis(test_case: TestCase, test_case_fixture: Fixture): @pytest.fixture -def genesis_file(to_geth_genesis: dict, temp_dir: tempfile.TemporaryDirectory) -> Path: - genesis_file = Path(temp_dir.name) / "genesis.json" - with open(genesis_file, "w") as f: - f.write(json.dumps(to_geth_genesis)) - return genesis_file +def buffered_genesis(to_geth_genesis: dict) -> io.BufferedReader: + genesis_json = json.dumps(to_geth_genesis) + genesis_bytes = genesis_json.encode("utf-8") + return io.BufferedReader(io.BytesIO(genesis_bytes)) @pytest.fixture -def block_rlp_files(temp_dir, blocks_rlp, start=1) -> List[Path]: +def buffered_blocks_rlp(blocks_rlp: List[str], start=1) -> List[io.BufferedReader]: block_rlp_files = [] for i, block_rlp in enumerate(blocks_rlp): - blocks_rlp_file = Path(temp_dir.name) / f"{i:04d}.rlp" - with open(blocks_rlp_file, "wb") as f: - f.write(bytes.fromhex(block_rlp[2:])) - block_rlp_files.append(blocks_rlp_file) - yield block_rlp_files + blocks_rlp_bytes = bytes.fromhex(block_rlp[2:]) + blocks_rlp_stream = io.BytesIO(blocks_rlp_bytes) + block_rlp_files.append(io.BufferedReader(blocks_rlp_stream)) + return block_rlp_files + + +@pytest.fixture +def files( + test_case: TestCase, + buffered_genesis: io.BufferedReader, + buffered_blocks_rlp: list[io.BufferedReader], +) -> Mapping[str, io.BufferedReader]: + """ + Define the files that hive will start the client with. + + The files are specified as a dictionary whose: + - Keys are the target file paths in the client's docker container, and, + - Values are in-memory buffered file objects. + """ + files = {f"/blocks/{i:04d}.rlp": block_rlp for i, block_rlp in enumerate(buffered_blocks_rlp)} + files["/genesis.json"] = buffered_genesis + return files @pytest.fixture @@ -129,27 +137,6 @@ def environment(test_case: TestCase) -> dict: return env -@pytest.fixture -def files(test_case: TestCase, genesis_file: Path, block_rlp_files: list[Path]): - """ - Define the files that will be sent to the client container upon initializing - the client. - - The files are specified as a dictionary whose: - - Keys are the target file paths in the client's docker container, and, - - Values are the source file paths in the simulator container, respectively - the host (if hive is running in --dev mode). - """ - target_block_files = [Path(f"/blocks/{file.name}") for file in block_rlp_files] - target_genesis_file = Path("/genesis.json") - files = { - str(target_file): str(source_file) - for target_file, source_file in zip(target_block_files, block_rlp_files) - } - files[str(target_genesis_file)] = str(genesis_file) - return files - - @pytest.fixture(scope="function") def client(hive_test: HiveTest, files: dict, environment: dict, network, client_type) -> Client: client = hive_test.start_client(client_type=client_type, environment=environment, files=files) From d18b0261c187b734ff54c0c4e3bf872f06bd4965 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Thu, 30 Nov 2023 12:38:22 +0100 Subject: [PATCH 19/34] refactor(rlp): remove unnecessary network & client connection setup/teardown --- tests_consume/test_via_rlp.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/tests_consume/test_via_rlp.py b/tests_consume/test_via_rlp.py index 65196891afe..4b63a16f094 100644 --- a/tests_consume/test_via_rlp.py +++ b/tests_consume/test_via_rlp.py @@ -15,7 +15,7 @@ import pytest import requests -from hive.client import Client +from hive.client import Client, ClientType from hive.testing import HiveTest from tenacity import retry, stop_after_attempt, wait_exponential @@ -51,11 +51,6 @@ def expected_state_root(test_case: TestCase) -> str: return test_case.fixture.blocks[-1]["blockHeader"]["stateRoot"] -@pytest.fixture(scope="session") -def network(test_suite): - return test_suite.create_network("execution_client_network") - - @pytest.fixture(scope="function") def blocks_rlp(test_case_fixture: Fixture) -> List[str]: """ @@ -138,12 +133,10 @@ def environment(test_case: TestCase) -> dict: @pytest.fixture(scope="function") -def client(hive_test: HiveTest, files: dict, environment: dict, network, client_type) -> Client: +def client(hive_test: HiveTest, files: dict, environment: dict, client_type: ClientType) -> Client: client = hive_test.start_client(client_type=client_type, environment=environment, files=files) assert client is not None - network.connect_client(client) yield client - network.disconnect_client(client) client.stop() From 9f04df9dc34aca37e0d2b3e47c454c1f010b4846 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Thu, 30 Nov 2023 12:50:45 +0100 Subject: [PATCH 20/34] refactor(fw): move json fixture loader helper to common.json --- src/ethereum_test_tools/common/json.py | 27 +++++++++++++++++++ src/ethereum_test_tools/common/types.py | 27 ------------------- .../consume_via_rlp/consume_via_rlp.py | 3 ++- 3 files changed, 29 insertions(+), 28 deletions(-) diff --git a/src/ethereum_test_tools/common/json.py b/src/ethereum_test_tools/common/json.py index 9503bad5125..3ef74af3a1a 100644 --- a/src/ethereum_test_tools/common/json.py +++ b/src/ethereum_test_tools/common/json.py @@ -156,3 +156,30 @@ def to_json(input: Any) -> Dict[str, Any]: Converts a value to its json representation. """ return JSONEncoder().default(input) + + +def load_dataclass_from_json(dataclass_type, json_as_dict: dict): + """ + Loads a dataclass from a JSON object. This could be as simple as, for example, + ``` + fixture = Fixture(**json_as_dict) + ``` + but as we name our dataclass fields differently than those we write to json, + we need to do a bit more work. + """ + init_args = {} + for field in fields(dataclass_type): + # Retrieve the JSONEncoder.Field instance from metadata + json_encoder_field = field.metadata.get("json_encoder") + + if json_encoder_field is None or json_encoder_field.skip: + continue + + json_key = json_encoder_field.name or field.name + if json_key in json_as_dict: + value = json_as_dict[json_key] + if json_encoder_field.cast_type: + value = json_encoder_field.cast_type(value) + init_args[field.name] = value + + return dataclass_type(**init_args) diff --git a/src/ethereum_test_tools/common/types.py b/src/ethereum_test_tools/common/types.py index f9ab5766bd5..344b9be37e1 100644 --- a/src/ethereum_test_tools/common/types.py +++ b/src/ethereum_test_tools/common/types.py @@ -45,33 +45,6 @@ from .json import JSONEncoder, SupportsJSON, field, to_json -def load_dataclass_from_json(dataclass_type, json_as_dict: dict): - """ - Loads a dataclass from a JSON object. This could be as simple as, for example, - ``` - fixture = Fixture(**json_as_dict) - ``` - but as we name our dataclass fields differently than those we write to json, - we need to do a bit more work. - """ - init_args = {} - for dataclass_field in fields(dataclass_type): - # Retrieve the JSONEncoder.Field instance from metadata - json_encoder_field = dataclass_field.metadata.get("json_encoder") - - if json_encoder_field is None or json_encoder_field.skip: - continue - - json_key = json_encoder_field.name or dataclass_field.name - if json_key in json_as_dict: - value = json_as_dict[json_key] - if json_encoder_field.cast_type: - value = json_encoder_field.cast_type(value) - init_args[dataclass_field.name] = value - - return dataclass_type(**init_args) - - # Sentinel classes class Removable: """ diff --git a/src/pytest_plugins/consume_via_rlp/consume_via_rlp.py b/src/pytest_plugins/consume_via_rlp/consume_via_rlp.py index 5b8ecfc71a9..424ba2a2b84 100644 --- a/src/pytest_plugins/consume_via_rlp/consume_via_rlp.py +++ b/src/pytest_plugins/consume_via_rlp/consume_via_rlp.py @@ -11,7 +11,8 @@ import pytest -from ethereum_test_tools.common.types import Fixture, load_dataclass_from_json +from ethereum_test_tools.common.json import load_dataclass_from_json +from ethereum_test_tools.common.types import Fixture from .network_ruleset_hive import ruleset From 520d161c6d517d40a8175ecf7d9907e4529c09fa Mon Sep 17 00:00:00 2001 From: danceratopz Date: Thu, 30 Nov 2023 13:11:46 +0100 Subject: [PATCH 21/34] docs: update changelog --- docs/CHANGELOG.md | 5 +++++ whitelist.txt | 1 + 2 files changed, 6 insertions(+) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f37279af44f..b58a7ae67df 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -10,6 +10,11 @@ Test fixtures for use by clients are available for each release on the [Github r ### 🛠️ Framework +- ✨ Adds two `consume` commands [#339](https://github.com/ethereum/execution-spec-tests/pull/339): + + 1. `consume direct` - Execute a test fixture directly against a client using a `blocktest`-like command (currently only geth supported). + 2. `consume rlp` - Execute a test fixture in a hive simulator against a client that imports the test's genesis config and blocks as RLP upon startup. This is a re-write of the [ethereum/consensus](https://github.com/ethereum/hive/tree/master/simulators/ethereum/consensus) Golang simulator. + - ✨ Add a `--single-fixture-per-file` flag to generate one fixture JSON file per test case ([#331](https://github.com/ethereum/execution-spec-tests/pull/331)). - 🔀 Rename test fixtures names to match the corresponding pytest node ID as generated using `fill` ([#342](https://github.com/ethereum/execution-spec-tests/pull/342)). - 💥 Replace "=" with "_" in pytest node ids and test fixture names ([#342](https://github.com/ethereum/execution-spec-tests/pull/342)). diff --git a/whitelist.txt b/whitelist.txt index fca03d7fd9c..8a9fdfc0a99 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -309,6 +309,7 @@ funcargs getfixturevalue getgroup getoption +Golang groupby hookimpl hookwrapper From 90d3a6254ffd21218eb6151bc96d9242fcbaae05 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Thu, 30 Nov 2023 15:54:09 +0100 Subject: [PATCH 22/34] fix: manually specify nethermind genesis target filename --- tests_consume/test_via_rlp.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests_consume/test_via_rlp.py b/tests_consume/test_via_rlp.py index 4b63a16f094..c308fc0d064 100644 --- a/tests_consume/test_via_rlp.py +++ b/tests_consume/test_via_rlp.py @@ -103,6 +103,7 @@ def buffered_blocks_rlp(blocks_rlp: List[str], start=1) -> List[io.BufferedReade @pytest.fixture def files( test_case: TestCase, + client_type: ClientType, buffered_genesis: io.BufferedReader, buffered_blocks_rlp: list[io.BufferedReader], ) -> Mapping[str, io.BufferedReader]: @@ -114,7 +115,10 @@ def files( - Values are in-memory buffered file objects. """ files = {f"/blocks/{i:04d}.rlp": block_rlp for i, block_rlp in enumerate(buffered_blocks_rlp)} - files["/genesis.json"] = buffered_genesis + if client_type.name == "nethermind": + files["/chainspec/test.json"] = buffered_genesis + else: + files["/genesis.json"] = buffered_genesis return files From 50085b64a4a0cb98ab72824795d3a0538f75e120 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Thu, 30 Nov 2023 16:17:14 +0100 Subject: [PATCH 23/34] feat(pytest): enable piping of `fill`'s json to `consume rlp` (#26) --- src/entry_points/cli.py | 20 +++++++ .../consume_via_rlp/consume_via_rlp.py | 52 +++++++++++++------ src/pytest_plugins/test_filler/test_filler.py | 23 ++++++++ whitelist.txt | 2 + 4 files changed, 82 insertions(+), 15 deletions(-) diff --git a/src/entry_points/cli.py b/src/entry_points/cli.py index fea6dafba75..61e7ddddf20 100644 --- a/src/entry_points/cli.py +++ b/src/entry_points/cli.py @@ -94,6 +94,23 @@ def handle_help_flags(pytest_args, help_flag, pytest_help_flag): return list(pytest_args) +def handle_stdout_flags(args): + """ + If the user has requested to write to stdout, add pytest arguments in order + to suppress pytest's test session header and summary output. + """ + writing_to_stdout = False + if any(arg == "--output=stdout" for arg in args): + writing_to_stdout = True + elif any(arg.startswith("--output") for arg in args): + output_index = args.index("--output") + if output_index < len(args) - 1 and args[output_index + 1] == "stdout": + writing_to_stdout = True + if writing_to_stdout: + args.extend(["-qq", "-s"]) + return args + + def get_hive_flags_from_env(): """ Read simulator flags from environment variables and convert them, as best as @@ -125,6 +142,7 @@ def fill(pytest_args, help_flag, pytest_help_flag): Entry point for the fill command. """ args = handle_help_flags(pytest_args, help_flag, pytest_help_flag) + args = handle_stdout_flags(args) pytest.main(args) @@ -155,6 +173,8 @@ def consume_via_rlp(pytest_args, help_flag, pytest_help_flag): """ args = handle_help_flags(pytest_args, help_flag, pytest_help_flag) args += ["-c", "pytest-consume-via-rlp.ini"] + if not sys.stdin.isatty(): # the command is receiving input on stdin + args.append("-s") pytest.main(args) diff --git a/src/pytest_plugins/consume_via_rlp/consume_via_rlp.py b/src/pytest_plugins/consume_via_rlp/consume_via_rlp.py index 424ba2a2b84..2a57a15f829 100644 --- a/src/pytest_plugins/consume_via_rlp/consume_via_rlp.py +++ b/src/pytest_plugins/consume_via_rlp/consume_via_rlp.py @@ -5,9 +5,10 @@ Implemented using the pytest framework as a pytest plugin. """ import json +import sys from dataclasses import dataclass, field from pathlib import Path -from typing import Any, List, Optional, Tuple +from typing import Any, List, Literal, Optional, Tuple, Union import pytest @@ -33,6 +34,9 @@ def test_suite_description() -> str: return "Execute blockchain tests by providing RLP-encoded blocks to a client upon start-up." +JsonSource = Union[Path, Literal["stdin"]] + + @dataclass class TestCase: # noqa: D101 """ @@ -41,7 +45,7 @@ class TestCase: # noqa: D101 """ fixture_name: str - json_file_path: Path + json_file: JsonSource json_as_dict: dict fixture: Optional[Fixture] = None marks: List[pytest.MarkDecorator] = field(default_factory=list) @@ -66,15 +70,19 @@ def __post_init__(self): ) -def create_test_cases_from_json(json_file_path: Path) -> Tuple[List[Any], List[str]]: +def create_test_cases_from_json(json_file: JsonSource) -> Tuple[List[Any], List[str]]: """ - Extract blockchain test cases from a JSON fixture file. + Extract blockchain test cases from a JSON file or from stdin. """ test_cases = [] test_case_ids = [] - # TODO: Consider try-except block here - with open(json_file_path, "r") as file: - json_data = json.load(file) + + # TODO: exception handling? + if json_file == "stdin": + json_data = json.load(sys.stdin) + else: + with open(json_file, "r") as file: + json_data = json.load(file) for fixture_name, fixture_data in json_data.items(): fixture = None @@ -85,12 +93,12 @@ def create_test_cases_from_json(json_file_path: Path) -> Tuple[List[Any], List[s # Or should we? (it'll be brittle). fixture = load_dataclass_from_json(Fixture, fixture_data) except Exception as e: - reason = f"Error creating test case {fixture_name} from {json_file_path}: {e}" + reason = f"Error creating test case {fixture_name} from {json_file}: {e}" # TODO: Add logger.error() entry here marks.append(pytest.mark.xfail(reason=reason, run=False)) test_case = TestCase( - json_file_path=json_file_path, + json_file=json_file, json_as_dict=fixture_data, fixture_name=fixture_name, fixture=fixture, @@ -98,10 +106,16 @@ def create_test_cases_from_json(json_file_path: Path) -> Tuple[List[Any], List[s ) test_cases.append(pytest.param(test_case, marks=test_case.marks)) - if "::.py" in fixture_name: # new format; fixture name if fill pytest node id + if ( + json_file == "stdin" + or "::.py" in fixture_name + or (isinstance(json_file, str) and json_file == "stdin") + ): + # stdin or new format; fixture name if fill pytest node id + # (if stdin, json_file_path is None) test_case_ids.append(str(fixture_name)) - else: # old format, pre v1.0.7 - test_case_ids.append(f"{json_file_path.name}_{str(fixture_name)}") + else: + test_case_ids.append(f"{json_file.name}_{str(fixture_name)}") return test_cases, test_case_ids @@ -109,15 +123,23 @@ def create_test_cases_from_json(json_file_path: Path) -> Tuple[List[Any], List[s def pytest_generate_tests(metafunc): """ Generate test cases for every test fixture in all the JSON fixture files - within the specified fixtures directory. + within the specified fixtures directory, or read from stdin if the directory is 'stdin'. """ fixtures_directory = metafunc.config.getoption("fixture_directory") test_cases: List[TestCase] = [] test_case_ids: List[str] = [] - for json_file in fixtures_directory.glob("**/*.json"): - cases, ids = create_test_cases_from_json(json_file) + + if not sys.stdin.isatty(): + cases, ids = create_test_cases_from_json("stdin") test_cases.extend(cases) test_case_ids.extend(ids) + else: + fixtures_directory = Path(fixtures_directory) + for json_file in fixtures_directory.glob("**/*.json"): + cases, ids = create_test_cases_from_json(json_file) + test_cases.extend(cases) + test_case_ids.extend(ids) + metafunc.parametrize("test_case", test_cases, ids=test_case_ids) if "client_type" in metafunc.fixturenames: client_ids = [client.name for client in metafunc.config.hive_execution_clients] diff --git a/src/pytest_plugins/test_filler/test_filler.py b/src/pytest_plugins/test_filler/test_filler.py index 9356a03d2dc..60180c74102 100644 --- a/src/pytest_plugins/test_filler/test_filler.py +++ b/src/pytest_plugins/test_filler/test_filler.py @@ -8,6 +8,7 @@ import json import os import re +import sys import warnings from pathlib import Path from typing import Any, Dict, Generator, List, Literal, Optional, Tuple, Type, Union @@ -194,6 +195,20 @@ def pytest_report_header(config, start_path): return [f"{t8n.version()}, solc version {solc_version_string}"] +def pytest_report_teststatus(report, config): + """ + Disable test session progress report if we're writing the JSON fixtures to + stdout to be read by a consume command on stdin. I.e., don't write this + type of output to the console: + + ```text + ...x... + ``` + """ + if config.getoption("output") == "stdout": + return report.outcome, "", report.outcome.upper() + + @pytest.fixture(autouse=True, scope="session") def evm_bin(request) -> Path: """ @@ -370,6 +385,8 @@ def get_fixture_collection_scope(fixture_name, config): See: https://docs.pytest.org/en/stable/how-to/fixtures.html#dynamic-scope """ + if config.getoption("output") == "stdout": + return "session" if config.getoption("single_fixture_per_file"): return "function" return "module" @@ -446,6 +463,12 @@ def dump_fixtures(self) -> None: """ Dumps all collected fixtures to their respective files. """ + if self.output_dir == "stdout": + combined_fixtures = { + k: v for fixture in self.all_fixtures.values() for k, v in fixture.items() + } + json.dump(combined_fixtures, sys.stdout, indent=4) + return os.makedirs(self.output_dir, exist_ok=True) for fixture_path, fixtures in self.all_fixtures.items(): if not self.flat_output: diff --git a/whitelist.txt b/whitelist.txt index 8a9fdfc0a99..e340c399955 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -316,6 +316,7 @@ hookwrapper IEXEC IGNORECASE inifile +isatty iterdir ljust longreprtext @@ -352,6 +353,7 @@ substring substrings tf testdir +teststatus tmpdir tryfirst trylast From 70969c335ed36c220c02a9217096655d216192e0 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Fri, 1 Dec 2023 08:26:49 +0100 Subject: [PATCH 24/34] fix(cli): fix pipe behaviour; make stdin explicit via cli flag --- src/entry_points/cli.py | 8 +++++--- src/pytest_plugins/consume/consume.py | 2 ++ src/pytest_plugins/consume_via_rlp/consume_via_rlp.py | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/entry_points/cli.py b/src/entry_points/cli.py index 61e7ddddf20..38682c1e7c1 100644 --- a/src/entry_points/cli.py +++ b/src/entry_points/cli.py @@ -102,11 +102,13 @@ def handle_stdout_flags(args): writing_to_stdout = False if any(arg == "--output=stdout" for arg in args): writing_to_stdout = True - elif any(arg.startswith("--output") for arg in args): + elif "--output" in args: output_index = args.index("--output") - if output_index < len(args) - 1 and args[output_index + 1] == "stdout": + if args[output_index + 1] == "stdout": writing_to_stdout = True if writing_to_stdout: + if any(arg == "-n" or arg.startswith("-n=") for arg in args): + sys.exit("error: xdist-plugin not supported with --output=stdout (remove -n args).") args.extend(["-qq", "-s"]) return args @@ -174,7 +176,7 @@ def consume_via_rlp(pytest_args, help_flag, pytest_help_flag): args = handle_help_flags(pytest_args, help_flag, pytest_help_flag) args += ["-c", "pytest-consume-via-rlp.ini"] if not sys.stdin.isatty(): # the command is receiving input on stdin - args.append("-s") + args.extend(["-s", "--input=stdin"]) pytest.main(args) diff --git a/src/pytest_plugins/consume/consume.py b/src/pytest_plugins/consume/consume.py index 8bbf5f5456c..f483b2b117a 100644 --- a/src/pytest_plugins/consume/consume.py +++ b/src/pytest_plugins/consume/consume.py @@ -65,6 +65,8 @@ def pytest_addoption(parser): # noqa: D103 def pytest_configure(config): # noqa: D103 input_source = config.getoption("fixture_directory") + if input_source == "stdin": + return download_directory = cached_downloads_directory if is_url(input_source): diff --git a/src/pytest_plugins/consume_via_rlp/consume_via_rlp.py b/src/pytest_plugins/consume_via_rlp/consume_via_rlp.py index 2a57a15f829..1131c35675f 100644 --- a/src/pytest_plugins/consume_via_rlp/consume_via_rlp.py +++ b/src/pytest_plugins/consume_via_rlp/consume_via_rlp.py @@ -129,7 +129,7 @@ def pytest_generate_tests(metafunc): test_cases: List[TestCase] = [] test_case_ids: List[str] = [] - if not sys.stdin.isatty(): + if fixtures_directory == "stdin": cases, ids = create_test_cases_from_json("stdin") test_cases.extend(cases) test_case_ids.extend(ids) From ca58d3359381a2dbd7d20b29bcde5e5433a46871 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Fri, 1 Dec 2023 11:30:19 +0100 Subject: [PATCH 25/34] feat(pytest): add an entrypoint that fills & runs all consume commands --- setup.cfg | 1 + src/entry_points/cli.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/setup.cfg b/setup.cfg index dcbec2b186a..6887b246fd0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -51,6 +51,7 @@ console_scripts = phil = entry_points.cli:fill tf = entry_points.cli:tf consume = entry_points.cli:consume + fca = entry_points.cli:fill_and_consume_all order_fixtures = entry_points.order_fixtures:main pyspelling_soft_fail = entry_points.pyspelling_soft_fail:main markdownlintcli2_soft_fail = entry_points.markdownlintcli2_soft_fail:main diff --git a/src/entry_points/cli.py b/src/entry_points/cli.py index 38682c1e7c1..5960470d379 100644 --- a/src/entry_points/cli.py +++ b/src/entry_points/cli.py @@ -34,7 +34,9 @@ """ import os import sys +import tempfile import warnings +from pathlib import Path import click import pytest @@ -193,6 +195,22 @@ def consume_via_engine_api(pytest_args, help_flag, pytest_help_flag): # pytest.main(args) +@click.command(context_settings=dict(ignore_unknown_options=True)) +@common_options +def fill_and_consume_all(pytest_args, help_flag, pytest_help_flag): + """ + Fill and consume test fixtures using all available consume commands. + """ + args = handle_help_flags(pytest_args, help_flag, pytest_help_flag) + + temp_dir = Path(tempfile.TemporaryDirectory().name) / "fixtures" + args += ["--output", temp_dir] + pytest.main(args) + pytest.main(["-c", "pytest-consume-direct.ini", "--input", temp_dir]) + pytest.main(["-c", "pytest-consume-via-rlp.ini", "--input", temp_dir]) + # pytest.main(["-c", "pytest-consume-via-engine.ini", "--input", temp_dir]) + + consume.add_command(consume_direct, name="direct") consume.add_command(consume_via_rlp, name="rlp") consume.add_command(consume_via_engine_api, name="engine") From 06a1ff0c3462541cead20bacbcc567c5e031e9df Mon Sep 17 00:00:00 2001 From: danceratopz Date: Fri, 1 Dec 2023 21:08:52 +0100 Subject: [PATCH 26/34] feat(pytest): exit pytest gracefully if hive server connection fails --- src/pytest_plugins/pytest_hive/pytest_hive.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/pytest_plugins/pytest_hive/pytest_hive.py b/src/pytest_plugins/pytest_hive/pytest_hive.py index e4af5ad7596..4218984fc75 100644 --- a/src/pytest_plugins/pytest_hive/pytest_hive.py +++ b/src/pytest_plugins/pytest_hive/pytest_hive.py @@ -59,9 +59,21 @@ def pytest_configure(config): # noqa: D103 # of client_type with hive_execution_clients. config.hive_simulator_url = hive_simulator_url config.hive_simulator = Simulation(url=hive_simulator_url) - config.hive_execution_clients = config.hive_simulator.client_types( - role=ClientRole.ExecutionClient - ) + try: + config.hive_execution_clients = config.hive_simulator.client_types( + role=ClientRole.ExecutionClient + ) + except Exception as e: + message = ( + f"Error connecting to hive simulator at {hive_simulator_url}.\n\n" + "Did you forget to start hive in --dev mode?\n" + "./hive --dev --client go-ethereum\n\n" + ) + if config.option.verbose > 0: + message += f"Error details:\n{str(e)}" + else: + message += "Re-run with -v for more details." + pytest.exit(message) @pytest.hookimpl(trylast=True) From 632e090c5893a73c3f1f51e59dd77218c81dc5b9 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Mon, 4 Dec 2023 11:21:33 +0100 Subject: [PATCH 27/34] feat(pytest): allow all consumer tests to run in a single pytest session --- src/entry_points/cli.py | 28 ++- src/pytest_plugins/consume/consume.py | 188 ++++++++++++++++-- .../consume_direct/consume_direct.py | 61 ++---- .../consume_via_engine_api.py | 21 +- .../consume_via_rlp/consume_via_rlp.py | 128 +----------- src/pytest_plugins/pytest_hive/pytest_hive.py | 2 - tests_consume/test_direct.py | 33 ++- tests_consume/test_via_engine_api.py | 27 ++- tests_consume/test_via_rlp.py | 2 +- 9 files changed, 284 insertions(+), 206 deletions(-) diff --git a/src/entry_points/cli.py b/src/entry_points/cli.py index 5960470d379..a6e4bb5c964 100644 --- a/src/entry_points/cli.py +++ b/src/entry_points/cli.py @@ -166,6 +166,8 @@ def consume_direct(pytest_args, help_flag, pytest_help_flag): """ args = handle_help_flags(pytest_args, help_flag, pytest_help_flag) args += ["-c", "pytest-consume-direct.ini"] + if not sys.stdin.isatty(): # the command is receiving input on stdin + args.extend(["-s", "--input=stdin"]) pytest.main(args) @@ -177,6 +179,7 @@ def consume_via_rlp(pytest_args, help_flag, pytest_help_flag): """ args = handle_help_flags(pytest_args, help_flag, pytest_help_flag) args += ["-c", "pytest-consume-via-rlp.ini"] + args += get_hive_flags_from_env() if not sys.stdin.isatty(): # the command is receiving input on stdin args.extend(["-s", "--input=stdin"]) pytest.main(args) @@ -191,8 +194,23 @@ def consume_via_engine_api(pytest_args, help_flag, pytest_help_flag): args = handle_help_flags(pytest_args, help_flag, pytest_help_flag) args += ["-c", "pytest-consume-via-engine-api.ini"] args += get_hive_flags_from_env() - raise NotImplementedError("Consume via Engine API simulator is not implemented yet.") - # pytest.main(args) + if not sys.stdin.isatty(): # the command is receiving input on stdin + args.extend(["-s", "--input=stdin"]) + pytest.main(args) + + +@click.command(context_settings=dict(ignore_unknown_options=True)) +@common_options +def consume_all(pytest_args, help_flag, pytest_help_flag): + """ + Clients consume via all available methods (direct, rlp, engine). + """ + args = handle_help_flags(pytest_args, help_flag, pytest_help_flag) + args += ["-c", "pytest-consume-all.ini"] + args += get_hive_flags_from_env() + if not sys.stdin.isatty(): # the command is receiving input on stdin + args.extend(["-s", "--input=stdin"]) + pytest.main(args) @click.command(context_settings=dict(ignore_unknown_options=True)) @@ -206,11 +224,11 @@ def fill_and_consume_all(pytest_args, help_flag, pytest_help_flag): temp_dir = Path(tempfile.TemporaryDirectory().name) / "fixtures" args += ["--output", temp_dir] pytest.main(args) - pytest.main(["-c", "pytest-consume-direct.ini", "--input", temp_dir]) - pytest.main(["-c", "pytest-consume-via-rlp.ini", "--input", temp_dir]) - # pytest.main(["-c", "pytest-consume-via-engine.ini", "--input", temp_dir]) + consume_args = get_hive_flags_from_env() + pytest.main(["-c", "pytest-consume-all.ini", "--input", temp_dir, "-v"] + consume_args) +consume.add_command(consume_all, name="all") consume.add_command(consume_direct, name="direct") consume.add_command(consume_via_rlp, name="rlp") consume.add_command(consume_via_engine_api, name="engine") diff --git a/src/pytest_plugins/consume/consume.py b/src/pytest_plugins/consume/consume.py index f483b2b117a..6690661c4ee 100644 --- a/src/pytest_plugins/consume/consume.py +++ b/src/pytest_plugins/consume/consume.py @@ -1,13 +1,22 @@ """ A pytest plugin providing common functionality for consuming test fixtures. """ +import json +import sys import tarfile +from dataclasses import dataclass, field from pathlib import Path +from typing import List, Literal, Mapping, Optional, Union, get_args from urllib.parse import urlparse import pytest import requests +from ethereum_test_tools.common.json import load_dataclass_from_json +from ethereum_test_tools.common.types import Fixture + +from ..consume_via_rlp.network_ruleset_hive import ruleset + cached_downloads_directory = Path("./cached_downloads") @@ -63,27 +72,174 @@ def pytest_addoption(parser): # noqa: D103 ) +def generate_test_cases(fixtures_directory): # noqa: D103 + test_cases = [] + + if fixtures_directory == "stdin": + test_cases.extend(create_test_cases_from_json("stdin")) + else: + fixtures_directory = Path(fixtures_directory) + for json_file in fixtures_directory.glob("**/*.json"): + test_cases.extend(create_test_cases_from_json(json_file)) + + return test_cases + + def pytest_configure(config): # noqa: D103 input_source = config.getoption("fixture_directory") - if input_source == "stdin": - return - download_directory = cached_downloads_directory - - if is_url(input_source): - download_directory.mkdir(parents=True, exist_ok=True) - input_source = download_and_extract(input_source, download_directory) - - input_source = Path(input_source) - if not input_source.exists(): - pytest.exit(f"Specified fixture directory '{input_source}' does not exist.") - if not any(input_source.glob("**/*.json")): - pytest.exit( - f"Specified fixture directory '{input_source}' does not contain any JSON files." - ) + if input_source != "stdin": + download_directory = cached_downloads_directory + + if is_url(input_source): + download_directory.mkdir(parents=True, exist_ok=True) + input_source = download_and_extract(input_source, download_directory) - config.option.fixture_directory = input_source + input_source = Path(input_source) + if not input_source.exists(): + pytest.exit(f"Specified fixture directory '{input_source}' does not exist.") + if not any(input_source.glob("**/*.json")): + pytest.exit( + f"Specified fixture directory '{input_source}' does not contain any JSON files." + ) + config.option.fixture_directory = input_source + # We generate the list of test cases here, it need only be done once + config.test_cases = generate_test_cases(config.option.fixture_directory) def pytest_report_header(config): # noqa: D103 input_source = config.getoption("fixture_directory") return f"fixtures: {input_source}" + + +JsonSource = Union[Path, Literal["stdin"]] +ConsumerTypes = Literal["all", "direct", "rlp", "engine"] + + +@dataclass +class TestCase: # noqa: D101 + """ + Define the test case data associated a JSON test fixture in blockchain test + format. + """ + + @classmethod + def _marks_default(cls): + return {consumer_type: [] for consumer_type in get_args(ConsumerTypes)} + + fixture_name: str + json_file: JsonSource + json_as_dict: dict + fixture: Optional[Fixture] = None + fixture_json: Optional[dict] = field(default_factory=dict) + marks: Mapping[ConsumerTypes, List[pytest.MarkDecorator]] = field( + default_factory=lambda: TestCase._marks_default() + ) + __test__ = False # stop pytest from collecting this dataclass as a test + + def __post_init__(self): + """ + Sanity check the loaded test-case and add pytest marks. + + Marks can be applied based on any issues detected with the fixture. In + the future, we can apply marks that were written into the json fixture + file from `fill`. + """ + if any(mark is pytest.mark.xfail for mark in self.marks): + return # no point continuing + if not all("blockHeader" in block for block in self.fixture.blocks): + print("Skipping fixture with missing block header", self.fixture_name) + self.marks["rlp"].append(pytest.mark.xfail(reason="Missing block header", run=False)) + self.marks["engine"].append( + pytest.mark.xfail(reason="Missing block header", run=False) + ) + if self.fixture.fork not in ruleset: + self.marks["rlp"].append( + pytest.mark.xfail(reason=f"Unsupported network '{self.fixture.fork}'", run=False) + ) + + +def create_test_cases_from_json(json_file: JsonSource) -> List[TestCase]: + """ + Extract blockchain test cases from a JSON file or from stdin. + """ + test_cases = [] + + # TODO: exception handling? + if json_file == "stdin": + json_data = json.load(sys.stdin) + else: + with open(json_file, "r") as file: + json_data = json.load(file) + + for fixture_name, fixture_data in json_data.items(): + fixture = None + + marks: List[pytest.MarkDecorator] + try: + # TODO: here we validate fixture.blocks, for example, but not nested fields. Can we? + # Or should we? (it'll be brittle). + fixture = load_dataclass_from_json(Fixture, fixture_data) + fixture_json = {fixture_name: fixture_data} + marks = [] + except Exception as e: + # TODO: Add logger.error() entry here + reason = f"Error creating test case {fixture_name} from {json_file}: {e}" + fixture = None + fixture_json = None + marks = [pytest.mark.xfail(reason=reason, run=False)] + + test_case = TestCase( + json_file=json_file, + json_as_dict=fixture_data, + fixture_name=fixture_name, + fixture=fixture, + fixture_json=fixture_json, + ) + test_case.marks["all"].extend(marks) + test_cases.append(test_case) + + return test_cases + + +def pytest_generate_tests(metafunc): + """ + Generate test cases for every test fixture in all the JSON fixture files + within the specified fixtures directory, or read from stdin if the directory is 'stdin'. + """ + test_cases = metafunc.config.test_cases + if "test_blocktest" in metafunc.function.__name__: + pytest_params = [ + pytest.param( + test_case, + id=test_case.fixture_name, + marks=test_case.marks["all"] + test_case.marks["direct"], + ) + for test_case in test_cases + ] + metafunc.parametrize("test_case", pytest_params) + + if "test_via_rlp" in metafunc.function.__name__: + pytest_params = [ + pytest.param( + test_case, + id=test_case.fixture_name, + marks=test_case.marks["all"] + test_case.marks["rlp"], + ) + for test_case in test_cases + ] + metafunc.parametrize("test_case", pytest_params) + + if "test_via_engine" in metafunc.function.__name__: + pytest_params = [ + pytest.param( + test_case, + id=test_case.fixture_name, + marks=test_case.marks["all"] + test_case.marks["engine"], + ) + for test_case in test_cases + ] + metafunc.parametrize("test_case", pytest_params) + + if "client_type" in metafunc.fixturenames: + client_ids = [client.name for client in metafunc.config.hive_execution_clients] + metafunc.parametrize("client_type", metafunc.config.hive_execution_clients, ids=client_ids) diff --git a/src/pytest_plugins/consume_direct/consume_direct.py b/src/pytest_plugins/consume_direct/consume_direct.py index 7b9efdeb6a5..18a62043d39 100644 --- a/src/pytest_plugins/consume_direct/consume_direct.py +++ b/src/pytest_plugins/consume_direct/consume_direct.py @@ -1,21 +1,13 @@ """ A pytest plugin to execute the blocktest on the specified fixture directory. """ -import json -from dataclasses import dataclass from pathlib import Path from typing import Generator, Optional import pytest -from evm_transition_tool import FixtureFormats, TransitionTool - - -@dataclass -class TestCase: # noqa: D101 - fixture_name: str - fixture_format: FixtureFormats - json_file_path: Path +from evm_transition_tool import TransitionTool +from pytest_plugins.consume.consume import TestCase def pytest_addoption(parser): # noqa: D103 @@ -102,53 +94,24 @@ def test_dump_dir( @pytest.fixture(scope="function") -def json_fixture_path(fixture_data: TestCase): +def json_fixture_path(test_case: TestCase): """ Provide the path to the current JSON fixture file. """ - return fixture_data.json_file_path + return test_case.json_file -@pytest.fixture(scope="function") -def fixture_format(fixture_data: TestCase): - """ - The format of the current fixture. - """ - return fixture_data.fixture_format +# @pytest.fixture(scope="function") +# def fixture_format(fixture_data: TestCase): +# """ +# The format of the current fixture. +# """ +# return fixture_data.fixture_format @pytest.fixture(scope="function") -def fixture_name(fixture_data: TestCase): +def fixture_name(test_case: TestCase): """ The name of the current fixture. """ - return fixture_data.fixture_name - - -def pytest_generate_tests(metafunc): - """ - Generate test cases for every fixture in all JSON fixture files within the - fixtures directory. - """ - if "fixture_name" in metafunc.fixturenames: - fixtures_directory = metafunc.config.getoption("fixture_directory") - - fixture_data = [] - test_case_ids = [] - for json_file in fixtures_directory.glob("**/*.json"): - with json_file.open() as f: - data = json.load(f) - if metafunc.config.evm_use_single_test: - for fixture_name in data.keys(): - fixture_data.append( - TestCase(fixture_name, FixtureFormats.BLOCKCHAIN_TEST, json_file) - ) - test_case_ids.append(f"{json_file.name}_{fixture_name}") - else: - # evm bin does not support --single-test - fixture_data.append( - TestCase(json_file.name, FixtureFormats.BLOCKCHAIN_TEST, json_file) - ) - test_case_ids.append(f"{json_file.name}") - - metafunc.parametrize("fixture_data", fixture_data, ids=test_case_ids) + return test_case.fixture_name diff --git a/src/pytest_plugins/consume_via_engine_api/consume_via_engine_api.py b/src/pytest_plugins/consume_via_engine_api/consume_via_engine_api.py index 801d4cd12d7..77767b4c043 100644 --- a/src/pytest_plugins/consume_via_engine_api/consume_via_engine_api.py +++ b/src/pytest_plugins/consume_via_engine_api/consume_via_engine_api.py @@ -1,5 +1,24 @@ """ -A hive simulator that feeds blocks from test fixtures to clients via the Engine API. +A hive simulator that executes test fixtures in the blockchain test format +against clients by providing them a genesis state and blocks via the Engine +API. Implemented using the pytest framework as a pytest plugin. """ +import pytest + + +@pytest.fixture(scope="session") +def test_suite_name() -> str: + """ + The name of the hive test suite used in this simulator. + """ + return "EEST Consume Blocks via Engine API" + + +@pytest.fixture(scope="session") +def test_suite_description() -> str: + """ + The description of the hive test suite used in this simulator. + """ + return "Execute blockchain tests by sending blocks to a client via the Engine API." diff --git a/src/pytest_plugins/consume_via_rlp/consume_via_rlp.py b/src/pytest_plugins/consume_via_rlp/consume_via_rlp.py index 1131c35675f..ec00f9b4e31 100644 --- a/src/pytest_plugins/consume_via_rlp/consume_via_rlp.py +++ b/src/pytest_plugins/consume_via_rlp/consume_via_rlp.py @@ -1,22 +1,12 @@ """ -A hive simulator that feeds test fixtures to clients as RLP-encoded blocks -upon start-up. +A hive simulator that executes test fixtures in the blockchain test format +against clients by providing them a genesis state and RLP-encoded blocks +that they consume upon start-up. Implemented using the pytest framework as a pytest plugin. """ -import json -import sys -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any, List, Literal, Optional, Tuple, Union - import pytest -from ethereum_test_tools.common.json import load_dataclass_from_json -from ethereum_test_tools.common.types import Fixture - -from .network_ruleset_hive import ruleset - @pytest.fixture(scope="session") def test_suite_name() -> str: @@ -32,115 +22,3 @@ def test_suite_description() -> str: The description of the hive test suite used in this simulator. """ return "Execute blockchain tests by providing RLP-encoded blocks to a client upon start-up." - - -JsonSource = Union[Path, Literal["stdin"]] - - -@dataclass -class TestCase: # noqa: D101 - """ - Define the test case data associated a JSON test fixture in blockchain test - format. - """ - - fixture_name: str - json_file: JsonSource - json_as_dict: dict - fixture: Optional[Fixture] = None - marks: List[pytest.MarkDecorator] = field(default_factory=list) - __test__ = False # stop pytest from collecting this dataclass as a test - - def __post_init__(self): - """ - Sanity check the loaded test-case and add pytest marks. - - Marks can be applied based on any issues detected with the fixture. In - the future, we can apply marks that were written into the json fixture - file from `fill`. - """ - if any(mark is pytest.mark.xfail for mark in self.marks): - return # no point continuing - if not all("blockHeader" in block for block in self.fixture.blocks): - print("Skipping fixture with missing block header", self.fixture_name) - self.marks.append(pytest.mark.xfail(reason="Missing block header", run=False)) - if self.fixture.fork not in ruleset: - self.marks.append( - pytest.mark.xfail(reason=f"Unsupported network '{self.fixture.fork}'", run=False) - ) - - -def create_test_cases_from_json(json_file: JsonSource) -> Tuple[List[Any], List[str]]: - """ - Extract blockchain test cases from a JSON file or from stdin. - """ - test_cases = [] - test_case_ids = [] - - # TODO: exception handling? - if json_file == "stdin": - json_data = json.load(sys.stdin) - else: - with open(json_file, "r") as file: - json_data = json.load(file) - - for fixture_name, fixture_data in json_data.items(): - fixture = None - marks = [] - - try: - # TODO: here we validate fixture.blocks, for example, but not nested fields. Can we? - # Or should we? (it'll be brittle). - fixture = load_dataclass_from_json(Fixture, fixture_data) - except Exception as e: - reason = f"Error creating test case {fixture_name} from {json_file}: {e}" - # TODO: Add logger.error() entry here - marks.append(pytest.mark.xfail(reason=reason, run=False)) - - test_case = TestCase( - json_file=json_file, - json_as_dict=fixture_data, - fixture_name=fixture_name, - fixture=fixture, - marks=marks, - ) - test_cases.append(pytest.param(test_case, marks=test_case.marks)) - - if ( - json_file == "stdin" - or "::.py" in fixture_name - or (isinstance(json_file, str) and json_file == "stdin") - ): - # stdin or new format; fixture name if fill pytest node id - # (if stdin, json_file_path is None) - test_case_ids.append(str(fixture_name)) - else: - test_case_ids.append(f"{json_file.name}_{str(fixture_name)}") - - return test_cases, test_case_ids - - -def pytest_generate_tests(metafunc): - """ - Generate test cases for every test fixture in all the JSON fixture files - within the specified fixtures directory, or read from stdin if the directory is 'stdin'. - """ - fixtures_directory = metafunc.config.getoption("fixture_directory") - test_cases: List[TestCase] = [] - test_case_ids: List[str] = [] - - if fixtures_directory == "stdin": - cases, ids = create_test_cases_from_json("stdin") - test_cases.extend(cases) - test_case_ids.extend(ids) - else: - fixtures_directory = Path(fixtures_directory) - for json_file in fixtures_directory.glob("**/*.json"): - cases, ids = create_test_cases_from_json(json_file) - test_cases.extend(cases) - test_case_ids.extend(ids) - - metafunc.parametrize("test_case", test_cases, ids=test_case_ids) - if "client_type" in metafunc.fixturenames: - client_ids = [client.name for client in metafunc.config.hive_execution_clients] - metafunc.parametrize("client_type", metafunc.config.hive_execution_clients, ids=client_ids) diff --git a/src/pytest_plugins/pytest_hive/pytest_hive.py b/src/pytest_plugins/pytest_hive/pytest_hive.py index 4218984fc75..0fda7c0c00d 100644 --- a/src/pytest_plugins/pytest_hive/pytest_hive.py +++ b/src/pytest_plugins/pytest_hive/pytest_hive.py @@ -42,8 +42,6 @@ def test_suite(request, simulator: Simulation): def pytest_configure(config): # noqa: D103 - if config.option.collectonly: - return hive_simulator_url = os.environ.get("HIVE_SIMULATOR") if hive_simulator_url is None: pytest.exit( diff --git a/tests_consume/test_direct.py b/tests_consume/test_direct.py index a2cd98e63a8..35acbc51bc6 100644 --- a/tests_consume/test_direct.py +++ b/tests_consume/test_direct.py @@ -2,23 +2,46 @@ Executes a JSON test fixture directly against a client using a dedicated client interface similar to geth's EVM 'blocktest' command. """ +import json +import tempfile from pathlib import Path from typing import Optional +import pytest + from evm_transition_tool import FixtureFormats, TransitionTool +from pytest_plugins.consume.consume import TestCase + + +@pytest.fixture +def write_stdin_fixture_to_file(test_case: TestCase): + """ + If json fixtures have been provided on stdin, write the current test case's + fixture to a file for the blocktest command. + """ + if test_case.json_file == "stdin": + temp_dir = tempfile.TemporaryDirectory() + test_case.json_file = ( + Path(temp_dir.name) / f"{test_case.fixture_name.replace('/','_')}.json" + ) + with open(test_case.json_file, "w") as f: + json.dump(test_case.fixture_json, f, indent=4) + yield + if test_case.json_file == "stdin": + temp_dir.cleanup() -def test_fixtures( # noqa: D103 +@pytest.mark.usefixtures("write_stdin_fixture_to_file") +def test_blocktest( # noqa: D103 + test_case: TestCase, evm: TransitionTool, - json_fixture_path: Path, evm_use_single_test: bool, - fixture_name: str, test_dump_dir: Optional[Path], ): evm.verify_fixture( FixtureFormats.BLOCKCHAIN_TEST, - json_fixture_path, + test_case.json_file, evm_use_single_test, - fixture_name, + test_case.fixture_name, test_dump_dir, ) diff --git a/tests_consume/test_via_engine_api.py b/tests_consume/test_via_engine_api.py index 5fa9035a35d..ac5766bdd7b 100644 --- a/tests_consume/test_via_engine_api.py +++ b/tests_consume/test_via_engine_api.py @@ -1,2 +1,25 @@ -def test_via_engine_api(): - pass +""" +Test module to test clients with blocks send via the Engine API. +""" + +import pytest +from hive.client import Client, ClientType +from hive.testing import HiveTest + +from pytest_plugins.consume.consume import TestCase + + +@pytest.fixture(scope="function") +def client(hive_test: HiveTest, files: dict, environment: dict, client_type: ClientType) -> Client: + """ + Return the hive client being used to execute the current test case. + """ + client = hive_test.start_client(client_type=client_type, environment=environment, files=files) + assert client is not None + yield client + client.stop() + + +@pytest.mark.skip(reason="Engine API consumer not implemented yet.") +def test_via_engine_api(test_case: TestCase, client: Client): # noqa: D103 + assert test_case is not None diff --git a/tests_consume/test_via_rlp.py b/tests_consume/test_via_rlp.py index c308fc0d064..49ecee9fb0c 100644 --- a/tests_consume/test_via_rlp.py +++ b/tests_consume/test_via_rlp.py @@ -20,7 +20,7 @@ from tenacity import retry, stop_after_attempt, wait_exponential from ethereum_test_tools.common.types import Fixture -from pytest_plugins.consume_via_rlp.consume_via_rlp import TestCase +from pytest_plugins.consume.consume import TestCase from pytest_plugins.consume_via_rlp.network_ruleset_hive import ruleset From f8918c2b823fde0d7a273f1b3273ad8cd42e9dfb Mon Sep 17 00:00:00 2001 From: spencer Date: Mon, 5 Feb 2024 17:08:59 +0700 Subject: [PATCH 28/34] feat(pytest): add the consume engine command (#27) Co-authored-by: danceratopz --- src/ethereum_test_tools/common/json.py | 5 +- src/ethereum_test_tools/common/types.py | 72 +++++-- src/ethereum_test_tools/rpc/__init__.py | 12 ++ src/ethereum_test_tools/rpc/base_rpc.py | 45 +++++ src/ethereum_test_tools/rpc/engine_rpc.py | 62 ++++++ src/ethereum_test_tools/rpc/eth_rpc.py | 62 ++++++ src/pytest_plugins/consume/consume.py | 10 + .../consume_via_engine_api/__init__.py | 3 + .../client_fork_ruleset.py | 80 ++++++++ .../consume_via_engine_api.py | 8 +- stubs/jwt/__init__.pyi | 3 + stubs/jwt/encode.pyi | 3 + tests_consume/test_via_engine_api.py | 186 +++++++++++++++++- whitelist.txt | 6 + 14 files changed, 532 insertions(+), 25 deletions(-) create mode 100644 src/ethereum_test_tools/rpc/__init__.py create mode 100644 src/ethereum_test_tools/rpc/base_rpc.py create mode 100644 src/ethereum_test_tools/rpc/engine_rpc.py create mode 100644 src/ethereum_test_tools/rpc/eth_rpc.py create mode 100644 src/pytest_plugins/consume_via_engine_api/__init__.py create mode 100644 src/pytest_plugins/consume_via_engine_api/client_fork_ruleset.py create mode 100644 stubs/jwt/__init__.pyi create mode 100644 stubs/jwt/encode.pyi diff --git a/src/ethereum_test_tools/common/json.py b/src/ethereum_test_tools/common/json.py index 3ef74af3a1a..4b9195f84eb 100644 --- a/src/ethereum_test_tools/common/json.py +++ b/src/ethereum_test_tools/common/json.py @@ -178,7 +178,10 @@ def load_dataclass_from_json(dataclass_type, json_as_dict: dict): json_key = json_encoder_field.name or field.name if json_key in json_as_dict: value = json_as_dict[json_key] - if json_encoder_field.cast_type: + if ( + json_encoder_field.default_value_skip_cast != value + and json_encoder_field.cast_type + ): value = json_encoder_field.cast_type(value) init_args[field.name] = value diff --git a/src/ethereum_test_tools/common/types.py b/src/ethereum_test_tools/common/types.py index 344b9be37e1..2a5fb4e53eb 100644 --- a/src/ethereum_test_tools/common/types.py +++ b/src/ethereum_test_tools/common/types.py @@ -42,7 +42,7 @@ to_fixed_size_bytes, to_number, ) -from .json import JSONEncoder, SupportsJSON, field, to_json +from .json import JSONEncoder, SupportsJSON, field, load_dataclass_from_json, to_json # Sentinel classes @@ -867,8 +867,11 @@ def from_withdrawal(cls, w: Withdrawal) -> "FixtureWithdrawal": """ Returns a FixtureWithdrawal from a Withdrawal. """ - kwargs = {field.name: getattr(w, field.name) for field in fields(w)} - return cls(**kwargs) + if isinstance(w, dict): + return load_dataclass_from_json(cls, w) + else: + kwargs = {field.name: getattr(w, field.name) for field in fields(w)} + return cls(**kwargs) DEFAULT_BASE_FEE = 7 @@ -1151,6 +1154,24 @@ def to_list(self) -> List[bytes | List[bytes]]: return [Address(self.address), [Hash(k) for k in self.storage_keys]] +@dataclass(kw_only=True) +class FixtureAccessList(AccessList): + """ + Representation of an Access List within a test Fixture. + """ + + @classmethod + def from_access_list(cls, al: AccessList) -> "FixtureAccessList": + """ + Returns a FixtureAccessList from a AccessList or Dict. + """ + if isinstance(al, dict): + return load_dataclass_from_json(cls, al) + else: + kwargs = {field.name: getattr(al, field.name) for field in fields(al)} + return cls(**kwargs) + + @dataclass(kw_only=True) class Transaction: """ @@ -1164,9 +1185,6 @@ class Transaction: cast_type=HexNumber, ), ) - """ - Transaction type value. - """ chain_id: int = field( default=1, json_encoder=JSONEncoder.Field( @@ -1416,7 +1434,7 @@ def payload_body(self) -> List[Any]: if self.gas_limit is None: raise ValueError("gas_limit must be set for all tx types") - to = Address(self.to) if self.to is not None else bytes() + to = Address(self.to) if self.to else bytes() if self.ty == 3: # EIP-4844: https://eips.ethereum.org/EIPS/eip-4844 @@ -1560,7 +1578,7 @@ def signing_envelope(self) -> List[Any]: """ if self.gas_limit is None: raise ValueError("gas_limit must be set for all tx types") - to = Address(self.to) if self.to is not None else bytes() + to = Address(self.to) if self.to else bytes() if self.ty == 3: # EIP-4844: https://eips.ethereum.org/EIPS/eip-4844 @@ -1773,6 +1791,10 @@ class FixtureTransaction(Transaction): Representation of an Ethereum transaction within a test Fixture. """ + @staticmethod + def _access_lists_encoder(access_lists: List[AccessList]) -> List[FixtureAccessList]: + return [FixtureAccessList.from_access_list(al) for al in access_lists] + ty: Optional[int] = field( default=None, json_encoder=JSONEncoder.Field( @@ -1780,9 +1802,6 @@ class FixtureTransaction(Transaction): cast_type=ZeroPaddedHexNumber, ), ) - """ - Transaction type value. - """ chain_id: int = field( default=1, json_encoder=JSONEncoder.Field( @@ -1843,6 +1862,14 @@ class FixtureTransaction(Transaction): cast_type=Bytes, ), ) + access_list: Optional[List[AccessList]] = field( + default=None, + json_encoder=JSONEncoder.Field( + name="accessList", + cast_type=_access_lists_encoder, + to_json=True, + ), + ) max_fee_per_blob_gas: Optional[int] = field( default=None, json_encoder=JSONEncoder.Field( @@ -1874,8 +1901,11 @@ def from_transaction(cls, tx: Transaction) -> "FixtureTransaction": """ Returns a FixtureTransaction from a Transaction. """ - kwargs = {field.name: getattr(tx, field.name) for field in fields(tx)} - return cls(**kwargs) + if isinstance(tx, dict): + return load_dataclass_from_json(cls, tx) + else: + kwargs = {field.name: getattr(tx, field.name) for field in fields(tx)} + return cls(**kwargs) @dataclass(kw_only=True) @@ -2223,6 +2253,17 @@ def collect( # Pass the collected fields as keyword arguments to the constructor return cls(**kwargs) + @classmethod + def from_header(cls, h: Header) -> "FixtureHeader": + """ + Returns a FixtureHeader from a Header. + """ + if isinstance(h, dict): + return load_dataclass_from_json(cls, h) + else: + kwargs = {field.name: getattr(h, field.name) for field in fields(h)} + return cls(**kwargs) + def join(self, modifier: Header) -> "FixtureHeader": """ Produces a fixture header copy with the set values from the modifier. @@ -2616,6 +2657,10 @@ class FixtureBlock: Representation of an Ethereum block within a test Fixture. """ + @staticmethod + def _header_encoder(header: Header) -> FixtureHeader: + return FixtureHeader.from_header(header) + @staticmethod def _txs_encoder(txs: List[Transaction]) -> List[FixtureTransaction]: return [FixtureTransaction.from_transaction(tx) for tx in txs] @@ -2632,6 +2677,7 @@ def _withdrawals_encoder(withdrawals: List[Withdrawal]) -> List[FixtureWithdrawa default=None, json_encoder=JSONEncoder.Field( name="blockHeader", + cast_type=_header_encoder, to_json=True, ), ) diff --git a/src/ethereum_test_tools/rpc/__init__.py b/src/ethereum_test_tools/rpc/__init__.py new file mode 100644 index 00000000000..6eb4d4e7a4a --- /dev/null +++ b/src/ethereum_test_tools/rpc/__init__.py @@ -0,0 +1,12 @@ +""" +Ethereum JSON-RPC methods and types used within EEST based hive simulators. +""" + +from .engine_rpc import EngineRPC +from .eth_rpc import BlockNumberType, EthRPC + +__all__ = ( + "BlockNumberType", + "EthRPC", + "EngineRPC", +) diff --git a/src/ethereum_test_tools/rpc/base_rpc.py b/src/ethereum_test_tools/rpc/base_rpc.py new file mode 100644 index 00000000000..f7ca63cfc1b --- /dev/null +++ b/src/ethereum_test_tools/rpc/base_rpc.py @@ -0,0 +1,45 @@ +""" +Base JSON-RPC class and helper functions for EEST based hive simulators. +""" +import time + +import requests +from jwt import encode + + +class BaseRPC: + """ + Represents a base RPC class for every RPC call used within EEST based hive simulators. + """ + + def __init__(self, client): + self.client = client + self.url = f"http://{client.ip}:8551" + self.jwt_secret = ( + b"secretsecretsecretsecretsecretse" # oh wow, guess its not a secret anymore + ) + + def generate_jwt_token(self): + """ + Generates a JWT token based on the issued at timestamp and JWT secret. + """ + iat = int(time.time()) + return encode({"iat": iat}, self.jwt_secret, algorithm="HS256") + + def post_request(self, method, params): + """ + Sends a JSON-RPC POST request to the client RPC server at port 8551. + """ + payload = { + "jsonrpc": "2.0", + "method": method, + "params": params, + "id": 1, + } + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {self.generate_jwt_token()}", + } + response = requests.post(self.url, json=payload, headers=headers) + response.raise_for_status() + return response.json().get("result") diff --git a/src/ethereum_test_tools/rpc/engine_rpc.py b/src/ethereum_test_tools/rpc/engine_rpc.py new file mode 100644 index 00000000000..dc55fa21ee4 --- /dev/null +++ b/src/ethereum_test_tools/rpc/engine_rpc.py @@ -0,0 +1,62 @@ +""" +Ethereum `engine_X` JSON-RPC Engine API methods used within EEST based hive simulators. +""" + +from typing import Dict + +from ..common.json import to_json +from ..common.types import FixtureEngineNewPayload +from .base_rpc import BaseRPC + +ForkchoiceStateV1 = Dict +PayloadAttributes = Dict + + +class EngineRPC(BaseRPC): + """ + Represents an Engine API RPC class for every Engine API method used within EEST based hive + simulators. + """ + + def new_payload( + self, + engine_new_payload: FixtureEngineNewPayload, + ): + """ + `engine_newPayloadVX`: Attempts to execute the given payload on an execution client. + """ + version = engine_new_payload.version + engine_new_payload_json = to_json(engine_new_payload) + formatted_json = [ + engine_new_payload_json.get("executionPayload", None), + ] + if version >= 3: + formatted_json.append(engine_new_payload_json.get("expectedBlobVersionedHashes", None)) + formatted_json.append(engine_new_payload_json.get("parentBeaconBlockRoot", None)) + + # TODO: This is a temporary workaround to convert remove zero padding from withdrawals + withdrawals = formatted_json[0]["withdrawals"] + if len(withdrawals) > 0: + formatted_json[0]["withdrawals"] = [ + { + "index": hex(int(withdrawal["index"], 16)), + "validatorIndex": hex(int(withdrawal["validatorIndex"], 16)), + "address": withdrawal["address"], + "amount": hex(int(withdrawal["amount"], 16)), + } + for withdrawal in withdrawals + ] + + return self.post_request(f"engine_newPayloadV{version}", formatted_json) + + def forkchoice_updated( + self, + forkchoice_state: ForkchoiceStateV1, + payload_attributes: PayloadAttributes, + version: int = 1, + ): + """ + `engine_forkchoiceUpdatedVX`: Updates the forkchoice state of the execution client. + """ + payload_params = [forkchoice_state, payload_attributes] + return self.post_request(f"engine_forkchoiceUpdatedV{version}", payload_params) diff --git a/src/ethereum_test_tools/rpc/eth_rpc.py b/src/ethereum_test_tools/rpc/eth_rpc.py new file mode 100644 index 00000000000..f9881079300 --- /dev/null +++ b/src/ethereum_test_tools/rpc/eth_rpc.py @@ -0,0 +1,62 @@ +""" +Ethereum `eth_X` JSON-RPC methods used within EEST based hive simulators. +""" + +from typing import Dict, List, Literal, Union + +from ..common import Address +from .base_rpc import BaseRPC + +BlockNumberType = Union[int, Literal["latest", "earliest", "pending"]] + + +class EthRPC(BaseRPC): + """ + Represents an `eth_X` RPC class for every default ethereum RPC method used within EEST based + hive simulators. + """ + + def get_block_by_number(self, block_number: BlockNumberType = "latest", full_txs: bool = True): + """ + `eth_getBlockByNumber`: Returns information about a block by block number. + """ + block = hex(block_number) if isinstance(block_number, int) else block_number + return self.post_request("eth_getBlockByNumber", [block, full_txs]) + + def get_balance(self, address: str, block_number: BlockNumberType = "latest"): + """ + `eth_getBalance`: Returns the balance of the account of given address. + """ + block = hex(block_number) if isinstance(block_number, int) else block_number + return self.post_request("eth_getBalance", [address, block]) + + def get_transaction_count(self, address: Address, block_number: BlockNumberType = "latest"): + """ + `eth_getTransactionCount`: Returns the number of transactions sent from an address. + """ + block = hex(block_number) if isinstance(block_number, int) else block_number + return self.post_request("eth_getTransactionCount", [address, block]) + + def get_storage_at( + self, address: str, position: str, block_number: BlockNumberType = "latest" + ): + """ + `eth_getStorageAt`: Returns the value from a storage position at a given address. + """ + block = hex(block_number) if isinstance(block_number, int) else block_number + return self.post_request("eth_getStorageAt", [address, position, block]) + + def storage_at_keys( + self, account: str, keys: List[str], block_number: BlockNumberType = "latest" + ) -> Dict: + """ + Helper to retrieve the storage values for the specified keys at a given address and block + number. + """ + if isinstance(block_number, int): + block_number + results: Dict = {} + for key in keys: + storage_value = self.get_storage_at(account, key, block_number) + results[key] = storage_value + return results diff --git a/src/pytest_plugins/consume/consume.py b/src/pytest_plugins/consume/consume.py index 6690661c4ee..2b7b3f8bf70 100644 --- a/src/pytest_plugins/consume/consume.py +++ b/src/pytest_plugins/consume/consume.py @@ -201,6 +201,16 @@ def create_test_cases_from_json(json_file: JsonSource) -> List[TestCase]: return test_cases +@pytest.fixture(scope="function") +def test_case_fixture(test_case: TestCase) -> Fixture: + """ + The test fixture as a dictionary. If we failed to parse a test case fixture, + it's None: We xfail/skip the test. + """ + assert test_case.fixture is not None + return test_case.fixture + + def pytest_generate_tests(metafunc): """ Generate test cases for every test fixture in all the JSON fixture files diff --git a/src/pytest_plugins/consume_via_engine_api/__init__.py b/src/pytest_plugins/consume_via_engine_api/__init__.py new file mode 100644 index 00000000000..1d78ad9b352 --- /dev/null +++ b/src/pytest_plugins/consume_via_engine_api/__init__.py @@ -0,0 +1,3 @@ +""" +A hive simulator that executes blocks against clients using the Engine API. +""" diff --git a/src/pytest_plugins/consume_via_engine_api/client_fork_ruleset.py b/src/pytest_plugins/consume_via_engine_api/client_fork_ruleset.py new file mode 100644 index 00000000000..32c0cf89864 --- /dev/null +++ b/src/pytest_plugins/consume_via_engine_api/client_fork_ruleset.py @@ -0,0 +1,80 @@ +""" +Fork rules for clients ran within hive, starting from the Merge fork as +we are executing blocks using the Engine API. +""" + +# TODO: 1) Can we programmatically generate this? +# TODO: 2) Can we generate a single ruleset for both rlp and engine_api simulators. +client_fork_ruleset = { + "Merge": { + "HIVE_FORK_HOMESTEAD": 0, + "HIVE_FORK_TANGERINE": 0, + "HIVE_FORK_SPURIOUS": 0, + "HIVE_FORK_BYZANTIUM": 0, + "HIVE_FORK_CONSTANTINOPLE": 0, + "HIVE_FORK_PETERSBURG": 0, + "HIVE_FORK_ISTANBUL": 0, + "HIVE_FORK_BERLIN": 0, + "HIVE_FORK_LONDON": 0, + "HIVE_FORK_MERGE": 0, + "HIVE_TERMINAL_TOTAL_DIFFICULTY": 0, + }, + "Shanghai": { + "HIVE_FORK_HOMESTEAD": 0, + "HIVE_FORK_TANGERINE": 0, + "HIVE_FORK_SPURIOUS": 0, + "HIVE_FORK_BYZANTIUM": 0, + "HIVE_FORK_CONSTANTINOPLE": 0, + "HIVE_FORK_PETERSBURG": 0, + "HIVE_FORK_ISTANBUL": 0, + "HIVE_FORK_BERLIN": 0, + "HIVE_FORK_LONDON": 0, + "HIVE_FORK_MERGE": 0, + "HIVE_TERMINAL_TOTAL_DIFFICULTY": 0, + "HIVE_SHANGHAI_TIMESTAMP": 0, + }, + "MergeToShanghaiAtTime15k": { + "HIVE_FORK_HOMESTEAD": 0, + "HIVE_FORK_TANGERINE": 0, + "HIVE_FORK_SPURIOUS": 0, + "HIVE_FORK_BYZANTIUM": 0, + "HIVE_FORK_CONSTANTINOPLE": 0, + "HIVE_FORK_PETERSBURG": 0, + "HIVE_FORK_ISTANBUL": 0, + "HIVE_FORK_BERLIN": 0, + "HIVE_FORK_LONDON": 0, + "HIVE_FORK_MERGE": 0, + "HIVE_TERMINAL_TOTAL_DIFFICULTY": 0, + "HIVE_SHANGHAI_TIMESTAMP": 15000, + }, + "Cancun": { + "HIVE_FORK_HOMESTEAD": 0, + "HIVE_FORK_TANGERINE": 0, + "HIVE_FORK_SPURIOUS": 0, + "HIVE_FORK_BYZANTIUM": 0, + "HIVE_FORK_CONSTANTINOPLE": 0, + "HIVE_FORK_PETERSBURG": 0, + "HIVE_FORK_ISTANBUL": 0, + "HIVE_FORK_BERLIN": 0, + "HIVE_FORK_LONDON": 0, + "HIVE_FORK_MERGE": 0, + "HIVE_TERMINAL_TOTAL_DIFFICULTY": 0, + "HIVE_SHANGHAI_TIMESTAMP": 0, + "HIVE_CANCUN_TIMESTAMP": 0, + }, + "ShanghaiToCancunAtTime15k": { + "HIVE_FORK_HOMESTEAD": 0, + "HIVE_FORK_TANGERINE": 0, + "HIVE_FORK_SPURIOUS": 0, + "HIVE_FORK_BYZANTIUM": 0, + "HIVE_FORK_CONSTANTINOPLE": 0, + "HIVE_FORK_PETERSBURG": 0, + "HIVE_FORK_ISTANBUL": 0, + "HIVE_FORK_BERLIN": 0, + "HIVE_FORK_LONDON": 0, + "HIVE_FORK_MERGE": 0, + "HIVE_TERMINAL_TOTAL_DIFFICULTY": 0, + "HIVE_SHANGHAI_TIMESTAMP": 0, + "HIVE_CANCUN_TIMESTAMP": 15000, + }, +} diff --git a/src/pytest_plugins/consume_via_engine_api/consume_via_engine_api.py b/src/pytest_plugins/consume_via_engine_api/consume_via_engine_api.py index 77767b4c043..bb85d035e1a 100644 --- a/src/pytest_plugins/consume_via_engine_api/consume_via_engine_api.py +++ b/src/pytest_plugins/consume_via_engine_api/consume_via_engine_api.py @@ -1,10 +1,10 @@ """ -A hive simulator that executes test fixtures in the blockchain test format -against clients by providing them a genesis state and blocks via the Engine -API. +A hive simulator that executes blocks against clients using the `engine_newPayloadVX` method from +the Engine API, verifying the appropriate VALID/INVALID responses. Implemented using the pytest framework as a pytest plugin. """ + import pytest @@ -21,4 +21,4 @@ def test_suite_description() -> str: """ The description of the hive test suite used in this simulator. """ - return "Execute blockchain tests by sending blocks to a client via the Engine API." + return "Execute blockchain tests by against clients using the `engine_newPayloadVX` method." diff --git a/stubs/jwt/__init__.pyi b/stubs/jwt/__init__.pyi new file mode 100644 index 00000000000..dee1918afdf --- /dev/null +++ b/stubs/jwt/__init__.pyi @@ -0,0 +1,3 @@ +from .encode import encode + +__all__ = ("encode",) diff --git a/stubs/jwt/encode.pyi b/stubs/jwt/encode.pyi new file mode 100644 index 00000000000..3bfe608a1a1 --- /dev/null +++ b/stubs/jwt/encode.pyi @@ -0,0 +1,3 @@ +from typing import Any, Dict + +def encode(payload: Dict[Any, Any], key: bytes, algorithm: str) -> str: ... diff --git a/tests_consume/test_via_engine_api.py b/tests_consume/test_via_engine_api.py index ac5766bdd7b..987737162b0 100644 --- a/tests_consume/test_via_engine_api.py +++ b/tests_consume/test_via_engine_api.py @@ -1,25 +1,197 @@ """ -Test module to test clients with blocks send via the Engine API. +A hive simulator that executes blocks against clients using the +`engine_newPayloadVX` method from the Engine API, verifying +the appropriate VALID/INVALID responses. + +Implemented using the pytest framework as a pytest plugin. """ +import io +import json +from typing import Dict, List, Mapping import pytest from hive.client import Client, ClientType from hive.testing import HiveTest +# TODO: Replace with entire fork enum +from ethereum_test_forks import ( # noqa: F401 + Berlin, + Cancun, + Frontier, + London, + Merge, + MergeToShanghaiAtTime15k, + Shanghai, + ShanghaiToCancunAtTime15k, +) +from ethereum_test_tools.common.json import load_dataclass_from_json +from ethereum_test_tools.common.types import Account, FixtureBlock, FixtureEngineNewPayload +from ethereum_test_tools.rpc import EngineRPC, EthRPC from pytest_plugins.consume.consume import TestCase +from pytest_plugins.consume_via_engine_api.client_fork_ruleset import client_fork_ruleset + + +@pytest.fixture(scope="function") +def buffered_genesis(test_case: TestCase) -> Dict[str, io.BufferedReader]: + """ + Convert the genesis block header of the current test fixture to a buffered reader + readable by the client under test within hive. + """ + # Extract genesis and pre-alloc from test case fixture + genesis = test_case.fixture.genesis + pre_alloc = test_case.json_as_dict["pre"] + genesis["alloc"] = pre_alloc + + # Convert client genesis to BufferedReader + genesis_json = json.dumps(genesis) + genesis_bytes = genesis_json.encode("utf-8") + return io.BufferedReader(io.BytesIO(genesis_bytes)) + + +@pytest.fixture(scope="function") +def client_files( + client_type: ClientType, buffered_genesis: io.BufferedReader +) -> Mapping[str, io.BufferedReader]: + """ + Defines the files that hive will start the client with. + The buffered genesis including the pre-alloc. + """ + files = {} + # Client specific genesis format + if client_type.name == "nethermind": + files["/chainspec/test.json"] = buffered_genesis + else: + files["/genesis.json"] = buffered_genesis + return files + + +@pytest.fixture(scope="function") +def client_environment(test_case: TestCase) -> Dict: + """ + Defines the environment that hive will start the client with + using the fork rules specific for the simulator. + """ + client_env = { + "HIVE_FORK_DAO_VOTE": "1", + "HIVE_CHAIN_ID": "1", + "HIVE_NODETYPE": "full", + **{k: f"{v:d}" for k, v in client_fork_ruleset.get(test_case.fixture.fork, {}).items()}, + } + return client_env @pytest.fixture(scope="function") -def client(hive_test: HiveTest, files: dict, environment: dict, client_type: ClientType) -> Client: +def client( + hive_test: HiveTest, client_files: Dict, client_environment: Dict, client_type: ClientType +) -> Client: """ - Return the hive client being used to execute the current test case. + Initializes the client with the appropriate files and environment variables. """ - client = hive_test.start_client(client_type=client_type, environment=environment, files=files) + client = hive_test.start_client( + client_type=client_type, environment=client_environment, files=client_files + ) assert client is not None yield client client.stop() -@pytest.mark.skip(reason="Engine API consumer not implemented yet.") -def test_via_engine_api(test_case: TestCase, client: Client): # noqa: D103 - assert test_case is not None +@pytest.fixture(scope="function") +def engine_rpc(client: Client) -> EngineRPC: + """ + Initializes the engine RPC for the client under test. + """ + return EngineRPC(client) + + +@pytest.fixture(scope="function") +def eth_rpc(client: Client) -> EngineRPC: + """ + Initializes the eth RPC for the client under test. + """ + return EthRPC(client) + + +@pytest.fixture(scope="function") +def engine_new_payloads(test_case: TestCase) -> List[FixtureEngineNewPayload]: + """ + Execution payloads extracted from each block within the test case fixture. + Sent to the client under test using the `engine_newPayloadVX` method from the Engine API. + """ + fixture_blocks = [ + load_dataclass_from_json(FixtureBlock, block.get("rlp_decoded", block)) + for block in test_case.fixture.blocks + ] + + return [ + FixtureEngineNewPayload.from_fixture_header( + globals()[test_case.fixture.fork], + block.block_header, + block.txs, + block.withdrawals, + False if block.expected_exception else True, + error_code=None, + ) + for block in fixture_blocks + ] + + +def test_via_engine_api( + test_case: TestCase, + engine_new_payloads: List[FixtureEngineNewPayload], + engine_rpc: EngineRPC, + eth_rpc: EthRPC, +): + """ + 1) Checks that the genesis block hash of the client matches that of the fixture. + 2) Executes the test case fixture blocks against the client under test using the + `engine_newPayloadVX` method from the Engine API, verifying the appropriate + VALID/INVALID responses. + 3) Performs a forkchoice update to finalize the chain and verify the post state. + 4) Checks that the post state of the client matches that of the fixture. + """ + genesis_block = eth_rpc.get_block_by_number(0, False) + assert genesis_block["hash"] == test_case.fixture.genesis["hash"], "genesis hash mismatch" + + for payload in engine_new_payloads: + payload_response = engine_rpc.new_payload(payload) + assert payload_response["status"] == ( + "VALID" if payload.valid else "INVALID" + ), f"unexpected status: {payload_response} " + + forkchoice_response = engine_rpc.forkchoice_updated( + forkchoice_state={"headBlockHash": engine_new_payloads[-1].payload.hash}, + payload_attributes=None, + version=engine_new_payloads[-1].version, + ) + assert ( + forkchoice_response["payloadStatus"]["status"] == "VALID" + ), f"forkchoice update failed: {forkchoice_response}" + + for address, account in test_case.fixture.post_state.items(): + verify_account_state_and_storage(eth_rpc, address.hex(), account, test_case.fixture_name) + + +def verify_account_state_and_storage(eth_rpc, address, account: Account, test_name): + """ + Verify the account state and storage matches the expected account state and storage. + """ + # Check final nonce matches expected in fixture + nonce = eth_rpc.get_transaction_count(address) + assert int(account.nonce, 16) == int( + nonce, 16 + ), f"Nonce mismatch for account {address} in test {test_name}" + + # Check final balance + balance = eth_rpc.get_balance(address) + assert int(account.balance, 16) == int( + balance, 16 + ), f"Balance mismatch for account {address} in test {test_name}" + + # Check final storage + if len(account.storage) > 0: + keys = list(account.storage.keys()) + storage = eth_rpc.storage_at_keys(address, keys) + for key in keys: + assert int(account.storage[key], 16) == int( + storage[key], 16 + ), f"Storage mismatch for account {address}, key {key} in test {test_name}" diff --git a/whitelist.txt b/whitelist.txt index e340c399955..1536055a9b3 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -75,6 +75,7 @@ eip eips EIPs endianness +enp env eof esbenp @@ -112,16 +113,19 @@ git's github Github glightbox +globals go-ethereum's gwei hash32 hasher hexary hexsha +hexbytes homebrew html https hyperledger +iat ignoreRevsFile img incrementing @@ -212,6 +216,7 @@ repos returndatacopy returndatasize rlp +rpc ruleset runtime sandboxed @@ -352,6 +357,7 @@ subcommand substring substrings tf +teardown testdir teststatus tmpdir From 5f3a986def5012b4e86a430d276ecaca8fd2cbf7 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Wed, 7 Feb 2024 16:30:04 +0100 Subject: [PATCH 29/34] fix: check withdrawals is present in engine_new_payload (hotfix) --- src/ethereum_test_tools/rpc/engine_rpc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ethereum_test_tools/rpc/engine_rpc.py b/src/ethereum_test_tools/rpc/engine_rpc.py index 78be73e997d..42db8184f6c 100644 --- a/src/ethereum_test_tools/rpc/engine_rpc.py +++ b/src/ethereum_test_tools/rpc/engine_rpc.py @@ -35,8 +35,8 @@ def new_payload( formatted_json.append(engine_new_payload_json.get("parentBeaconBlockRoot", None)) # TODO: This is a temporary workaround to convert remove zero padding from withdrawals - withdrawals = formatted_json[0]["withdrawals"] - if len(withdrawals) > 0: + if "withdrawals" in formatted_json[0] and len(formatted_json[0]["withdrawals"]) > 0: + withdrawals = formatted_json[0]["withdrawals"] formatted_json[0]["withdrawals"] = [ { "index": hex(int(withdrawal["index"], 16)), From 2f46bcaf797f1748ac720ed0c270ab37bc06e9b3 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Wed, 7 Feb 2024 16:34:56 +0100 Subject: [PATCH 30/34] fix: hotfix that manually maps merge fixture network to Paris fork --- tests_consume/test_via_engine_api.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests_consume/test_via_engine_api.py b/tests_consume/test_via_engine_api.py index 86b32b4de1f..a7789c22ed6 100644 --- a/tests_consume/test_via_engine_api.py +++ b/tests_consume/test_via_engine_api.py @@ -123,9 +123,15 @@ def engine_new_payloads(test_case: TestCase) -> List[FixtureEngineNewPayload]: for block in test_case.fixture.blocks ] + fixture_fork = test_case.fixture.fork + if fixture_fork == "Merge": + # TODO: Handle mapping following rename of Merge fork + fork = Paris + else: + fork = globals()[fixture_fork] return [ FixtureEngineNewPayload.from_fixture_header( - globals()[test_case.fixture.fork], + fork, block.block_header, block.txs, block.withdrawals, From 77513adbd1dd37d31ce6bf4aeef20ed74f51b86b Mon Sep 17 00:00:00 2001 From: danceratopz Date: Wed, 7 Feb 2024 16:37:38 +0100 Subject: [PATCH 31/34] fix: hotfix to avoid block.expected_exception check; we should use InvalidFixtureBlock --- tests_consume/test_via_engine_api.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests_consume/test_via_engine_api.py b/tests_consume/test_via_engine_api.py index a7789c22ed6..539621efa4a 100644 --- a/tests_consume/test_via_engine_api.py +++ b/tests_consume/test_via_engine_api.py @@ -135,7 +135,7 @@ def engine_new_payloads(test_case: TestCase) -> List[FixtureEngineNewPayload]: block.block_header, block.txs, block.withdrawals, - False if block.expected_exception else True, + True, # TODO: Add support for InvalidFixtureBlock error_code=None, ) for block in fixture_blocks @@ -161,9 +161,11 @@ def test_via_engine_api( for payload in engine_new_payloads: payload_response = engine_rpc.new_payload(payload) - assert payload_response["status"] == ( - "VALID" if payload.valid else "INVALID" - ), f"unexpected status: {payload_response} " + assert payload_response["status"] == "VALID" + # TODO: Add support for InvalidFixtureBlock + # assert payload_response["status"] == ( + # "VALID" if payload.valid else "INVALID" + # ), f"unexpected status: {payload_response} " forkchoice_response = engine_rpc.forkchoice_updated( forkchoice_state={"headBlockHash": engine_new_payloads[-1].payload.hash}, From a8a1f54c4e32078c78178077b486570db95c40bb Mon Sep 17 00:00:00 2001 From: Spencer Taylor-Brown Date: Thu, 8 Feb 2024 05:50:42 +0000 Subject: [PATCH 32/34] chore: Skip blockchain_hive and state tests directories. --- src/pytest_plugins/consume/consume.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/pytest_plugins/consume/consume.py b/src/pytest_plugins/consume/consume.py index cfa796357ba..9a252480d72 100644 --- a/src/pytest_plugins/consume/consume.py +++ b/src/pytest_plugins/consume/consume.py @@ -1,6 +1,7 @@ """ A pytest plugin providing common functionality for consuming test fixtures. """ + import json import sys import tarfile @@ -74,14 +75,15 @@ def pytest_addoption(parser): # noqa: D103 def generate_test_cases(fixtures_directory): # noqa: D103 test_cases = [] - if fixtures_directory == "stdin": test_cases.extend(create_test_cases_from_json("stdin")) else: fixtures_directory = Path(fixtures_directory) - for json_file in fixtures_directory.glob("**/*.json"): + ignored_directories = {"blockchain_tests_hive", "state_tests"} + for json_file in fixtures_directory.rglob("*.json"): + if any(ignored_dir in json_file.parts for ignored_dir in ignored_directories): + continue test_cases.extend(create_test_cases_from_json(json_file)) - return test_cases From 5f69b17d2aa41deebf1f9b6ccb0f5f38678e8750 Mon Sep 17 00:00:00 2001 From: Spencer Taylor-Brown Date: Thu, 8 Feb 2024 05:55:08 +0000 Subject: [PATCH 33/34] chore: small logging tweak as we expect to skip invalid blockchain tests. --- src/pytest_plugins/consume/consume.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pytest_plugins/consume/consume.py b/src/pytest_plugins/consume/consume.py index 9a252480d72..94359c4f994 100644 --- a/src/pytest_plugins/consume/consume.py +++ b/src/pytest_plugins/consume/consume.py @@ -75,6 +75,8 @@ def pytest_addoption(parser): # noqa: D103 def generate_test_cases(fixtures_directory): # noqa: D103 test_cases = [] + # TODO: update to allow for invalid fixtures + print("---> Skipping all invalid blockchain fixtures. <---") if fixtures_directory == "stdin": test_cases.extend(create_test_cases_from_json("stdin")) else: @@ -148,8 +150,9 @@ def __post_init__(self): """ if any(mark is pytest.mark.xfail for mark in self.marks): return # no point continuing + # TODO: update to allow for invalid fixtures if not all("blockHeader" in block for block in self.fixture.blocks): - print("Skipping fixture with missing block header", self.fixture_name) + # print("Skipping fixture with missing block header", self.fixture_name) self.marks["rlp"].append(pytest.mark.xfail(reason="Missing block header", run=False)) self.marks["engine"].append( pytest.mark.xfail(reason="Missing block header", run=False) From c08a29db0736d87c1f2ce3363db3ac1092b4f749 Mon Sep 17 00:00:00 2001 From: Spencer Taylor-Brown Date: Mon, 5 Feb 2024 02:29:05 +0000 Subject: [PATCH 34/34] feat(fw): Add consume engine types and tests. --- docs/consuming_tests/blockchain_test.md | 2 +- setup.cfg | 1 + src/ethereum_test_forks/base_fork.py | 3 +- src/ethereum_test_forks/forks/forks.py | 5 +- src/ethereum_test_forks/tests/test_forks.py | 16 +- src/ethereum_test_tools/common/types.py | 14 +- src/ethereum_test_tools/consume/__init__.py | 3 + .../consume/engine/__init__.py | 3 + .../consume/engine/types.py | 445 ++++++++++++++++++ src/ethereum_test_tools/rpc/base_rpc.py | 1 + src/ethereum_test_tools/rpc/engine_rpc.py | 37 +- .../spec/blockchain/blockchain_test.py | 40 +- .../spec/blockchain/types.py | 54 +-- src/ethereum_test_tools/spec/state/types.py | 3 +- .../engine_json/valid_simple_cancun_enp.json | 25 + .../engine_json/valid_simple_paris_enp.json | 18 + .../valid_simple_shanghai_enp.json | 19 + .../valid_simple_cancun_blockchain.json | 128 +++++ .../valid_simple_paris_blockchain.json | 105 +++++ .../valid_simple_shanghai_blockchain.json | 108 +++++ .../test_consume/test_consume_engine_types.py | 226 +++++++++ .../tests/test_filling/test_fixtures.py | 10 +- src/ethereum_test_tools/tests/test_types.py | 50 +- .../tests/test_types_blockchain_test.py | 35 +- .../tests/test_evaluate.py | 8 +- src/evm_transition_tool/transition_tool.py | 3 +- .../eip4844_blobs/test_excess_blob_gas.py | 23 +- tests_consume/test_via_engine_api.py | 58 ++- whitelist.txt | 3 + 29 files changed, 1267 insertions(+), 179 deletions(-) create mode 100644 src/ethereum_test_tools/consume/__init__.py create mode 100644 src/ethereum_test_tools/consume/engine/__init__.py create mode 100644 src/ethereum_test_tools/consume/engine/types.py create mode 100644 src/ethereum_test_tools/tests/test_consume/engine_json/valid_simple_cancun_enp.json create mode 100644 src/ethereum_test_tools/tests/test_consume/engine_json/valid_simple_paris_enp.json create mode 100644 src/ethereum_test_tools/tests/test_consume/engine_json/valid_simple_shanghai_enp.json create mode 100644 src/ethereum_test_tools/tests/test_consume/fixtures/valid_simple_cancun_blockchain.json create mode 100644 src/ethereum_test_tools/tests/test_consume/fixtures/valid_simple_paris_blockchain.json create mode 100644 src/ethereum_test_tools/tests/test_consume/fixtures/valid_simple_shanghai_blockchain.json create mode 100644 src/ethereum_test_tools/tests/test_consume/test_consume_engine_types.py diff --git a/docs/consuming_tests/blockchain_test.md b/docs/consuming_tests/blockchain_test.md index 46a3d18b068..f9b2777a801 100644 --- a/docs/consuming_tests/blockchain_test.md +++ b/docs/consuming_tests/blockchain_test.md @@ -110,7 +110,7 @@ Root hash of the transactions trie. Root hash of the receipts trie. -#### - `bloom`: [`Bloom`](./common_types.md#bloom) +#### - `logsBloom`: [`Bloom`](./common_types.md#bloom) Bloom filter composed of the logs of all the transactions in the block. diff --git a/setup.cfg b/setup.cfg index 4ec7e09e22f..1d5cad510d7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -36,6 +36,7 @@ install_requires = tenacity>8.2.0,<9 trie==2.1.1 semver==3.0.1 + PyJWT==2.8.0 [options.package_data] ethereum_test_tools = diff --git a/src/ethereum_test_forks/base_fork.py b/src/ethereum_test_forks/base_fork.py index 645f9490b2d..9862f1fc375 100644 --- a/src/ethereum_test_forks/base_fork.py +++ b/src/ethereum_test_forks/base_fork.py @@ -1,6 +1,7 @@ """ Abstract base class for Ethereum forks """ + from abc import ABC, ABCMeta, abstractmethod from typing import Any, ClassVar, List, Mapping, Optional, Protocol, Type @@ -91,7 +92,7 @@ def __init_subclass__( # Header information abstract methods @classmethod @abstractmethod - def header_base_fee_required(cls, block_number: int = 0, timestamp: int = 0) -> bool: + def header_base_fee_per_gas_required(cls, block_number: int = 0, timestamp: int = 0) -> bool: """ Returns true if the header must contain base fee """ diff --git a/src/ethereum_test_forks/forks/forks.py b/src/ethereum_test_forks/forks/forks.py index 51094de37ea..120191186dd 100644 --- a/src/ethereum_test_forks/forks/forks.py +++ b/src/ethereum_test_forks/forks/forks.py @@ -1,6 +1,7 @@ """ All Ethereum fork class definitions. """ + from typing import List, Mapping, Optional from semver import Version @@ -40,7 +41,7 @@ def solc_min_version(cls) -> Version: return Version.parse("0.8.20") @classmethod - def header_base_fee_required(cls, block_number: int = 0, timestamp: int = 0) -> bool: + def header_base_fee_per_gas_required(cls, block_number: int = 0, timestamp: int = 0) -> bool: """ At genesis, header must not contain base fee """ @@ -258,7 +259,7 @@ class London(Berlin): """ @classmethod - def header_base_fee_required(cls, block_number: int = 0, timestamp: int = 0) -> bool: + def header_base_fee_per_gas_required(cls, block_number: int = 0, timestamp: int = 0) -> bool: """ Base Fee is required starting from London. """ diff --git a/src/ethereum_test_forks/tests/test_forks.py b/src/ethereum_test_forks/tests/test_forks.py index efe206fdfb6..3c365639dd6 100644 --- a/src/ethereum_test_forks/tests/test_forks.py +++ b/src/ethereum_test_forks/tests/test_forks.py @@ -49,8 +49,8 @@ def test_transition_forks(): assert ParisToShanghaiAtTime15k.transition_tool_name(0, 15_000) == "Shanghai" assert ParisToShanghaiAtTime15k.transition_tool_name() == "Shanghai" - assert BerlinToLondonAt5.header_base_fee_required(4, 0) is False - assert BerlinToLondonAt5.header_base_fee_required(5, 0) is True + assert BerlinToLondonAt5.header_base_fee_per_gas_required(4, 0) is False + assert BerlinToLondonAt5.header_base_fee_per_gas_required(5, 0) is True assert ParisToShanghaiAtTime15k.header_withdrawals_required(0, 14_999) is False assert ParisToShanghaiAtTime15k.header_withdrawals_required(0, 15_000) is True @@ -99,15 +99,15 @@ def test_forks(): assert ParisToShanghaiAtTime15k.blockchain_test_network_name() == "ParisToShanghaiAtTime15k" # Test some fork properties - assert Berlin.header_base_fee_required(0, 0) is False - assert London.header_base_fee_required(0, 0) is True - assert Paris.header_base_fee_required(0, 0) is True + assert Berlin.header_base_fee_per_gas_required(0, 0) is False + assert London.header_base_fee_per_gas_required(0, 0) is True + assert Paris.header_base_fee_per_gas_required(0, 0) is True # Default values of normal forks if the genesis block - assert Paris.header_base_fee_required() is True + assert Paris.header_base_fee_per_gas_required() is True # Transition forks too - assert cast(Fork, BerlinToLondonAt5).header_base_fee_required(4, 0) is False - assert cast(Fork, BerlinToLondonAt5).header_base_fee_required(5, 0) is True + assert cast(Fork, BerlinToLondonAt5).header_base_fee_per_gas_required(4, 0) is False + assert cast(Fork, BerlinToLondonAt5).header_base_fee_per_gas_required(5, 0) is True assert cast(Fork, ParisToShanghaiAtTime15k).header_withdrawals_required(0, 14_999) is False assert cast(Fork, ParisToShanghaiAtTime15k).header_withdrawals_required(0, 15_000) is True assert cast(Fork, ParisToShanghaiAtTime15k).header_withdrawals_required() is True diff --git a/src/ethereum_test_tools/common/types.py b/src/ethereum_test_tools/common/types.py index e37823c2311..16272f8b8dc 100644 --- a/src/ethereum_test_tools/common/types.py +++ b/src/ethereum_test_tools/common/types.py @@ -689,7 +689,7 @@ def withdrawals_root(withdrawals: List[Withdrawal]) -> bytes: return t.root_hash -DEFAULT_BASE_FEE = 7 +DEFAULT_BASE_FEE_PER_GAS = 7 @dataclass(kw_only=True) @@ -764,7 +764,7 @@ class Environment: to_json=True, ), ) - base_fee: Optional[NumberConvertible] = field( + base_fee_per_gas: Optional[NumberConvertible] = field( default=None, json_encoder=JSONEncoder.Field( name="currentBaseFee", @@ -785,7 +785,7 @@ class Environment: cast_type=Number, ), ) - parent_base_fee: Optional[NumberConvertible] = field( + parent_base_fee_per_gas: Optional[NumberConvertible] = field( default=None, json_encoder=JSONEncoder.Field( name="parentBaseFee", @@ -880,11 +880,11 @@ def set_fork_requirements(self, fork: Fork, in_place: bool = False) -> "Environm res.withdrawals = [] if ( - fork.header_base_fee_required(number, timestamp) - and res.base_fee is None - and res.parent_base_fee is None + fork.header_base_fee_per_gas_required(number, timestamp) + and res.base_fee_per_gas is None + and res.parent_base_fee_per_gas is None ): - res.base_fee = DEFAULT_BASE_FEE + res.base_fee_per_gas = DEFAULT_BASE_FEE_PER_GAS if fork.header_zero_difficulty_required(number, timestamp): res.difficulty = 0 diff --git a/src/ethereum_test_tools/consume/__init__.py b/src/ethereum_test_tools/consume/__init__.py new file mode 100644 index 00000000000..e16b259e8e4 --- /dev/null +++ b/src/ethereum_test_tools/consume/__init__.py @@ -0,0 +1,3 @@ +""" +Consume methods and types used for EEST based test runners (consumers). +""" diff --git a/src/ethereum_test_tools/consume/engine/__init__.py b/src/ethereum_test_tools/consume/engine/__init__.py new file mode 100644 index 00000000000..2d0d9d25764 --- /dev/null +++ b/src/ethereum_test_tools/consume/engine/__init__.py @@ -0,0 +1,3 @@ +""" +Consume methods and types for the Engine API. +""" diff --git a/src/ethereum_test_tools/consume/engine/types.py b/src/ethereum_test_tools/consume/engine/types.py new file mode 100644 index 00000000000..2f3db05c677 --- /dev/null +++ b/src/ethereum_test_tools/consume/engine/types.py @@ -0,0 +1,445 @@ +""" +Useful types for consuming EEST test runners or hive simulators. +""" + +from dataclasses import dataclass, fields +from typing import Any, Dict, List, Union, cast + +from ...common.json import JSONEncoder, field, to_json +from ...common.types import ( + Address, + Bytes, + Hash, + HexNumber, + blob_versioned_hashes_from_transactions, +) +from ...spec.blockchain.types import Bloom, FixtureBlock + + +class EngineParis: + """ + Paris (Merge) Engine API structures: + https://github.com/ethereum/execution-apis/blob/main/src/engine/paris.md + """ + + @dataclass(kw_only=True) + class ExecutionPayloadV1: + """ + Structure of a version 1 execution payload: + https://github.com/ethereum/execution-apis/blob/main/src/engine/paris.md#executionpayloadv1 + """ + + parent_hash: Hash = field( + json_encoder=JSONEncoder.Field( + name="parentHash", + ), + ) + """ + parentHash: DATA, 32 Bytes + """ + coinbase: Address = field( + json_encoder=JSONEncoder.Field( + name="feeRecipient", + ), + ) + """ + feeRecipient: DATA, 20 Bytes + """ + state_root: Hash = field( + json_encoder=JSONEncoder.Field( + name="stateRoot", + ), + ) + """ + stateRoot: DATA, 32 Bytes + """ + receipts_root: Hash = field( + json_encoder=JSONEncoder.Field( + name="receiptsRoot", + ), + ) + """ + receiptsRoot: DATA, 32 Bytes + """ + logs_bloom: Bloom = field( + json_encoder=JSONEncoder.Field( + name="logsBloom", + ), + ) + """ + logsBloom: DATA, 256 Bytes + """ + prev_randao: Hash = field( + json_encoder=JSONEncoder.Field( + name="prevRandao", + ), + ) + """ + prevRandao: DATA, 32 Bytes + """ + number: int = field( + json_encoder=JSONEncoder.Field( + name="blockNumber", + cast_type=HexNumber, + ), + ) + """ + blockNumber: QUANTITY, 64 Bits + """ + gas_limit: int = field( + json_encoder=JSONEncoder.Field( + name="gasLimit", + cast_type=HexNumber, + ), + ) + """ + gasLimit: QUANTITY, 64 Bits + """ + gas_used: int = field( + json_encoder=JSONEncoder.Field( + name="gasUsed", + cast_type=HexNumber, + ), + ) + """ + gasUsed: QUANTITY, 64 Bits + """ + timestamp: int = field( + json_encoder=JSONEncoder.Field( + name="timestamp", + cast_type=HexNumber, + ), + ) + """ + timestamp: QUANTITY, 64 Bits + """ + extra_data: Bytes = field( + json_encoder=JSONEncoder.Field( + name="extraData", + ), + ) + """ + extraData: DATA, 0 to 32 Bytes + """ + base_fee_per_gas: int = field( + json_encoder=JSONEncoder.Field( + name="baseFeePerGas", + cast_type=HexNumber, + ), + ) + """ + baseFeePerGas: QUANTITY, 64 Bits + """ + block_hash: Hash = field( + json_encoder=JSONEncoder.Field( + name="blockHash", + ), + ) + """ + blockHash: DATA, 32 Bytes + """ + transactions: List[str] = field( + json_encoder=JSONEncoder.Field( + name="transactions", + to_json=True, + ), + ) + """ + transactions: Array of DATA + """ + + @classmethod + def from_fixture_block( + cls, fixture_block: FixtureBlock + ) -> "EngineParis.ExecutionPayloadV1": + """ + Converts a fixture block to a Paris execution payload. + """ + header = fixture_block.block_header + transactions = [ + "0x" + tx.serialized_bytes().hex() for tx in fixture_block.transactions + ] + + kwargs = { + field.name: getattr(header, field.name) + for field in fields(header) + if field.name in {f.name for f in fields(cls)} + } + + return cls(**kwargs, transactions=transactions) + + @dataclass(kw_only=True) + class NewPayloadV1: + """ + Structure of a version 1 engine new payload: + https://github.com/ethereum/execution-apis/blob/main/src/engine/paris.md#engine_newpayloadv1 + """ + + execution_payload: "EngineParis.ExecutionPayloadV1" = field( + json_encoder=JSONEncoder.Field( + name="executionPayload", + to_json=True, + ), + ) + + @classmethod + def from_fixture_block(cls, fixture_block: FixtureBlock) -> "EngineParis.NewPayloadV1": + """ + Creates a Paris engine new payload from a fixture block. + """ + return EngineParis.NewPayloadV1( + execution_payload=EngineParis.ExecutionPayloadV1.from_fixture_block(fixture_block) + ) + + @classmethod + def version(cls) -> int: + """ + Returns the version of the engine new payload. + """ + return 1 + + def to_json_rpc(self) -> List[Dict[str, Any]]: + """ + Serializes a Paris engine new payload dataclass to its JSON-RPC representation. + """ + return [to_json(self.execution_payload)] + + +class EngineShanghai: + """ + Shanghai Engine API structures: + https://github.com/ethereum/execution-apis/blob/main/src/engine/shanghai.md + """ + + @dataclass(kw_only=True) + class WithdrawalV1: + """ + Structure of a version 1 withdrawal: + https://github.com/ethereum/execution-apis/blob/main/src/engine/shanghai.md#withdrawalv1 + """ + + index: int = field( + json_encoder=JSONEncoder.Field( + name="index", + cast_type=HexNumber, + ), + ) + """ + index: QUANTITY, 64 Bits + """ + validator_index: int = field( + json_encoder=JSONEncoder.Field( + name="validatorIndex", + cast_type=HexNumber, + ), + ) + """ + validatorIndex: QUANTITY, 64 Bits + """ + address: Address = field( + json_encoder=JSONEncoder.Field( + name="address", + ), + ) + """ + address: DATA, 20 Bytes + """ + amount: int = field( + json_encoder=JSONEncoder.Field( + name="amount", + cast_type=HexNumber, + ), + ) + """ + amount: QUANTITY, 64 Bits + """ + + @dataclass(kw_only=True) + class ExecutionPayloadV2(EngineParis.ExecutionPayloadV1): + """ + Structure of a version 2 execution payload: + https://github.com/ethereum/execution-apis/blob/main/src/engine/shanghai.md#executionpayloadv2 + """ + + withdrawals: List["EngineShanghai.WithdrawalV1"] = field( + json_encoder=JSONEncoder.Field( + name="withdrawals", + to_json=True, + ), + ) + """ + withdrawals: Array of WithdrawalV1 + """ + + @classmethod + def from_fixture_block( + cls, fixture_block: FixtureBlock + ) -> "EngineShanghai.ExecutionPayloadV2": + """ + Converts a fixture block to a Shanghai execution payload. + """ + header = fixture_block.block_header + transactions = [ + "0x" + tx.serialized_bytes().hex() for tx in fixture_block.transactions + ] + withdrawals = cast(List[EngineShanghai.WithdrawalV1], fixture_block.withdrawals) + kwargs = { + field.name: getattr(header, field.name) + for field in fields(header) + if field.name in {f.name for f in fields(cls)} + } + return cls(**kwargs, transactions=transactions, withdrawals=withdrawals) + + @dataclass(kw_only=True) + class NewPayloadV2: + """ + Structure of a version 2 engine new payload: + https://github.com/ethereum/execution-apis/blob/main/src/engine/shanghai.md#engine_newpayloadv2 + """ + + execution_payload: Union[ + "EngineShanghai.ExecutionPayloadV2", "EngineParis.ExecutionPayloadV1" + ] = field( + json_encoder=JSONEncoder.Field( + name="executionPayload", + to_json=True, + ), + ) + + @classmethod + def from_fixture_block(cls, fixture_block: FixtureBlock) -> "EngineShanghai.NewPayloadV2": + """ + Creates a Shanghai engine new payload from a fixture block. + """ + return EngineShanghai.NewPayloadV2( + execution_payload=EngineShanghai.ExecutionPayloadV2.from_fixture_block( + fixture_block + ) + ) + + @classmethod + def version(cls) -> int: + """ + Returns the version of the engine new payload. + """ + return 2 + + def to_json_rpc(self) -> List[Dict[str, Any]]: + """ + Serializes a Shanghai engine new payload dataclass to its JSON-RPC representation. + """ + return [to_json(self.execution_payload)] + + +class EngineCancun: + """ + Cancun Engine API structures: + https://github.com/ethereum/execution-apis/blob/main/src/engine/cancun.md + """ + + @dataclass(kw_only=True) + class ExecutionPayloadV3(EngineShanghai.ExecutionPayloadV2): + """ + Structure of a version 3 execution payload: + https://github.com/ethereum/execution-apis/blob/main/src/engine/cancun.md#executionpayloadv3 + """ + + blob_gas_used: int = field( + json_encoder=JSONEncoder.Field( + name="blobGasUsed", + cast_type=HexNumber, + ), + ) + """ + blobGasUsed: QUANTITY, 64 Bits + """ + excess_blob_gas: int = field( + json_encoder=JSONEncoder.Field( + name="excessBlobGas", + cast_type=HexNumber, + ), + ) + """ + excessBlobGas: QUANTITY, 64 Bits + """ + + @classmethod + def from_fixture_block( + cls, fixture_block: FixtureBlock + ) -> "EngineCancun.ExecutionPayloadV3": + """ + Converts a fixture block to a Cancun execution payload. + """ + header = fixture_block.block_header + transactions = [ + "0x" + tx.serialized_bytes().hex() for tx in fixture_block.transactions + ] + withdrawals = cast(List[EngineShanghai.WithdrawalV1], fixture_block.withdrawals) + + kwargs = { + field.name: getattr(header, field.name) + for field in fields(header) + if field.name in {f.name for f in fields(cls)} + } + + return cls(**kwargs, transactions=transactions, withdrawals=withdrawals) + + @dataclass(kw_only=True) + class NewPayloadV3: + """ + Structure of a version 3 engine new payload: + https://github.com/ethereum/execution-apis/blob/main/src/engine/cancun.md#engine_newpayloadv3 + """ + + execution_payload: "EngineCancun.ExecutionPayloadV3" = field( + json_encoder=JSONEncoder.Field( + to_json=True, + ), + ) + """ + executionPayload: ExecutionPayloadV3 + """ + expected_blob_versioned_hashes: List[Hash] + """ + expectedBlobVersionedHashes: Array of DATA + """ + parent_beacon_block_root: Hash + """ + parentBeaconBlockRoot: DATA, 32 Bytes + """ + + @classmethod + def from_fixture_block(cls, fixture_block: FixtureBlock) -> "EngineCancun.NewPayloadV3": + """ + Creates a Cancun engine new payload from a fixture block. + """ + execution_payload = EngineCancun.ExecutionPayloadV3.from_fixture_block(fixture_block) + expected_blob_versioned_hashes = blob_versioned_hashes_from_transactions( + fixture_block.transactions + ) + parent_beacon_block_root = cast(Hash, fixture_block.block_header.beacon_root) + return cls( + execution_payload=execution_payload, + expected_blob_versioned_hashes=[ + Hash(blob_versioned_hash) + for blob_versioned_hash in expected_blob_versioned_hashes + ], + parent_beacon_block_root=parent_beacon_block_root, + ) + + @classmethod + def version(cls) -> int: + """ + Returns the version of the engine new payload. + """ + return 3 + + def to_json_rpc(self) -> List[Union[Dict[str, Any], List[Hash], Hash]]: + """ + Serializes a Cancun engine new payload dataclass to its JSON-RPC representation. + """ + return [ + to_json(self.execution_payload), + self.expected_blob_versioned_hashes, + self.parent_beacon_block_root, + ] diff --git a/src/ethereum_test_tools/rpc/base_rpc.py b/src/ethereum_test_tools/rpc/base_rpc.py index f7ca63cfc1b..ba6658cce6c 100644 --- a/src/ethereum_test_tools/rpc/base_rpc.py +++ b/src/ethereum_test_tools/rpc/base_rpc.py @@ -1,6 +1,7 @@ """ Base JSON-RPC class and helper functions for EEST based hive simulators. """ + import time import requests diff --git a/src/ethereum_test_tools/rpc/engine_rpc.py b/src/ethereum_test_tools/rpc/engine_rpc.py index 42db8184f6c..f6a8a3e602f 100644 --- a/src/ethereum_test_tools/rpc/engine_rpc.py +++ b/src/ethereum_test_tools/rpc/engine_rpc.py @@ -2,10 +2,9 @@ Ethereum `engine_X` JSON-RPC Engine API methods used within EEST based hive simulators. """ -from typing import Dict +from typing import Dict, Union -from ..common.json import to_json -from ..spec.blockchain.types import FixtureEngineNewPayload +from ..consume.engine.types import EngineCancun, EngineParis, EngineShanghai from .base_rpc import BaseRPC ForkchoiceStateV1 = Dict @@ -20,34 +19,18 @@ class EngineRPC(BaseRPC): def new_payload( self, - engine_new_payload: FixtureEngineNewPayload, + engine_new_payload: Union[ + EngineCancun.NewPayloadV3, + EngineShanghai.NewPayloadV2, + EngineParis.NewPayloadV1, + ], ): """ `engine_newPayloadVX`: Attempts to execute the given payload on an execution client. """ - version = engine_new_payload.version - engine_new_payload_json = to_json(engine_new_payload) - formatted_json = [ - engine_new_payload_json.get("executionPayload", None), - ] - if version >= 3: - formatted_json.append(engine_new_payload_json.get("expectedBlobVersionedHashes", None)) - formatted_json.append(engine_new_payload_json.get("parentBeaconBlockRoot", None)) - - # TODO: This is a temporary workaround to convert remove zero padding from withdrawals - if "withdrawals" in formatted_json[0] and len(formatted_json[0]["withdrawals"]) > 0: - withdrawals = formatted_json[0]["withdrawals"] - formatted_json[0]["withdrawals"] = [ - { - "index": hex(int(withdrawal["index"], 16)), - "validatorIndex": hex(int(withdrawal["validatorIndex"], 16)), - "address": withdrawal["address"], - "amount": hex(int(withdrawal["amount"], 16)), - } - for withdrawal in withdrawals - ] - - return self.post_request(f"engine_newPayloadV{version}", formatted_json) + return self.post_request( + f"engine_newPayloadV{engine_new_payload.version()}", engine_new_payload.to_json_rpc() + ) def forkchoice_updated( self, diff --git a/src/ethereum_test_tools/spec/blockchain/blockchain_test.py b/src/ethereum_test_tools/spec/blockchain/blockchain_test.py index cef192ad7fc..1737e777868 100644 --- a/src/ethereum_test_tools/spec/blockchain/blockchain_test.py +++ b/src/ethereum_test_tools/spec/blockchain/blockchain_test.py @@ -5,7 +5,7 @@ from copy import copy from dataclasses import dataclass, field, replace from pprint import pprint -from typing import Any, Callable, Dict, Generator, List, Mapping, Optional, Tuple, Type +from typing import Any, Callable, Dict, Generator, List, Mapping, Optional, Tuple, Type, Union from ethereum_test_forks import Fork from evm_transition_tool import FixtureFormats, TransitionTool @@ -28,13 +28,7 @@ ) from ...common.constants import EmptyOmmersRoot from ...common.json import to_json -from ..base.base_test import ( - BaseFixture, - BaseTest, - verify_post_alloc, - verify_result, - verify_transactions, -) +from ..base.base_test import BaseTest, verify_post_alloc, verify_result, verify_transactions from ..debugging import print_traces from .types import ( Block, @@ -54,13 +48,13 @@ def environment_from_parent_header(parent: "FixtureHeader") -> "Environment": return Environment( parent_difficulty=parent.difficulty, parent_timestamp=parent.timestamp, - parent_base_fee=parent.base_fee, + parent_base_fee_per_gas=parent.base_fee_per_gas, parent_blob_gas_used=parent.blob_gas_used, parent_excess_blob_gas=parent.excess_blob_gas, parent_gas_used=parent.gas_used, parent_gas_limit=parent.gas_limit, parent_ommers_hash=parent.ommers_hash, - block_hashes={parent.number: parent.hash if parent.hash is not None else 0}, + block_hashes={parent.number: parent.block_hash if parent.block_hash is not None else 0}, ) @@ -71,13 +65,15 @@ def apply_new_parent(env: Environment, new_parent: FixtureHeader) -> "Environmen env = copy(env) env.parent_difficulty = new_parent.difficulty env.parent_timestamp = new_parent.timestamp - env.parent_base_fee = new_parent.base_fee + env.parent_base_fee_per_gas = new_parent.base_fee_per_gas env.parent_blob_gas_used = new_parent.blob_gas_used env.parent_excess_blob_gas = new_parent.excess_blob_gas env.parent_gas_used = new_parent.gas_used env.parent_gas_limit = new_parent.gas_limit env.parent_ommers_hash = new_parent.ommers_hash - env.block_hashes[new_parent.number] = new_parent.hash if new_parent.hash is not None else 0 + env.block_hashes[new_parent.number] = ( + new_parent.block_hash if new_parent.block_hash is not None else 0 + ) return env @@ -153,17 +149,17 @@ def make_genesis( coinbase=Address(0), state_root=Hash(state_root), transactions_root=Hash(EmptyTrieRoot), - receipt_root=Hash(EmptyTrieRoot), - bloom=Bloom(0), + receipts_root=Hash(EmptyTrieRoot), + logs_bloom=Bloom(0), difficulty=ZeroPaddedHexNumber(0x20000 if env.difficulty is None else env.difficulty), number=0, gas_limit=ZeroPaddedHexNumber(env.gas_limit), gas_used=0, timestamp=0, extra_data=Bytes([0]), - mix_digest=Hash(0), + prev_randao=Hash(0), nonce=HeaderNonce(0), - base_fee=ZeroPaddedHexNumber.or_none(env.base_fee), + base_fee_per_gas=ZeroPaddedHexNumber.or_none(env.base_fee_per_gas), blob_gas_used=ZeroPaddedHexNumber.or_none(env.blob_gas_used), excess_blob_gas=ZeroPaddedHexNumber.or_none(env.excess_blob_gas), withdrawals_root=Hash.or_none( @@ -172,7 +168,7 @@ def make_genesis( beacon_root=Hash.or_none(env.beacon_root), ) - genesis_rlp, genesis.hash = genesis.build( + genesis_rlp, genesis.block_hash = genesis.build( txs=[], ommers=[], withdrawals=env.withdrawals, @@ -276,7 +272,7 @@ def generate_block_data( # transition tool processing. header = header.join(block.rlp_modifier) - rlp, header.hash = header.build( + rlp, header.block_hash = header.build( txs=txs, ommers=[], withdrawals=env.withdrawals, @@ -319,7 +315,7 @@ def make_fixture( alloc = to_json(pre) env = environment_from_parent_header(genesis) - head = genesis.hash if genesis.hash is not None else Hash(0) + head = genesis.block_hash if genesis.block_hash is not None else Hash(0) for block in self.blocks: if block.rlp is None: @@ -338,7 +334,7 @@ def make_fixture( rlp=rlp, block_header=header, block_number=Number(header.number), - txs=txs, + transactions=txs, ommers=[], withdrawals=new_env.withdrawals, ) @@ -347,7 +343,7 @@ def make_fixture( # Update env, alloc and last block hash for the next block. alloc = new_alloc env = apply_new_parent(new_env, header) - head = header.hash if header.hash is not None else Hash(0) + head = header.block_hash if header.block_hash is not None else Hash(0) else: fixture_blocks.append( InvalidFixtureBlock( @@ -435,7 +431,7 @@ def generate( t8n: TransitionTool, fork: Fork, eips: Optional[List[int]] = None, - ) -> BaseFixture: + ) -> Union[Fixture, HiveFixture]: """ Generate the BlockchainTest fixture. """ diff --git a/src/ethereum_test_tools/spec/blockchain/types.py b/src/ethereum_test_tools/spec/blockchain/types.py index 019ab8836f8..3a75c93f1d1 100644 --- a/src/ethereum_test_tools/spec/blockchain/types.py +++ b/src/ethereum_test_tools/spec/blockchain/types.py @@ -54,22 +54,22 @@ class Header: coinbase: Optional[FixedSizeBytesConvertible] = None state_root: Optional[FixedSizeBytesConvertible] = None transactions_root: Optional[FixedSizeBytesConvertible] = None - receipt_root: Optional[FixedSizeBytesConvertible] = None - bloom: Optional[FixedSizeBytesConvertible] = None + receipts_root: Optional[FixedSizeBytesConvertible] = None + logs_bloom: Optional[FixedSizeBytesConvertible] = None difficulty: Optional[NumberConvertible] = None number: Optional[NumberConvertible] = None gas_limit: Optional[NumberConvertible] = None gas_used: Optional[NumberConvertible] = None timestamp: Optional[NumberConvertible] = None extra_data: Optional[BytesConvertible] = None - mix_digest: Optional[FixedSizeBytesConvertible] = None + prev_randao: Optional[FixedSizeBytesConvertible] = None nonce: Optional[FixedSizeBytesConvertible] = None - base_fee: Optional[NumberConvertible | Removable] = None + base_fee_per_gas: Optional[NumberConvertible | Removable] = None withdrawals_root: Optional[FixedSizeBytesConvertible | Removable] = None blob_gas_used: Optional[NumberConvertible | Removable] = None excess_blob_gas: Optional[NumberConvertible | Removable] = None beacon_root: Optional[FixedSizeBytesConvertible | Removable] = None - hash: Optional[FixedSizeBytesConvertible] = None + block_hash: Optional[FixedSizeBytesConvertible] = None REMOVE_FIELD: ClassVar[Removable] = Removable() """ @@ -220,19 +220,19 @@ class FixtureHeader: ), json_encoder=JSONEncoder.Field(name="transactionsTrie"), ) - receipt_root: Hash = header_field( + receipts_root: Hash = header_field( source=HeaderFieldSource( parse_type=Hash, source_transition_tool="receiptsRoot", ), json_encoder=JSONEncoder.Field(name="receiptTrie"), ) - bloom: Bloom = header_field( + logs_bloom: Bloom = header_field( source=HeaderFieldSource( parse_type=Bloom, source_transition_tool="logsBloom", ), - json_encoder=JSONEncoder.Field(), + json_encoder=JSONEncoder.Field(name="bloom"), ) difficulty: int = header_field( source=HeaderFieldSource( @@ -279,7 +279,7 @@ class FixtureHeader: ), json_encoder=JSONEncoder.Field(name="extraData"), ) - mix_digest: Hash = header_field( + prev_randao: Hash = header_field( source=HeaderFieldSource( parse_type=Hash, source_environment="prev_randao", @@ -294,13 +294,13 @@ class FixtureHeader: ), json_encoder=JSONEncoder.Field(), ) - base_fee: Optional[int] = header_field( + base_fee_per_gas: Optional[int] = header_field( default=None, source=HeaderFieldSource( parse_type=Number, - fork_requirement_check="header_base_fee_required", + fork_requirement_check="header_base_fee_per_gas_required", source_transition_tool="currentBaseFee", - source_environment="base_fee", + source_environment="base_fee_per_gas", ), json_encoder=JSONEncoder.Field(name="baseFeePerGas", cast_type=ZeroPaddedHexNumber), ) @@ -340,12 +340,12 @@ class FixtureHeader: ), json_encoder=JSONEncoder.Field(name="parentBeaconBlockRoot"), ) - hash: Optional[Hash] = header_field( + block_hash: Optional[Hash] = header_field( default=None, source=HeaderFieldSource( required=False, ), - json_encoder=JSONEncoder.Field(), + json_encoder=JSONEncoder.Field(name="hash"), ) @classmethod @@ -464,19 +464,19 @@ def build( self.coinbase, self.state_root, self.transactions_root, - self.receipt_root, - self.bloom, + self.receipts_root, + self.logs_bloom, Uint(int(self.difficulty)), Uint(int(self.number)), Uint(int(self.gas_limit)), Uint(int(self.gas_used)), Uint(int(self.timestamp)), self.extra_data, - self.mix_digest, + self.prev_randao, self.nonce, ] - if self.base_fee is not None: - header.append(Uint(int(self.base_fee))) + if self.base_fee_per_gas is not None: + header.append(Uint(int(self.base_fee_per_gas))) if self.withdrawals_root is not None: header.append(self.withdrawals_root) if self.blob_gas_used is not None: @@ -563,8 +563,8 @@ def set_environment(self, env: Environment) -> Environment: new_env.gas_limit = ( self.gas_limit if self.gas_limit is not None else environment_default.gas_limit ) - if not isinstance(self.base_fee, Removable): - new_env.base_fee = self.base_fee + if not isinstance(self.base_fee_per_gas, Removable): + new_env.base_fee_per_gas = self.base_fee_per_gas new_env.withdrawals = self.withdrawals if not isinstance(self.excess_blob_gas, Removable): new_env.excess_blob_gas = self.excess_blob_gas @@ -642,22 +642,22 @@ class FixtureExecutionPayload(FixtureHeader): name="feeRecipient", ) ) - receipt_root: Hash = field( + receipts_root: Hash = field( json_encoder=JSONEncoder.Field( name="receiptsRoot", ), ) - bloom: Bloom = field( + logs_bloom: Bloom = field( json_encoder=JSONEncoder.Field( name="logsBloom", ) ) - mix_digest: Hash = field( + prev_randao: Hash = field( json_encoder=JSONEncoder.Field( name="prevRandao", ), ) - hash: Optional[Hash] = field( + block_hash: Optional[Hash] = field( default=None, json_encoder=JSONEncoder.Field( name="blockHash", @@ -674,7 +674,7 @@ class FixtureExecutionPayload(FixtureHeader): gas_limit: int = field(json_encoder=JSONEncoder.Field(name="gasLimit", cast_type=HexNumber)) gas_used: int = field(json_encoder=JSONEncoder.Field(name="gasUsed", cast_type=HexNumber)) timestamp: int = field(json_encoder=JSONEncoder.Field(cast_type=HexNumber)) - base_fee: Optional[int] = field( + base_fee_per_gas: Optional[int] = field( default=None, json_encoder=JSONEncoder.Field(name="baseFeePerGas", cast_type=HexNumber), ) @@ -1003,7 +1003,7 @@ class FixtureBlock: cast_type=Number, ), ) - txs: List[Transaction] = field( + transactions: List[Transaction] = field( json_encoder=JSONEncoder.Field( name="transactions", cast_type=lambda txs: [FixtureTransaction.from_transaction(tx) for tx in txs], diff --git a/src/ethereum_test_tools/spec/state/types.py b/src/ethereum_test_tools/spec/state/types.py index 88e0d9fa131..99f0707d195 100644 --- a/src/ethereum_test_tools/spec/state/types.py +++ b/src/ethereum_test_tools/spec/state/types.py @@ -1,6 +1,7 @@ """ StateTest types """ + import json from dataclasses import dataclass, fields from pathlib import Path @@ -64,7 +65,7 @@ class FixtureEnvironment: cast_type=ZeroPaddedHexNumber, ), ) - base_fee: Optional[NumberConvertible] = field( + base_fee_per_gas: Optional[NumberConvertible] = field( default=None, json_encoder=JSONEncoder.Field( name="currentBaseFee", diff --git a/src/ethereum_test_tools/tests/test_consume/engine_json/valid_simple_cancun_enp.json b/src/ethereum_test_tools/tests/test_consume/engine_json/valid_simple_cancun_enp.json new file mode 100644 index 00000000000..d64c236c5b3 --- /dev/null +++ b/src/ethereum_test_tools/tests/test_consume/engine_json/valid_simple_cancun_enp.json @@ -0,0 +1,25 @@ +[ + { + "parentHash": "0xda9249b7aff004bcdfadfb5f668899746e36a5eee8197d1589deb4a3842251ce", + "feeRecipient": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "stateRoot": "0x3d760432d38fbf795fb9addd6b25a692d82498bd5f7b703a6da6d8647c5b7820", + "receiptsRoot": "0xc598f69a5674cae9337261b669970e24abc0b46e6d284372a239ec8ccbf20b0a", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "prevRandao": "0x0000000000000000000000000000000000000000000000000000000000000000", + "blockNumber": "0x1", + "gasLimit": "0x16345785d8a0000", + "gasUsed": "0xa861", + "timestamp": "0xc", + "extraData": "0x", + "baseFeePerGas": "0x7", + "blockHash": "0x546546d8a2d99b3135a47debdfc708e6a2199b8d90e43325d2c0b3adc3613709", + "transactions": [ + "0xf861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509b" + ], + "withdrawals": [], + "blobGasUsed": "0x0", + "excessBlobGas": "0x0" + }, + [], + "0x0000000000000000000000000000000000000000000000000000000000000000" +] \ No newline at end of file diff --git a/src/ethereum_test_tools/tests/test_consume/engine_json/valid_simple_paris_enp.json b/src/ethereum_test_tools/tests/test_consume/engine_json/valid_simple_paris_enp.json new file mode 100644 index 00000000000..808b234beaa --- /dev/null +++ b/src/ethereum_test_tools/tests/test_consume/engine_json/valid_simple_paris_enp.json @@ -0,0 +1,18 @@ +[ + { + "parentHash": "0x86c6dc9cb7b8ada9e27b1cf16fd81f366a0ad8127f42ff13d778eb2ddf7eaa90", + "feeRecipient": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "stateRoot": "0x19919608275963e6e20a1191996f5b19db8208dd8df54097cfd2b9cb14f682b6", + "receiptsRoot": "0xc598f69a5674cae9337261b669970e24abc0b46e6d284372a239ec8ccbf20b0a", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "prevRandao": "0x0000000000000000000000000000000000000000000000000000000000000000", + "blockNumber": "0x1", + "gasLimit": "0x16345785d8a0000", + "gasUsed": "0xa861", + "timestamp": "0xc", + "extraData": "0x", + "baseFeePerGas": "0x7", + "blockHash": "0xe9694e4b99986d312c6891cd7839b73d9e1b451537896818cefeeae97d7e3ea6", + "transactions": ["0xf861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509b"] + } +] \ No newline at end of file diff --git a/src/ethereum_test_tools/tests/test_consume/engine_json/valid_simple_shanghai_enp.json b/src/ethereum_test_tools/tests/test_consume/engine_json/valid_simple_shanghai_enp.json new file mode 100644 index 00000000000..53c038ff4d8 --- /dev/null +++ b/src/ethereum_test_tools/tests/test_consume/engine_json/valid_simple_shanghai_enp.json @@ -0,0 +1,19 @@ +[ + { + "parentHash": "0xccb89b5b6043aa73114e6857f0783a02808ea6ff4cabd104a308eb4fe0114a9b", + "feeRecipient": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "stateRoot": "0x19919608275963e6e20a1191996f5b19db8208dd8df54097cfd2b9cb14f682b6", + "receiptsRoot": "0xc598f69a5674cae9337261b669970e24abc0b46e6d284372a239ec8ccbf20b0a", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "prevRandao": "0x0000000000000000000000000000000000000000000000000000000000000000", + "blockNumber": "0x1", + "gasLimit": "0x16345785d8a0000", + "gasUsed": "0xa861", + "timestamp": "0xc", + "extraData": "0x", + "baseFeePerGas": "0x7", + "blockHash": "0xc970b6bcf304cd5c71d24548a7d65dd907a24a3b66229378e2ac42677c1eec2b", + "transactions": ["0xf861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509b"], + "withdrawals": [] + } +] \ No newline at end of file diff --git a/src/ethereum_test_tools/tests/test_consume/fixtures/valid_simple_cancun_blockchain.json b/src/ethereum_test_tools/tests/test_consume/fixtures/valid_simple_cancun_blockchain.json new file mode 100644 index 00000000000..f00b9037752 --- /dev/null +++ b/src/ethereum_test_tools/tests/test_consume/fixtures/valid_simple_cancun_blockchain.json @@ -0,0 +1,128 @@ +{ + "000/my_blockchain_test/Cancun": { + "network": "Cancun", + "genesisRLP": "0xf90240f9023aa00000000000000000000000000000000000000000000000000000000000000000a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347940000000000000000000000000000000000000000a0789d559bf5d313e15da4139b57627160d23146cf6cdf9995e0394d165b1527efa056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b9010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000808088016345785d8a0000808000a0000000000000000000000000000000000000000000000000000000000000000088000000000000000007a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b4218080a00000000000000000000000000000000000000000000000000000000000000000c0c0c0", + "genesisBlockHeader": { + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "uncleHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "coinbase": "0x0000000000000000000000000000000000000000", + "stateRoot": "0x789d559bf5d313e15da4139b57627160d23146cf6cdf9995e0394d165b1527ef", + "transactionsTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "difficulty": "0x00", + "number": "0x00", + "gasLimit": "0x016345785d8a0000", + "gasUsed": "0x00", + "timestamp": "0x00", + "extraData": "0x00", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "baseFeePerGas": "0x07", + "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "blobGasUsed": "0x00", + "excessBlobGas": "0x00", + "parentBeaconBlockRoot": "0x0000000000000000000000000000000000000000000000000000000000000000", + "hash": "0xda9249b7aff004bcdfadfb5f668899746e36a5eee8197d1589deb4a3842251ce" + }, + "blocks": [ + { + "rlp": "0xf902a6f9023ca0da9249b7aff004bcdfadfb5f668899746e36a5eee8197d1589deb4a3842251cea01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa03d760432d38fbf795fb9addd6b25a692d82498bd5f7b703a6da6d8647c5b7820a08151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcba0c598f69a5674cae9337261b669970e24abc0b46e6d284372a239ec8ccbf20b0ab9010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800188016345785d8a000082a8610c80a0000000000000000000000000000000000000000000000000000000000000000088000000000000000007a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b4218080a00000000000000000000000000000000000000000000000000000000000000000f863f861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509bc0c0", + "blockHeader": { + "parentHash": "0xda9249b7aff004bcdfadfb5f668899746e36a5eee8197d1589deb4a3842251ce", + "uncleHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "coinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "stateRoot": "0x3d760432d38fbf795fb9addd6b25a692d82498bd5f7b703a6da6d8647c5b7820", + "transactionsTrie": "0x8151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcb", + "receiptTrie": "0xc598f69a5674cae9337261b669970e24abc0b46e6d284372a239ec8ccbf20b0a", + "bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "difficulty": "0x00", + "number": "0x01", + "gasLimit": "0x016345785d8a0000", + "gasUsed": "0xa861", + "timestamp": "0x0c", + "extraData": "0x", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "baseFeePerGas": "0x07", + "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "blobGasUsed": "0x00", + "excessBlobGas": "0x00", + "parentBeaconBlockRoot": "0x0000000000000000000000000000000000000000000000000000000000000000", + "hash": "0x546546d8a2d99b3135a47debdfc708e6a2199b8d90e43325d2c0b3adc3613709" + }, + "blocknumber": "1", + "transactions": [ + { + "type": "0x00", + "chainId": "0x00", + "nonce": "0x00", + "gasPrice": "0x0a", + "gasLimit": "0x05f5e100", + "to": "0x1000000000000000000000000000000000000000", + "value": "0x00", + "data": "0x", + "v": "0x1b", + "r": "0x7e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37", + "s": "0x5f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509b", + "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b" + } + ], + "uncleHeaders": [], + "withdrawals": [] + } + ], + "lastblockhash": "0x546546d8a2d99b3135a47debdfc708e6a2199b8d90e43325d2c0b3adc3613709", + "pre": { + "0x000f3df6d732807ef1319fb7b8bb8522d0beac02": { + "nonce": "0x01", + "balance": "0x00", + "code": "0x3373fffffffffffffffffffffffffffffffffffffffe14604d57602036146024575f5ffd5b5f35801560495762001fff810690815414603c575f5ffd5b62001fff01545f5260205ff35b5f5ffd5b62001fff42064281555f359062001fff015500", + "storage": {} + }, + "0x1000000000000000000000000000000000000000": { + "nonce": "0x00", + "balance": "0x00", + "code": "0x4660015500", + "storage": {} + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "nonce": "0x00", + "balance": "0x3635c9adc5dea00000", + "code": "0x", + "storage": {} + } + }, + "postState": { + "0x000f3df6d732807ef1319fb7b8bb8522d0beac02": { + "nonce": "0x01", + "balance": "0x00", + "code": "0x3373fffffffffffffffffffffffffffffffffffffffe14604d57602036146024575f5ffd5b5f35801560495762001fff810690815414603c575f5ffd5b62001fff01545f5260205ff35b5f5ffd5b62001fff42064281555f359062001fff015500", + "storage": { + "0x0c": "0x0c" + } + }, + "0x1000000000000000000000000000000000000000": { + "nonce": "0x00", + "balance": "0x00", + "code": "0x4660015500", + "storage": { + "0x01": "0x01" + } + }, + "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba": { + "nonce": "0x00", + "balance": "0x01f923", + "code": "0x", + "storage": {} + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "nonce": "0x01", + "balance": "0x3635c9adc5de996c36", + "code": "0x", + "storage": {} + } + }, + "sealEngine": "NoProof" + } +} \ No newline at end of file diff --git a/src/ethereum_test_tools/tests/test_consume/fixtures/valid_simple_paris_blockchain.json b/src/ethereum_test_tools/tests/test_consume/fixtures/valid_simple_paris_blockchain.json new file mode 100644 index 00000000000..5e49218f227 --- /dev/null +++ b/src/ethereum_test_tools/tests/test_consume/fixtures/valid_simple_paris_blockchain.json @@ -0,0 +1,105 @@ +{ + "000/my_blockchain_test/Paris": { + "network": "Merge", + "genesisRLP": "0xf901fbf901f6a00000000000000000000000000000000000000000000000000000000000000000a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347940000000000000000000000000000000000000000a0aff9f63320a482f8c4e4f15f659e6a7ac382138fbbb6919243b0cba4c5988a5aa056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b9010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000808088016345785d8a0000808000a0000000000000000000000000000000000000000000000000000000000000000088000000000000000007c0c0", + "genesisBlockHeader": { + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "uncleHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "coinbase": "0x0000000000000000000000000000000000000000", + "stateRoot": "0xaff9f63320a482f8c4e4f15f659e6a7ac382138fbbb6919243b0cba4c5988a5a", + "transactionsTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "difficulty": "0x00", + "number": "0x00", + "gasLimit": "0x016345785d8a0000", + "gasUsed": "0x00", + "timestamp": "0x00", + "extraData": "0x00", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "baseFeePerGas": "0x07", + "hash": "0x86c6dc9cb7b8ada9e27b1cf16fd81f366a0ad8127f42ff13d778eb2ddf7eaa90" + }, + "blocks": [ + { + "rlp": "0xf90261f901f8a086c6dc9cb7b8ada9e27b1cf16fd81f366a0ad8127f42ff13d778eb2ddf7eaa90a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa019919608275963e6e20a1191996f5b19db8208dd8df54097cfd2b9cb14f682b6a08151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcba0c598f69a5674cae9337261b669970e24abc0b46e6d284372a239ec8ccbf20b0ab9010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800188016345785d8a000082a8610c80a0000000000000000000000000000000000000000000000000000000000000000088000000000000000007f863f861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509bc0", + "blockHeader": { + "parentHash": "0x86c6dc9cb7b8ada9e27b1cf16fd81f366a0ad8127f42ff13d778eb2ddf7eaa90", + "uncleHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "coinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "stateRoot": "0x19919608275963e6e20a1191996f5b19db8208dd8df54097cfd2b9cb14f682b6", + "transactionsTrie": "0x8151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcb", + "receiptTrie": "0xc598f69a5674cae9337261b669970e24abc0b46e6d284372a239ec8ccbf20b0a", + "bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "difficulty": "0x00", + "number": "0x01", + "gasLimit": "0x016345785d8a0000", + "gasUsed": "0xa861", + "timestamp": "0x0c", + "extraData": "0x", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "baseFeePerGas": "0x07", + "hash": "0xe9694e4b99986d312c6891cd7839b73d9e1b451537896818cefeeae97d7e3ea6" + }, + "blocknumber": "1", + "transactions": [ + { + "type": "0x00", + "chainId": "0x00", + "nonce": "0x00", + "gasPrice": "0x0a", + "gasLimit": "0x05f5e100", + "to": "0x1000000000000000000000000000000000000000", + "value": "0x00", + "data": "0x", + "v": "0x1b", + "r": "0x7e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37", + "s": "0x5f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509b", + "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b" + } + ], + "uncleHeaders": [] + } + ], + "lastblockhash": "0xe9694e4b99986d312c6891cd7839b73d9e1b451537896818cefeeae97d7e3ea6", + "pre": { + "0x1000000000000000000000000000000000000000": { + "nonce": "0x00", + "balance": "0x00", + "code": "0x4660015500", + "storage": {} + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "nonce": "0x00", + "balance": "0x3635c9adc5dea00000", + "code": "0x", + "storage": {} + } + }, + "postState": { + "0x1000000000000000000000000000000000000000": { + "nonce": "0x00", + "balance": "0x00", + "code": "0x4660015500", + "storage": { + "0x01": "0x01" + } + }, + "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba": { + "nonce": "0x00", + "balance": "0x01f923", + "code": "0x", + "storage": {} + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "nonce": "0x01", + "balance": "0x3635c9adc5de996c36", + "code": "0x", + "storage": {} + } + }, + "sealEngine": "NoProof" + } +} \ No newline at end of file diff --git a/src/ethereum_test_tools/tests/test_consume/fixtures/valid_simple_shanghai_blockchain.json b/src/ethereum_test_tools/tests/test_consume/fixtures/valid_simple_shanghai_blockchain.json new file mode 100644 index 00000000000..302b2dcabba --- /dev/null +++ b/src/ethereum_test_tools/tests/test_consume/fixtures/valid_simple_shanghai_blockchain.json @@ -0,0 +1,108 @@ +{ + "000/my_blockchain_test/Shanghai": { + "network": "Shanghai", + "genesisRLP": "0xf9021df90217a00000000000000000000000000000000000000000000000000000000000000000a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347940000000000000000000000000000000000000000a0aff9f63320a482f8c4e4f15f659e6a7ac382138fbbb6919243b0cba4c5988a5aa056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b9010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000808088016345785d8a0000808000a0000000000000000000000000000000000000000000000000000000000000000088000000000000000007a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421c0c0c0", + "genesisBlockHeader": { + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "uncleHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "coinbase": "0x0000000000000000000000000000000000000000", + "stateRoot": "0xaff9f63320a482f8c4e4f15f659e6a7ac382138fbbb6919243b0cba4c5988a5a", + "transactionsTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "difficulty": "0x00", + "number": "0x00", + "gasLimit": "0x016345785d8a0000", + "gasUsed": "0x00", + "timestamp": "0x00", + "extraData": "0x00", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "baseFeePerGas": "0x07", + "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "hash": "0xccb89b5b6043aa73114e6857f0783a02808ea6ff4cabd104a308eb4fe0114a9b" + }, + "blocks": [ + { + "rlp": "0xf90283f90219a0ccb89b5b6043aa73114e6857f0783a02808ea6ff4cabd104a308eb4fe0114a9ba01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa019919608275963e6e20a1191996f5b19db8208dd8df54097cfd2b9cb14f682b6a08151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcba0c598f69a5674cae9337261b669970e24abc0b46e6d284372a239ec8ccbf20b0ab9010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800188016345785d8a000082a8610c80a0000000000000000000000000000000000000000000000000000000000000000088000000000000000007a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421f863f861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509bc0c0", + "blockHeader": { + "parentHash": "0xccb89b5b6043aa73114e6857f0783a02808ea6ff4cabd104a308eb4fe0114a9b", + "uncleHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "coinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "stateRoot": "0x19919608275963e6e20a1191996f5b19db8208dd8df54097cfd2b9cb14f682b6", + "transactionsTrie": "0x8151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcb", + "receiptTrie": "0xc598f69a5674cae9337261b669970e24abc0b46e6d284372a239ec8ccbf20b0a", + "bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "difficulty": "0x00", + "number": "0x01", + "gasLimit": "0x016345785d8a0000", + "gasUsed": "0xa861", + "timestamp": "0x0c", + "extraData": "0x", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "baseFeePerGas": "0x07", + "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "hash": "0xc970b6bcf304cd5c71d24548a7d65dd907a24a3b66229378e2ac42677c1eec2b" + }, + "blocknumber": "1", + "transactions": [ + { + "type": "0x00", + "chainId": "0x00", + "nonce": "0x00", + "gasPrice": "0x0a", + "gasLimit": "0x05f5e100", + "to": "0x1000000000000000000000000000000000000000", + "value": "0x00", + "data": "0x", + "v": "0x1b", + "r": "0x7e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37", + "s": "0x5f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509b", + "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b" + } + ], + "uncleHeaders": [], + "withdrawals": [] + } + ], + "lastblockhash": "0xc970b6bcf304cd5c71d24548a7d65dd907a24a3b66229378e2ac42677c1eec2b", + "pre": { + "0x1000000000000000000000000000000000000000": { + "nonce": "0x00", + "balance": "0x00", + "code": "0x4660015500", + "storage": {} + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "nonce": "0x00", + "balance": "0x3635c9adc5dea00000", + "code": "0x", + "storage": {} + } + }, + "postState": { + "0x1000000000000000000000000000000000000000": { + "nonce": "0x00", + "balance": "0x00", + "code": "0x4660015500", + "storage": { + "0x01": "0x01" + } + }, + "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba": { + "nonce": "0x00", + "balance": "0x01f923", + "code": "0x", + "storage": {} + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "nonce": "0x01", + "balance": "0x3635c9adc5de996c36", + "code": "0x", + "storage": {} + } + }, + "sealEngine": "NoProof" + } +} \ No newline at end of file diff --git a/src/ethereum_test_tools/tests/test_consume/test_consume_engine_types.py b/src/ethereum_test_tools/tests/test_consume/test_consume_engine_types.py new file mode 100644 index 00000000000..20b9634861a --- /dev/null +++ b/src/ethereum_test_tools/tests/test_consume/test_consume_engine_types.py @@ -0,0 +1,226 @@ +""" +Test suite for `ethereum_test_tools.consume.types` module, with a focus on the engine payloads. +""" + +import json +import os +from typing import Any, Dict, Type, Union + +import pytest + +from ethereum_test_forks import Cancun, Fork, Paris, Shanghai +from ethereum_test_tools.common import Account, Environment, Hash, TestAddress, Transaction +from ethereum_test_tools.common.json import load_dataclass_from_json +from ethereum_test_tools.consume.engine.types import EngineCancun, EngineParis, EngineShanghai +from ethereum_test_tools.spec import BlockchainTest +from ethereum_test_tools.spec.blockchain.types import Block, Fixture, FixtureBlock +from evm_transition_tool import FixtureFormats, GethTransitionTool + + +def remove_info(fixture_json: Dict[str, Any]): # noqa: D103 + for t in fixture_json: + if "_info" in fixture_json[t]: + del fixture_json[t]["_info"] + + +common_execution_payload_fields = { + "coinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "receipts_root": "0xc598f69a5674cae9337261b669970e24abc0b46e6d284372a239ec8ccbf20b0a", + "logs_bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", # noqa: E501 + "prev_randao": "0x0000000000000000000000000000000000000000000000000000000000000000", + "number": 1, + "gas_limit": 100000000000000000, + "gas_used": 43105, + "timestamp": 12, + "extra_data": "0x", + "base_fee_per_gas": 7, + "transactions": [ + "0xf861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509b" # noqa: E501 + ], +} + + +@pytest.mark.parametrize( + "fork, expected_json_fixture, expected_engine_new_payload, expected_enp_json", + [ + ( + Paris, + "valid_simple_paris_blockchain.json", + EngineParis.NewPayloadV1( + execution_payload=EngineParis.ExecutionPayloadV1( + **common_execution_payload_fields, # type: ignore + parent_hash=Hash( + 0x86C6DC9CB7B8ADA9E27B1CF16FD81F366A0AD8127F42FF13D778EB2DDF7EAA90 + ), + state_root=Hash( + 0x19919608275963E6E20A1191996F5B19DB8208DD8DF54097CFD2B9CB14F682B6 + ), + block_hash=Hash( + 0xE9694E4B99986D312C6891CD7839B73D9E1B451537896818CEFEEAE97D7E3EA6 + ), + ) + ), + "valid_simple_paris_enp.json", + ), + ( + Shanghai, + "valid_simple_shanghai_blockchain.json", + EngineShanghai.NewPayloadV2( + execution_payload=EngineShanghai.ExecutionPayloadV2( + **common_execution_payload_fields, # type: ignore + parent_hash=Hash( + 0xCCB89B5B6043AA73114E6857F0783A02808EA6FF4CABD104A308EB4FE0114A9B + ), + state_root=Hash( + 0x19919608275963E6E20A1191996F5B19DB8208DD8DF54097CFD2B9CB14F682B6 + ), + block_hash=Hash( + 0xC970B6BCF304CD5C71D24548A7D65DD907A24A3B66229378E2AC42677C1EEC2B + ), + withdrawals=[], + ) + ), + "valid_simple_shanghai_enp.json", + ), + ( + Cancun, + "valid_simple_cancun_blockchain.json", + EngineCancun.NewPayloadV3( + execution_payload=EngineCancun.ExecutionPayloadV3( + **common_execution_payload_fields, # type: ignore + parent_hash=Hash( + 0xDA9249B7AFF004BCDFADFB5F668899746E36A5EEE8197D1589DEB4A3842251CE + ), + state_root=Hash( + 0x3D760432D38FBF795FB9ADDD6B25A692D82498BD5F7B703A6DA6D8647C5B7820 + ), + block_hash=Hash( + 0x546546D8A2D99B3135A47DEBDFC708E6A2199B8D90E43325D2C0B3ADC3613709 + ), + withdrawals=[], + blob_gas_used=0, + excess_blob_gas=0, + ), + expected_blob_versioned_hashes=[], + parent_beacon_block_root=Hash( + 0x0000000000000000000000000000000000000000000000000000000000000000 + ), + ), + "valid_simple_cancun_enp.json", + ), + ], +) +def test_valid_engine_new_payload_fields( + fork: Fork, + expected_json_fixture: str, + expected_engine_new_payload: Any, + expected_enp_json: str, +): + """ + Test ... + """ + # Create a blockchain test fixture + t8n = GethTransitionTool() + blockchain_fixture = BlockchainTest( # type: ignore + pre={ + 0x1000000000000000000000000000000000000000: Account(code="0x4660015500"), + TestAddress: Account(balance=1000000000000000000000), + }, + post={ + "0x1000000000000000000000000000000000000000": Account( + code="0x4660015500", storage={"0x01": "0x01"} + ), + }, + blocks=[ + Block( + txs=[ + Transaction( + ty=0x0, + chain_id=0x0, + nonce=0, + to="0x1000000000000000000000000000000000000000", + gas_limit=100000000, + gas_price=10, + protected=False, + ) + ] + ) + ], + genesis_environment=Environment(), + tag="my_blockchain_test_valid_txs", + fixture_format=FixtureFormats.BLOCKCHAIN_TEST, + ).generate( + t8n=t8n, + fork=fork, + ) + + # Sanity check the fixture is equal to the expected + with open( + os.path.join( + "src", + "ethereum_test_tools", + "tests", + "test_consume", + "fixtures", + expected_json_fixture, + ) + ) as f: + expected = json.load(f) + blockchain_fixture_json = { + f"000/my_blockchain_test/{fork.name()}": blockchain_fixture.to_json(), + } + remove_info(blockchain_fixture_json) + assert blockchain_fixture_json == expected + + # Load json fixture into Fixture dataclass + fixture: Fixture + for _, fixture_data in blockchain_fixture_json.items(): + fixture = load_dataclass_from_json(Fixture, fixture_data) + + # Extract the engine payloads from the fixture blocks + # Ideally we don't know the fork at this point + for fixture_block in fixture.blocks: + fixture_block = load_dataclass_from_json(FixtureBlock, fixture_block) # type: ignore + if fixture.fork == "Merge": + fork = Paris + else: + fork = globals()[fixture.fork] + version = fork.engine_new_payload_version( + fixture_block.block_header.number, fixture_block.block_header.timestamp # type: ignore + ) + PayloadClass: Type[ + Union[ + EngineParis.NewPayloadV1, + EngineShanghai.NewPayloadV2, + EngineCancun.NewPayloadV3, + ] + ] + if version == 1: + PayloadClass = EngineParis.NewPayloadV1 + elif version == 2: + PayloadClass = EngineShanghai.NewPayloadV2 + elif version == 3: + PayloadClass = EngineCancun.NewPayloadV3 + else: + ValueError(f"Unexpected payload version: {version}") + continue + + engine_new_payload = PayloadClass.from_fixture_block(fixture_block) # type: ignore + + # Compare the engine payloads with the expected payloads + assert engine_new_payload == expected_engine_new_payload + + # Check the json representation of the engine payloads that would be sent over json-rpc + with open( + os.path.join( + "src", + "ethereum_test_tools", + "tests", + "test_consume", + "engine_json", + expected_enp_json, + ) + ) as f: + expected = json.load(f) + print(engine_new_payload.to_json_rpc()) + assert engine_new_payload.to_json_rpc() == expected diff --git a/src/ethereum_test_tools/tests/test_filling/test_fixtures.py b/src/ethereum_test_tools/tests/test_filling/test_fixtures.py index 113bd889c43..b19eb0ec8fa 100644 --- a/src/ethereum_test_tools/tests/test_filling/test_fixtures.py +++ b/src/ethereum_test_tools/tests/test_filling/test_fixtures.py @@ -89,8 +89,8 @@ def test_make_genesis(fork: Fork, hash: bytes): # noqa: D103 assert isinstance(fixture, BlockchainFixture) assert fixture.genesis is not None - assert fixture.genesis.hash is not None - assert fixture.genesis.hash.startswith(hash) + assert fixture.genesis.block_hash is not None + assert fixture.genesis.block_hash.startswith(hash) @pytest.mark.parametrize( @@ -450,7 +450,7 @@ def post(self): # noqa: D102 @pytest.fixture def genesis_environment(self): # noqa: D102 return Environment( - base_fee=1000, + base_fee_per_gas=1000, coinbase="0xba5e000000000000000000000000000000000000", ) @@ -544,7 +544,7 @@ def test_fixture_header_join(self, blockchain_test_fixture: BlockchainFixture): assert updated_block_header.difficulty == new_difficulty assert updated_block_header.state_root == new_state_root assert updated_block_header.transactions_root == Hash(new_transactions_root) - assert updated_block_header.hash == block.block_header.hash # type: ignore + assert updated_block_header.block_hash == block.block_header.block_hash # type: ignore assert isinstance(updated_block_header.transactions_root, Hash) @@ -851,7 +851,7 @@ def test_fill_blockchain_invalid_txs( # We start genesis with a baseFee of 1000 genesis_environment = Environment( - base_fee=1000, + base_fee_per_gas=1000, coinbase="0xba5e000000000000000000000000000000000000", ) diff --git a/src/ethereum_test_tools/tests/test_types.py b/src/ethereum_test_tools/tests/test_types.py index 54b130eac70..a348f3978ff 100644 --- a/src/ethereum_test_tools/tests/test_types.py +++ b/src/ethereum_test_tools/tests/test_types.py @@ -439,10 +439,10 @@ def test_account_merge( coinbase=0x1234, difficulty=0x5, prev_randao=0x6, - base_fee=0x7, + base_fee_per_gas=0x7, parent_difficulty=0x8, parent_timestamp=0x9, - parent_base_fee=0xA, + parent_base_fee_per_gas=0xA, parent_gas_used=0xB, parent_gas_limit=0xC, parent_ommers_hash=0xD, @@ -737,15 +737,15 @@ def test_account_merge( coinbase=Address(2), state_root=Hash(3), transactions_root=Hash(4), - receipt_root=Hash(5), - bloom=Bloom(6), + receipts_root=Hash(5), + logs_bloom=Bloom(6), difficulty=7, number=8, gas_limit=9, gas_used=10, timestamp=11, extra_data=Bytes([12]), - mix_digest=Hash(13), + prev_randao=Hash(13), nonce=HeaderNonce(14), ), { @@ -774,21 +774,21 @@ def test_account_merge( coinbase=Address(2), state_root=Hash(3), transactions_root=Hash(4), - receipt_root=Hash(5), - bloom=Bloom(6), + receipts_root=Hash(5), + logs_bloom=Bloom(6), difficulty=7, number=8, gas_limit=9, gas_used=10, timestamp=11, extra_data=Bytes([12]), - mix_digest=Hash(13), + prev_randao=Hash(13), nonce=HeaderNonce(14), - base_fee=15, + base_fee_per_gas=15, withdrawals_root=Hash(16), blob_gas_used=17, excess_blob_gas=18, - hash=Hash(19), + block_hash=Hash(19), ), { "parentHash": Hash(0).hex(), @@ -822,21 +822,21 @@ def test_account_merge( coinbase=Address(2), state_root=Hash(3), transactions_root=Hash(4), - receipt_root=Hash(5), - bloom=Bloom(6), + receipts_root=Hash(5), + logs_bloom=Bloom(6), difficulty=7, number=8, gas_limit=9, gas_used=10, timestamp=11, extra_data=Bytes([12]), - mix_digest=Hash(13), + prev_randao=Hash(13), nonce=HeaderNonce(14), - base_fee=15, + base_fee_per_gas=15, withdrawals_root=Hash(16), blob_gas_used=17, excess_blob_gas=18, - hash=Hash(19), + block_hash=Hash(19), ), transactions=[ Transaction( @@ -907,21 +907,21 @@ def test_account_merge( coinbase=Address(2), state_root=Hash(3), transactions_root=Hash(4), - receipt_root=Hash(5), - bloom=Bloom(6), + receipts_root=Hash(5), + logs_bloom=Bloom(6), difficulty=7, number=8, gas_limit=9, gas_used=10, timestamp=11, extra_data=Bytes([12]), - mix_digest=Hash(13), + prev_randao=Hash(13), nonce=HeaderNonce(14), - base_fee=15, + base_fee_per_gas=15, withdrawals_root=Hash(16), blob_gas_used=17, excess_blob_gas=18, - hash=Hash(19), + block_hash=Hash(19), ), transactions=[ Transaction( @@ -999,21 +999,21 @@ def test_account_merge( coinbase=Address(2), state_root=Hash(3), transactions_root=Hash(4), - receipt_root=Hash(5), - bloom=Bloom(6), + receipts_root=Hash(5), + logs_bloom=Bloom(6), difficulty=7, number=8, gas_limit=9, gas_used=10, timestamp=11, extra_data=Bytes([12]), - mix_digest=Hash(13), + prev_randao=Hash(13), nonce=HeaderNonce(14), - base_fee=15, + base_fee_per_gas=15, withdrawals_root=Hash(16), blob_gas_used=17, excess_blob_gas=18, - hash=Hash(19), + block_hash=Hash(19), ), transactions=[ Transaction( diff --git a/src/ethereum_test_tools/tests/test_types_blockchain_test.py b/src/ethereum_test_tools/tests/test_types_blockchain_test.py index 1f809ca0e9d..f0ecb588f1c 100644 --- a/src/ethereum_test_tools/tests/test_types_blockchain_test.py +++ b/src/ethereum_test_tools/tests/test_types_blockchain_test.py @@ -1,6 +1,7 @@ """ Test the blockchain test types. """ + from dataclasses import replace import pytest @@ -14,21 +15,21 @@ coinbase=Address(1), state_root=Hash(1), transactions_root=Hash(1), - receipt_root=Hash(1), - bloom=Bloom(1), + receipts_root=Hash(1), + logs_bloom=Bloom(1), difficulty=1, number=1, gas_limit=1, gas_used=1, timestamp=1, extra_data=Bytes([1]), - mix_digest=Hash(1), + prev_randao=Hash(1), nonce=HeaderNonce(1), - base_fee=1, + base_fee_per_gas=1, withdrawals_root=Hash(1), blob_gas_used=1, excess_blob_gas=1, - hash=Hash(1), + block_hash=Hash(1), ) @@ -73,26 +74,28 @@ ), pytest.param( fixture_header_ones, - Header(bloom="0x100"), - replace(fixture_header_ones, bloom=Bloom("0x100")), - id="bloom_as_str", + Header(logs_bloom="0x100"), + replace(fixture_header_ones, logs_bloom=Bloom("0x100")), + id="logs_bloom_as_str", ), pytest.param( fixture_header_ones, - Header(bloom=100), - replace(fixture_header_ones, bloom=Bloom(100)), - id="bloom_as_int", + Header(logs_bloom=100), + replace(fixture_header_ones, logs_bloom=Bloom(100)), + id="logs_bloom_as_int", ), pytest.param( fixture_header_ones, - Header(bloom=Hash(100)), - replace(fixture_header_ones, bloom=Bloom(100)), - id="bloom_as_hash", + Header(logs_bloom=Hash(100)), + replace(fixture_header_ones, logs_bloom=Bloom(100)), + id="logs_bloom_as_hash", ), pytest.param( fixture_header_ones, - Header(state_root="0x100", bloom=Hash(200), difficulty=300), - replace(fixture_header_ones, state_root=Hash(0x100), bloom=Bloom(200), difficulty=300), + Header(state_root="0x100", logs_bloom=Hash(200), difficulty=300), + replace( + fixture_header_ones, state_root=Hash(0x100), logs_bloom=Bloom(200), difficulty=300 + ), id="multiple_fields", ), ], diff --git a/src/evm_transition_tool/tests/test_evaluate.py b/src/evm_transition_tool/tests/test_evaluate.py index 57e15060cff..802c4dffef4 100644 --- a/src/evm_transition_tool/tests/test_evaluate.py +++ b/src/evm_transition_tool/tests/test_evaluate.py @@ -15,7 +15,7 @@ @pytest.mark.parametrize("t8n", [GethTransitionTool()]) @pytest.mark.parametrize("fork", [London, Istanbul]) @pytest.mark.parametrize( - "alloc,base_fee,hash", + "alloc,base_fee_per_gas,hash", [ ( { @@ -68,14 +68,14 @@ def test_calc_state_root( # noqa: D103 t8n: TransitionTool, fork: Fork, alloc: Dict, - base_fee: int | None, + base_fee_per_gas: int | None, hash: bytes, ) -> None: class TestEnv: - base_fee: int | None + base_fee_per_gas: int | None env = TestEnv() - env.base_fee = base_fee + env.base_fee_per_gas = base_fee_per_gas assert t8n.calc_state_root(alloc=alloc, fork=fork)[1].startswith(hash) diff --git a/src/evm_transition_tool/transition_tool.py b/src/evm_transition_tool/transition_tool.py index 4a981148932..63128c02931 100644 --- a/src/evm_transition_tool/transition_tool.py +++ b/src/evm_transition_tool/transition_tool.py @@ -1,6 +1,7 @@ """ Transition tool abstract class. """ + import json import os import shutil @@ -576,7 +577,7 @@ def calc_state_root( "currentTimestamp": "0", } - if fork.header_base_fee_required(0, 0): + if fork.header_base_fee_per_gas_required(0, 0): env["currentBaseFee"] = "7" if fork.header_prev_randao_required(0, 0): diff --git a/tests/cancun/eip4844_blobs/test_excess_blob_gas.py b/tests/cancun/eip4844_blobs/test_excess_blob_gas.py index f841e790268..f16580ec8cf 100644 --- a/tests/cancun/eip4844_blobs/test_excess_blob_gas.py +++ b/tests/cancun/eip4844_blobs/test_excess_blob_gas.py @@ -20,6 +20,7 @@ cases. """ # noqa: E501 + import itertools from typing import Dict, Iterator, List, Mapping, Optional, Tuple @@ -108,29 +109,31 @@ def block_fee_per_blob_gas( # noqa: D103 @pytest.fixture -def block_base_fee() -> int: # noqa: D103 +def block_base_fee_per_gas() -> int: # noqa: D103 return 7 @pytest.fixture def env( # noqa: D103 parent_excess_blob_gas: int, - block_base_fee: int, + block_base_fee_per_gas: int, parent_blobs: int, ) -> Environment: return Environment( - excess_blob_gas=parent_excess_blob_gas - if parent_blobs == 0 - else parent_excess_blob_gas + Spec.TARGET_BLOB_GAS_PER_BLOCK, - base_fee=block_base_fee, + excess_blob_gas=( + parent_excess_blob_gas + if parent_blobs == 0 + else parent_excess_blob_gas + Spec.TARGET_BLOB_GAS_PER_BLOCK + ), + base_fee_per_gas=block_base_fee_per_gas, ) @pytest.fixture def tx_max_fee_per_gas( # noqa: D103 - block_base_fee: int, + block_base_fee_per_gas: int, ) -> int: - return block_base_fee + return block_base_fee_per_gas @pytest.fixture @@ -340,9 +343,7 @@ def test_correct_excess_blob_gas_calculation( 2**32, # blob gas cost 2^32 2**64 // Spec.GAS_PER_BLOB, # Data tx wei cost 2^64 2**64, # blob gas cost 2^64 - ( - 120_000_000 * (10**18) // Spec.GAS_PER_BLOB - ), # Data tx wei is current total Ether supply + 120_000_000 * (10**18) // Spec.GAS_PER_BLOB, # Data tx wei is current total Ether supply ] ] diff --git a/tests_consume/test_via_engine_api.py b/tests_consume/test_via_engine_api.py index 539621efa4a..423fa31d9ab 100644 --- a/tests_consume/test_via_engine_api.py +++ b/tests_consume/test_via_engine_api.py @@ -5,9 +5,10 @@ Implemented using the pytest framework as a pytest plugin. """ + import io import json -from typing import Dict, List, Mapping +from typing import Dict, List, Mapping, Type, Union import pytest from hive.client import Client, ClientType @@ -26,8 +27,9 @@ ) from ethereum_test_tools.common.json import load_dataclass_from_json from ethereum_test_tools.common.types import Account +from ethereum_test_tools.consume.engine.types import EngineCancun, EngineParis, EngineShanghai from ethereum_test_tools.rpc import EngineRPC, EthRPC -from ethereum_test_tools.spec.blockchain.types import FixtureBlock, FixtureEngineNewPayload +from ethereum_test_tools.spec.blockchain.types import FixtureBlock from pytest_plugins.consume.consume import TestCase from pytest_plugins.consume_via_engine_api.client_fork_ruleset import client_fork_ruleset @@ -113,7 +115,9 @@ def eth_rpc(client: Client) -> EngineRPC: @pytest.fixture(scope="function") -def engine_new_payloads(test_case: TestCase) -> List[FixtureEngineNewPayload]: +def engine_new_payloads( + test_case: TestCase, +) -> List[Union[EngineShanghai, EngineCancun, EngineParis]]: """ Execution payloads extracted from each block within the test case fixture. Sent to the client under test using the `engine_newPayloadVX` method from the Engine API. @@ -122,29 +126,41 @@ def engine_new_payloads(test_case: TestCase) -> List[FixtureEngineNewPayload]: load_dataclass_from_json(FixtureBlock, block.get("rlp_decoded", block)) for block in test_case.fixture.blocks ] - - fixture_fork = test_case.fixture.fork - if fixture_fork == "Merge": - # TODO: Handle mapping following rename of Merge fork + if test_case.fixture.fork == "Merge": fork = Paris else: - fork = globals()[fixture_fork] - return [ - FixtureEngineNewPayload.from_fixture_header( - fork, - block.block_header, - block.txs, - block.withdrawals, - True, # TODO: Add support for InvalidFixtureBlock - error_code=None, + fork = globals()[test_case.fixture.fork] + engine_new_payloads = [] + for fixture_block in fixture_blocks: + version = fork.engine_new_payload_version( + fixture_block.block_header.number, fixture_block.block_header.timestamp # type: ignore ) - for block in fixture_blocks - ] + PayloadClass: Type[ + Union[ + EngineParis.NewPayloadV1, + EngineShanghai.NewPayloadV2, + EngineCancun.NewPayloadV3, + ] + ] + if version == 1: + PayloadClass = EngineParis.NewPayloadV1 + elif version == 2: + PayloadClass = EngineShanghai.NewPayloadV2 + elif version == 3: + PayloadClass = EngineCancun.NewPayloadV3 + else: + ValueError(f"Unexpected payload version: {version}") + continue + engine_new_payloads.append(PayloadClass.from_fixture_block(fixture_block)) + + return engine_new_payloads def test_via_engine_api( test_case: TestCase, - engine_new_payloads: List[FixtureEngineNewPayload], + engine_new_payloads: List[ + Union[EngineShanghai.NewPayloadV2, EngineCancun.NewPayloadV3, EngineParis.NewPayloadV1] + ], engine_rpc: EngineRPC, eth_rpc: EthRPC, ): @@ -168,9 +184,9 @@ def test_via_engine_api( # ), f"unexpected status: {payload_response} " forkchoice_response = engine_rpc.forkchoice_updated( - forkchoice_state={"headBlockHash": engine_new_payloads[-1].payload.hash}, + forkchoice_state={"headBlockHash": engine_new_payloads[-1].execution_payload.block_hash}, payload_attributes=None, - version=engine_new_payloads[-1].version, + version=engine_new_payloads[-1].version(), ) assert ( forkchoice_response["payloadStatus"]["status"] == "VALID" diff --git a/whitelist.txt b/whitelist.txt index a7a9235c500..2433d6a4fd0 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -82,6 +82,7 @@ eip eips EIPs endianness +enp enum env eof @@ -289,6 +290,8 @@ util utils v0 v1 +v2 +v3 validator venv visualstudio