From 0b3986bbb354e78b46a2498b295c21336a4b1d6c Mon Sep 17 00:00:00 2001 From: danceratopz Date: Wed, 26 Nov 2025 13:46:55 +0100 Subject: [PATCH 01/10] feat(consume): initial sequential implementation of enginex pytest-xdist is supported, but not optimised. --- .../cli/pytest_commands/consume.py | 11 +- .../plugins/consume/consume.py | 25 +- .../plugins/consume/simulators/__init__.py | 2 +- .../plugins/consume/simulators/base.py | 6 +- .../consume/simulators/engine/conftest.py | 26 +- .../plugins/consume/simulators/engine_api.py | 38 +++ .../consume/simulators/enginex/__init__.py | 1 + .../consume/simulators/enginex/conftest.py | 213 +++++++++++++ .../simulators/helpers/test_tracker.py | 115 +++++++ .../consume/simulators/multi_test_client.py | 284 ++++++++++++++++++ .../simulator_logic/test_via_engine.py | 104 ++++--- .../consume/simulators/sync/conftest.py | 16 +- .../plugins/pytest_hive/pytest_hive.py | 54 ++++ .../cli/pytest_commands/processors.py | 7 + whitelist.txt | 1 + 15 files changed, 828 insertions(+), 75 deletions(-) create mode 100644 packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/engine_api.py create mode 100644 packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/enginex/__init__.py create mode 100644 packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/enginex/conftest.py create mode 100644 packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/helpers/test_tracker.py create mode 100644 packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/multi_test_client.py diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/consume.py b/packages/testing/src/execution_testing/cli/pytest_commands/consume.py index 44364b89402..3059f463c05 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/consume.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/consume.py @@ -43,12 +43,13 @@ def create_consume_command( def get_command_logic_test_paths(command_name: str) -> List[Path]: """Determine the command paths based on the command name and hive flag.""" base_path = Path("cli/pytest_commands/plugins/consume") - if command_name in ["engine", "rlp"]: + if command_name in ["engine", "enginex", "rlp"]: + test_command = "engine" if command_name == "enginex" else command_name command_logic_test_paths = [ base_path / "simulators" / "simulator_logic" - / f"test_via_{command_name}.py" + / f"test_via_{test_command}.py" ] elif command_name == "sync": command_logic_test_paths = [ @@ -119,6 +120,12 @@ def engine() -> None: pass +@consume_command(is_hive=True) +def enginex() -> None: + """Consume via Engine API with pre-alloc optimization.""" + pass + + @consume_command(is_hive=True) def sync() -> None: """Client consumes via the Engine API with sync testing.""" diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/consume.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/consume.py index f95ad6bf1fa..68e8fdfcecf 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/consume.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/consume.py @@ -19,7 +19,11 @@ from execution_testing.cli.gen_index import ( generate_fixtures_index, ) -from execution_testing.fixtures import BaseFixture, FixtureFormat +from execution_testing.fixtures import ( + BaseFixture, + BlockchainEngineXFixture, + FixtureFormat, +) from execution_testing.fixtures.consume import IndexFile, TestCases from execution_testing.forks import ( get_forks, @@ -596,11 +600,26 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: fork_markers = get_relative_fork_markers( test_case.fork, strict_mode=False ) + + # Build base marks (fork and format) + marks = [getattr(pytest.mark, m) for m in fork_markers] + [ + getattr(pytest.mark, test_case.format.format_name) + ] + + # Add xdist_group marker for engine x tests to enable + # client reuse tracking + if test_case.format is BlockchainEngineXFixture: + assert hasattr(test_case, "pre_hash") and test_case.pre_hash, ( + f"BlockchainEngineXFixture test case " + f"'{test_case.id}' missing pre_hash" + ) + group_identifier = test_case.pre_hash + marks.append(pytest.mark.xdist_group(name=group_identifier)) + param = pytest.param( test_case, id=test_case.id, - marks=[getattr(pytest.mark, m) for m in fork_markers] - + [getattr(pytest.mark, test_case.format.format_name)], + marks=marks, ) param_list.append(param) diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/__init__.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/__init__.py index 858f14ab076..d2cafc02613 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/__init__.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/__init__.py @@ -1 +1 @@ -"""Consume hive simulators test functions.""" +"""Consume Hive simulators test functions.""" diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/base.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/base.py index cd80201b905..5780353cf68 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/base.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/base.py @@ -30,7 +30,11 @@ def check_live_port(test_suite_name: str) -> Literal[8545, 8551]: """Port used by hive to check for liveness of the client.""" if test_suite_name == "eels/consume-rlp": return 8545 - elif test_suite_name in {"eels/consume-engine", "eels/consume-sync"}: + elif test_suite_name in { + "eels/consume-engine", + "eels/consume-enginex", + "eels/consume-sync", + }: return 8551 raise ValueError( f"Unexpected test suite name '{test_suite_name}' while setting " diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/engine/conftest.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/engine/conftest.py index 518ed1799b9..12269050b11 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/engine/conftest.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/engine/conftest.py @@ -8,11 +8,9 @@ from typing import Mapping import pytest -from hive.client import Client -from execution_testing.exceptions import ExceptionMapper from execution_testing.fixtures import BlockchainEngineFixture -from execution_testing.rpc import EngineRPC +from execution_testing.fixtures.blockchain import FixtureHeader pytest_plugins = ( "execution_testing.cli.pytest_commands.plugins.pytest_hive.pytest_hive", @@ -21,6 +19,7 @@ "execution_testing.cli.pytest_commands.plugins.consume.simulators.test_case_description", "execution_testing.cli.pytest_commands.plugins.consume.simulators.timing_data", "execution_testing.cli.pytest_commands.plugins.consume.simulators.exceptions", + "execution_testing.cli.pytest_commands.plugins.consume.simulators.engine_api", ) @@ -29,21 +28,6 @@ def pytest_configure(config: pytest.Config) -> None: config.supported_fixture_formats = [BlockchainEngineFixture] # type: ignore[attr-defined] -@pytest.fixture(scope="function") -def engine_rpc( - client: Client, client_exception_mapper: ExceptionMapper | None -) -> EngineRPC: - """Initialize engine RPC client for the execution client under test.""" - if client_exception_mapper: - return EngineRPC( - f"http://{client.ip}:8551", - response_validation_context={ - "exception_mapper": client_exception_mapper, - }, - ) - return EngineRPC(f"http://{client.ip}:8551") - - @pytest.fixture(scope="module") def test_suite_name() -> str: """The name of the hive test suite used in this simulator.""" @@ -64,3 +48,9 @@ def client_files( files = {} files["/genesis.json"] = buffered_genesis return files + + +@pytest.fixture(scope="function") +def genesis_header(fixture: BlockchainEngineFixture) -> "FixtureHeader": + """Provide the genesis header from the fixture.""" + return fixture.genesis diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/engine_api.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/engine_api.py new file mode 100644 index 00000000000..93768cfdf2e --- /dev/null +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/engine_api.py @@ -0,0 +1,38 @@ +"""Pytest fixtures for Engine API RPC clients.""" + +import pytest +from hive.client import Client + +from execution_testing.exceptions import ExceptionMapper +from execution_testing.rpc import EngineRPC + + +@pytest.fixture(scope="function") +def engine_rpc( + client: Client, client_exception_mapper: ExceptionMapper | None +) -> EngineRPC: + """ + Initialize Engine RPC client for the execution client under test. + + Provide a configured EngineRPC instance that communicates + with the client's Engine API endpoint (port 8551). If an + exception mapper is available, it will be used for response + validation to map client-specific error messages to standard + exception types. + + Args: + client: The Hive client instance to connect to. + client_exception_mapper: Optional exception mapper. + + Returns: + Configured EngineRPC instance for making Engine API calls. + + """ + if client_exception_mapper: + return EngineRPC( + f"http://{client.ip}:8551", + response_validation_context={ + "exception_mapper": client_exception_mapper, + }, + ) + return EngineRPC(f"http://{client.ip}:8551") diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/enginex/__init__.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/enginex/__init__.py new file mode 100644 index 00000000000..83f66d1f3c0 --- /dev/null +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/enginex/__init__.py @@ -0,0 +1 @@ +"""Engine X simulator for `blockchain_test_engine_x` fixtures.""" diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/enginex/conftest.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/enginex/conftest.py new file mode 100644 index 00000000000..f5384389031 --- /dev/null +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/enginex/conftest.py @@ -0,0 +1,213 @@ +""" +Pytest fixtures for the `consume enginex` simulator. + +Configure the hive back-end & EL clients for test execution +with `BlockchainEngineXFixtures`. Use multi-test client +architecture to reuse clients across tests with the same +pre-alloc group. +""" + +import io +import json +import logging +from typing import Generator, cast + +import pytest +from hive.client import Client, ClientType +from hive.testing import HiveTest + +from execution_testing.fixtures import BlockchainEngineXFixture +from execution_testing.fixtures.blockchain import FixtureHeader +from execution_testing.fixtures.pre_alloc_groups import PreAllocGroup + +from ..helpers.test_tracker import ( + PreAllocGroupTestTracker, + enginex_group_counts_key, + format_group_identifier, +) +from ..multi_test_client import MultiTestClientManager +from ..timing_data import TimingData + +logger = logging.getLogger(__name__) + +pytest_plugins = ( + "execution_testing.cli.pytest_commands.plugins.pytest_hive.pytest_hive", + "execution_testing.cli.pytest_commands.plugins.consume.simulators.base", + "execution_testing.cli.pytest_commands.plugins.consume.simulators.multi_test_client", + "execution_testing.cli.pytest_commands.plugins.consume.simulators.test_case_description", + "execution_testing.cli.pytest_commands.plugins.consume.simulators.timing_data", + "execution_testing.cli.pytest_commands.plugins.consume.simulators.exceptions", + "execution_testing.cli.pytest_commands.plugins.consume.simulators.helpers.test_tracker", + "execution_testing.cli.pytest_commands.plugins.consume.simulators.engine_api", +) + + +def pytest_configure(config: pytest.Config) -> None: + """Set the supported fixture formats for the enginex simulator.""" + config.supported_fixture_formats = [BlockchainEngineXFixture] # type: ignore[attr-defined] + + +@pytest.hookimpl(trylast=True) +def pytest_collection_modifyitems( + session: pytest.Session, config: pytest.Config, items: list[pytest.Item] +) -> None: + """ + Count tests per pre-allocation group during collection phase. + + This hook analyzes all collected test items to determine how many tests + belong to each pre-alloc group, enabling automatic client cleanup when + all tests in a group complete. + + Use `trylast=True` to run after test deselection + (from `-k`, `-m` filters). + Reads group identifiers from `xdist_group` markers added in + `pytest_generate_tests`. + """ + supported_formats = getattr(config, "supported_fixture_formats", []) + if BlockchainEngineXFixture not in supported_formats: + return + + group_counts: dict[str, int] = {} + + for item in items: + # Extract group identifier from xdist_group marker + # (marker was added in pytest_generate_tests in consume.py) + group_identifier = None + for marker in item.iter_markers("xdist_group"): + if hasattr(marker, "kwargs") and "name" in marker.kwargs: + group_identifier = marker.kwargs["name"] + break + + if group_identifier: + group_counts[group_identifier] = ( + group_counts.get(group_identifier, 0) + 1 + ) + + if group_counts: + # Store counts in session stash for the test tracker fixture to use + session.stash[enginex_group_counts_key] = group_counts + logger.info( + f"Counted {len(group_counts)} pre-alloc groups with " + f"{sum(group_counts.values())} total tests" + ) + + # Sort tests by group_identifier to ensure consecutive execution + # This minimizes client thrashing and enables immediate client cleanup + def get_group_key(item: pytest.Item) -> str: + """Extract group identifier from item for sorting.""" + for marker in item.iter_markers("xdist_group"): + if hasattr(marker, "kwargs") and "name" in marker.kwargs: + return marker.kwargs["name"] + raise AssertionError( + f"EngineX test '{item.nodeid}' missing xdist_group marker" + ) + + items.sort(key=get_group_key) + logger.info( + "Sorted tests by pre-alloc group for consecutive execution" + ) + else: + logger.warning("No enginex test groups found during collection") + + +@pytest.fixture(scope="session", autouse=True) +def _configure_client_manager( + multi_test_client_manager: MultiTestClientManager, + pre_alloc_group_test_tracker: PreAllocGroupTestTracker, +) -> None: + """Wire the test tracker to the client manager at session start.""" + multi_test_client_manager.set_test_tracker(pre_alloc_group_test_tracker) + + +@pytest.fixture(scope="module") +def test_suite_name() -> str: + """The name of the hive test suite used in this simulator.""" + return "eels/consume-enginex" + + +@pytest.fixture(scope="module") +def test_suite_description() -> str: + """The description of the hive test suite used in this simulator.""" + return ( + "Execute blockchain tests against clients using the Engine API with " + "pre-allocation group optimization using Engine X fixtures." + ) + + +@pytest.fixture(scope="function") +def client( + multi_test_hive_test: HiveTest, + multi_test_client_manager: MultiTestClientManager, + fixture: BlockchainEngineXFixture, + client_type: ClientType, + environment: dict, + client_genesis: dict, + total_timing_data: TimingData, + request: pytest.FixtureRequest, +) -> Generator[Client, None, None]: + """ + Get or create a multi-test client for this pre-allocation group. + + Called for each test, but reuses clients across tests that + share the same pre-allocation group. + """ + group_identifier = fixture.pre_hash + test_id = request.node.nodeid + + # Check for existing client + existing_client = multi_test_client_manager.get_client(group_identifier) + if existing_client is not None: + logger.info( + f"♻️ Reusing client for group " + f"{format_group_identifier(group_identifier)}" + ) + try: + yield existing_client + finally: + multi_test_client_manager.mark_test_completed( + group_identifier, test_id + ) + return + + # Start new client; calculate genesis + genesis_bytes = json.dumps(client_genesis).encode("utf-8") + buffered_genesis = io.BufferedReader( + cast(io.RawIOBase, io.BytesIO(genesis_bytes)) + ) + + logger.info( + f"🚀 Starting client ({client_type.name}) for group " + f"{format_group_identifier(group_identifier)}" + ) + + with total_timing_data.time("Start client"): + client = multi_test_hive_test.start_client( + client_type=client_type, + environment=environment, + files={"/genesis.json": buffered_genesis}, + ) + + assert client is not None, ( + f"Unable to connect to client ({client_type.name}) via Hive. " + "Check the client or Hive server logs for more information." + ) + + logger.info( + f"Client ({client_type.name}) ready for group " + f"{format_group_identifier(group_identifier)}" + ) + + multi_test_client_manager.register_client(group_identifier, client) + + try: + yield client + finally: + multi_test_client_manager.mark_test_completed( + group_identifier, test_id + ) + + +@pytest.fixture(scope="function") +def genesis_header(pre_alloc_group: PreAllocGroup) -> FixtureHeader: + """Provide the genesis header from the pre-allocation group.""" + return pre_alloc_group.genesis diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/helpers/test_tracker.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/helpers/test_tracker.py new file mode 100644 index 00000000000..48f85680266 --- /dev/null +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/helpers/test_tracker.py @@ -0,0 +1,115 @@ +"""Test completion tracking for multi-test client architectures.""" + +import logging + +import pytest +from pytest import StashKey + +logger = logging.getLogger(__name__) + +# Typed stash keys for session-scoped data (replaces dynamic attributes) +enginex_group_counts_key: StashKey[dict[str, int]] = StashKey() + + +def format_group_identifier(group_identifier: str, max_len: int = 16) -> str: + """ + Safely format group identifier for logging. + + """ + if len(group_identifier) <= max_len: + return group_identifier + return group_identifier[:max_len] + + +class PreAllocGroupTestTracker: + """ + Track test completion per pre-allocation group. + + Enable automatic client cleanup. Maintain counts of expected + vs. completed tests for each group. When all tests in a group + complete, signal that the associated client can be stopped. + """ + + def __init__(self) -> None: + """Initialize the test tracker.""" + self.expected_counts: dict[ + str, int + ] = {} # group_identifier -> total expected tests + self.completed_tests: dict[ + str, set[str] + ] = {} # group_identifier -> set of completed test IDs + logger.debug("PreAllocGroupTestTracker initialized") + + def set_group_test_count(self, group_identifier: str, count: int) -> None: + """ + Set the expected number of tests for a group. + + This is typically called during pytest collection phase. + + """ + self.expected_counts[group_identifier] = count + self.completed_tests[group_identifier] = set() + logger.debug( + f"Set expected test count for group " + f"{format_group_identifier(group_identifier)}: " + f"{count}" + ) + + def mark_test_completed(self, group_identifier: str, test_id: str) -> bool: + """ + Mark a test as completed and check if the group is now complete. + + """ + if group_identifier not in self.completed_tests: + logger.warning( + "Marking test complete for unknown group " + f"{format_group_identifier(group_identifier)}" + ", initializing" + ) + self.completed_tests[group_identifier] = set() + + self.completed_tests[group_identifier].add(test_id) + completed = len(self.completed_tests[group_identifier]) + expected = self.expected_counts.get(group_identifier, 0) + + logger.debug( + f"Group " + f"{format_group_identifier(group_identifier)}: " + f"{completed}/{expected} tests completed" + ) + + # Check if group is complete + is_complete = completed >= expected and expected > 0 + if is_complete: + logger.info( + f"✓ Pre-alloc group " + f"{format_group_identifier(group_identifier)}" + f" complete ({completed}/{expected} tests)" + ) + + return is_complete + + +@pytest.fixture(scope="session") +def pre_alloc_group_test_tracker( + request: pytest.FixtureRequest, +) -> PreAllocGroupTestTracker: + """ + Provide session-scoped test tracker for automatic client cleanup. + + This fixture initializes the tracker and populates it with test counts + from the collection phase (if available via pytest stash). + """ + tracker = PreAllocGroupTestTracker() + + # Load test counts from session stash (set during collection) + session = request.session + group_counts = session.stash.get(enginex_group_counts_key, None) + if group_counts is not None: + for group_identifier, count in group_counts.items(): + tracker.set_group_test_count(group_identifier, count) + logger.info( + f"Loaded {len(group_counts)} group counts from session stash" + ) + + return tracker diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/multi_test_client.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/multi_test_client.py new file mode 100644 index 00000000000..dcd866b7ba5 --- /dev/null +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/multi_test_client.py @@ -0,0 +1,284 @@ +"""Pytest fixtures for multi-test client architecture.""" + +import logging +from typing import Generator + +import pytest +from hive.client import Client + +from execution_testing.base_types import to_json +from execution_testing.fixtures import BlockchainEngineXFixture +from execution_testing.fixtures.pre_alloc_groups import PreAllocGroup + +from ..consume import FixturesSource +from .helpers.ruleset import ruleset +from .helpers.test_tracker import ( + PreAllocGroupTestTracker, + format_group_identifier, +) + +logger = logging.getLogger(__name__) + + +class MultiTestClientManager: + """ + Session-scoped manager for client lifecycle across multiple tests. + + Coordinate client reuse across tests sharing the same + pre-allocation group, enabling efficient test execution + by avoiding redundant client restarts. + """ + + def __init__(self) -> None: + """Initialize the multi-test client manager.""" + self.clients: dict[str, Client] = {} # group_identifier -> Client + self.test_tracker: PreAllocGroupTestTracker | None = None + logger.debug("MultiTestClientManager initialized") + + def set_test_tracker(self, tracker: PreAllocGroupTestTracker) -> None: + """ + Set the test tracker for automatic client cleanup. + + """ + self.test_tracker = tracker + logger.debug("Test tracker registered with MultiTestClientManager") + + def get_client(self, group_identifier: str) -> Client | None: + """ + Get the client instance for a group. + + """ + if group_identifier in self.clients: + logger.debug( + f"Found existing client for group " + f"{format_group_identifier(group_identifier)}" + ) + return self.clients[group_identifier] + + logger.debug( + f"No existing client for group " + f"{format_group_identifier(group_identifier)}" + ) + return None + + def register_client(self, group_identifier: str, client: Client) -> None: + """ + Register a newly started client for a group. + + """ + if group_identifier in self.clients: + raise RuntimeError( + f"Client already exists for group " + f"{format_group_identifier(group_identifier)}" + ) + + self.clients[group_identifier] = client + logger.info( + f"Registered client for group " + f"{format_group_identifier(group_identifier)}" + ) + + def mark_test_completed(self, group_identifier: str, test_id: str) -> None: + """ + Mark a test as completed and trigger cleanup. + + """ + if self.test_tracker is None: + logger.warning( + "Test tracker not set, cannot perform automatic cleanup" + ) + return + + is_group_complete = self.test_tracker.mark_test_completed( + group_identifier, test_id + ) + + # Stop the client immediately when all tests in the group are complete + if is_group_complete: + logger.info( + f"✓ Group {format_group_identifier(group_identifier)} complete" + ) + if group_identifier in self.clients: + client = self.clients[group_identifier] + try: + logger.info( + f"🛑 Stopping client for group " + f"{format_group_identifier(group_identifier)}" + ) + client.stop() + except Exception as e: + logger.error( + f"Error stopping client for group " + f"{format_group_identifier(group_identifier)}: {e}" + ) + finally: + # Always remove from tracking, even if stop failed + del self.clients[group_identifier] + + def stop_all_clients(self) -> None: + """Stop all remaining clients (called at session end).""" + if not self.clients: + logger.info("No clients to clean up") + return + + logger.info(f"Stopping {len(self.clients)} remaining client(s)...") + for group_identifier, client in list(self.clients.items()): + try: + logger.info( + f"Stopping client for group " + f"{format_group_identifier(group_identifier)}" + ) + client.stop() + except Exception as e: + logger.error( + f"Error stopping client for group " + f"{format_group_identifier(group_identifier)}: {e}" + ) + + self.clients.clear() + logger.info("All clients stopped") + + +@pytest.fixture(scope="session") +def multi_test_client_manager() -> Generator[ + MultiTestClientManager, None, None +]: + """ + Provide session-scoped MultiTestClientManager with automatic cleanup. + + """ + manager = MultiTestClientManager() + try: + yield manager + finally: + logger.info("Session ending, cleaning up multi-test clients...") + manager.stop_all_clients() + + +@pytest.fixture(scope="session") +def pre_alloc_group_cache() -> dict[str, PreAllocGroup]: + """Cache for pre-allocation groups to avoid reloading from disk.""" + return {} + + +@pytest.fixture(scope="session") +def client_genesis_cache() -> dict[str, dict]: + """Cache for client genesis configs to avoid redundant to_json calls.""" + return {} + + +@pytest.fixture(scope="session") +def environment_cache() -> dict[str, dict]: + """Cache for environment configs to avoid redundant computation.""" + return {} + + +@pytest.fixture(scope="function") +def pre_alloc_group( + fixture: BlockchainEngineXFixture, + fixtures_source: FixturesSource, + pre_alloc_group_cache: dict[str, PreAllocGroup], +) -> PreAllocGroup: + """Load the pre-allocation group for the current test case.""" + pre_hash = fixture.pre_hash + + # Check cache first + if pre_hash in pre_alloc_group_cache: + logger.debug( + f"Using cached pre-alloc group for " + f"{format_group_identifier(pre_hash)}" + ) + return pre_alloc_group_cache[pre_hash] + + # Load from disk + if fixtures_source.is_stdin: + raise ValueError( + "Pre-allocation groups require file-based fixture input" + ) + + # Look for pre-allocation group file + pre_alloc_path = ( + fixtures_source.path + / "blockchain_tests_engine_x" + / "pre_alloc" + / f"{pre_hash}.json" + ) + + if not pre_alloc_path.exists(): + raise FileNotFoundError( + f"Pre-allocation group file not found: {pre_alloc_path}" + ) + + # Load and cache + logger.debug(f"Loading pre-alloc group from {pre_alloc_path}") + pre_alloc_group_obj = PreAllocGroup.from_file(pre_alloc_path) + + pre_alloc_group_cache[pre_hash] = pre_alloc_group_obj + logger.info( + f"Loaded pre-alloc group for {format_group_identifier(pre_hash)}" + ) + + return pre_alloc_group_obj + + +@pytest.fixture(scope="function") +def client_genesis( + pre_alloc_group: PreAllocGroup, + fixture: BlockchainEngineXFixture, + client_genesis_cache: dict[str, dict], +) -> dict: + """ + Convert pre-alloc group genesis header and pre-state to client genesis. + + Parallel to single_test_client.client_genesis but uses + PreAllocGroup. Use caching to avoid redundant to_json calls + for tests sharing the same pre_hash. + """ + pre_hash = fixture.pre_hash + + if pre_hash in client_genesis_cache: + return client_genesis_cache[pre_hash] + + genesis = to_json(pre_alloc_group.genesis) + alloc = to_json(pre_alloc_group.pre) + # NOTE: nethermind requires account keys without '0x' prefix + genesis["alloc"] = {k.replace("0x", ""): v for k, v in alloc.items()} + + client_genesis_cache[pre_hash] = genesis + return genesis + + +@pytest.fixture(scope="function") +def environment( + pre_alloc_group: PreAllocGroup, + fixture: BlockchainEngineXFixture, + check_live_port: int, + environment_cache: dict[str, dict], +) -> dict: + """ + Define environment variables for multi-test client startup. + + Parallel to single_test_client.environment but uses + PreAllocGroup. Use caching to avoid redundant computation + for tests sharing the same pre_hash. + """ + pre_hash = fixture.pre_hash + + if pre_hash in environment_cache: + return environment_cache[pre_hash] + + fork = pre_alloc_group.fork + assert fork in ruleset, f"fork '{fork}' missing in hive ruleset" + env = { + "HIVE_CHAIN_ID": "1", + "HIVE_NETWORK_ID": "1", + "HIVE_FORK_DAO_VOTE": "1", + "HIVE_NODETYPE": "full", + "HIVE_CHECK_LIVE_PORT": str(check_live_port), + **{k: f"{v:d}" for k, v in ruleset[fork].items()}, + "HIVE_FORK": pre_alloc_group.fork.name(), + "HIVE_GROUP_ID": format_group_identifier(fixture.pre_hash), + } + + environment_cache[pre_hash] = env + return env diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/simulator_logic/test_via_engine.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/simulator_logic/test_via_engine.py index 9abe0c74a0c..9370c77b0b6 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/simulator_logic/test_via_engine.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/simulator_logic/test_via_engine.py @@ -1,14 +1,26 @@ """ A hive based simulator that executes blocks against clients using the -`engine_newPayloadVX` method from the Engine API. The simulator uses the -`BlockchainEngineFixtures` to test against clients. +`engine_newPayloadVX` method from the Engine API. + +The unified test function in this module supports both: +- `BlockchainEngineFixtures`, the original engine mode with a + 1-to-1 relationship between client instance and test, i.e., + each test is executed against a fresh client instance. +- `BlockchainEngineXFixtures`, enginex mode with client reuse + across tests with shared pre-alloc groups. Each `engine_newPayloadVX` is verified against the appropriate VALID/INVALID responses. """ +from typing import Union + from execution_testing.exceptions import UndefinedException -from execution_testing.fixtures import BlockchainEngineFixture +from execution_testing.fixtures import ( + BlockchainEngineFixture, + BlockchainEngineXFixture, +) +from execution_testing.fixtures.blockchain import FixtureHeader from execution_testing.logging import get_logger from execution_testing.rpc import ( EngineRPC, @@ -34,54 +46,74 @@ def test_blockchain_via_engine( timing_data: TimingData, eth_rpc: EthRPC, engine_rpc: EngineRPC, - fixture: BlockchainEngineFixture, + fixture: Union[BlockchainEngineFixture, BlockchainEngineXFixture], strict_exception_matching: bool, + genesis_header: FixtureHeader, ) -> None: """ - 1. Check the client genesis block hash matches - `fixture.genesis.block_hash`. - 2. Execute the test case fixture blocks against the client under test using - the `engine_newPayloadVX` method from the Engine API. - 3. For valid payloads a forkchoice update is performed to finalize the - chain. + Execute blockchain test fixtures against a client using the Engine API. + + This function supports two modes: + + 1. **Engine Mode** (`BlockchainEngineFixture`): + - Uses per-test clients (started fresh for each test). + - Always performs initial FCU to genesis. + - Always performs FCU after valid payloads. + - genesis_header comes from fixture.genesis (via fixture). + - needs_genesis_init is always True (via fixture). + + 2. **EngineX Mode** (`BlockchainEngineXFixture`): + - Reuses clients across tests with same pre-alloc group. + - Skips initial FCU for reused clients. + - Skips FCU after valid payloads to keep client at genesis. + - genesis_header comes from separate pre_alloc_group fixture. + - needs_genesis_init is False for reused clients. + + Steps: + 1. Check the client genesis block hash matches genesis_header.block_hash + 2. Execute test fixture blocks using engine_newPayloadVX + 3. For valid payloads, perform forkchoice update to finalize chain + (unless client is being reused, in which case skip FCU) """ - # Send initial forkchoice update - with timing_data.time("Initial forkchoice update"): - logger.info("Sending initial forkchoice update to genesis block...") - try: - response = engine_rpc.forkchoice_updated_with_retry( - forkchoice_state=ForkchoiceState( - head_block_hash=fixture.genesis.block_hash, - ), - forkchoice_version=fixture.payloads[ - 0 - ].forkchoice_updated_version, - max_attempts=30, - wait_fixed=1.0, + if isinstance(fixture, BlockchainEngineFixture): + with timing_data.time("Initial forkchoice update"): + logger.info( + "Sending initial forkchoice update to genesis block..." ) - if response.payload_status.status != PayloadStatusEnum.VALID: - raise LoggedError( - f"Unexpected status on forkchoice updated to genesis: " - f"{response.payload_status.status}" + try: + response = engine_rpc.forkchoice_updated_with_retry( + forkchoice_state=ForkchoiceState( + head_block_hash=fixture.genesis.block_hash, + ), + forkchoice_version=fixture.payloads[ + 0 + ].forkchoice_updated_version, + max_attempts=30, + wait_fixed=1.0, ) - except ForkchoiceUpdateTimeoutError as e: - raise LoggedError( - f"Timed out waiting for forkchoice update to genesis: {e}" - ) from None + if response.payload_status.status != PayloadStatusEnum.VALID: + raise LoggedError( + f"Unexpected status on forkchoice updated to genesis: " + f"{response.payload_status.status}" + ) + except ForkchoiceUpdateTimeoutError as e: + raise LoggedError( + f"Timed out waiting for forkchoice update to genesis: {e}" + ) from None with timing_data.time("Get genesis block"): logger.info("Calling getBlockByNumber to get genesis block...") genesis_block = eth_rpc.get_block_by_number(0) assert genesis_block is not None, "genesis_block is None" - if genesis_block["hash"] != str(fixture.genesis.block_hash): - expected = fixture.genesis.block_hash + if genesis_block["hash"] != str(genesis_header.block_hash): + expected = genesis_header.block_hash got = genesis_block["hash"] logger.fail( f"Genesis block hash mismatch. " f"Expected: {expected}, Got: {got}" ) raise GenesisBlockMismatchExceptionError( - expected_header=fixture.genesis, + expected_header=genesis_header, got_genesis_block=genesis_block, ) @@ -182,7 +214,9 @@ def test_blockchain_via_engine( f"expected: {payload.error_code}" ) from e - if payload.valid(): + if payload.valid() and isinstance( + fixture, BlockchainEngineFixture + ): with payload_timing.time( f"engine_forkchoiceUpdatedV{payload.forkchoice_updated_version}" ): diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/sync/conftest.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/sync/conftest.py index ad01454d7c7..e8de135b155 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/sync/conftest.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/sync/conftest.py @@ -24,6 +24,7 @@ "execution_testing.cli.pytest_commands.plugins.consume.simulators.test_case_description", "execution_testing.cli.pytest_commands.plugins.consume.simulators.timing_data", "execution_testing.cli.pytest_commands.plugins.consume.simulators.exceptions", + "execution_testing.cli.pytest_commands.plugins.consume.simulators.engine_api", ) @@ -104,21 +105,6 @@ def pytest_collection_modifyitems( item._nodeid = base + new_suffix -@pytest.fixture(scope="function") -def engine_rpc( - client: Client, client_exception_mapper: ExceptionMapper | None -) -> EngineRPC: - """Initialize engine RPC client for the execution client under test.""" - if client_exception_mapper: - return EngineRPC( - f"http://{client.ip}:8551", - response_validation_context={ - "exception_mapper": client_exception_mapper, - }, - ) - return EngineRPC(f"http://{client.ip}:8551") - - @pytest.fixture(scope="function") def eth_rpc(client: Client) -> EthRPC: """Initialize eth RPC client for the execution client under test.""" diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/pytest_hive/pytest_hive.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/pytest_hive/pytest_hive.py index 62a6b113cd4..89e24f1f5e9 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/pytest_hive/pytest_hive.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/pytest_hive/pytest_hive.py @@ -247,6 +247,60 @@ def test_suite( users_file.unlink() +@pytest.fixture(scope="module") +def multi_test_hive_test( + test_suite: HiveTestSuite, + test_suite_name: str, +) -> Generator[HiveTest, None, None]: + """ + Create a module-scoped Hive test for multi-test client reuse. + + Provide a Hive test context that persists across multiple + pytest tests within a module. Multi-test clients are started + under this context and reused across tests, avoiding + redundant client restarts. + + This test lives for the entire module and prevents Hive from + terminating clients between individual pytest tests. This is + essential for simulators that batch multiple pytest tests + against the same client instance. + + Usage: + Simulators start clients using this context via: + ``multi_test_hive_test: HiveTest`` fixture dependency. + + Example: + ```python + @pytest.fixture(scope="function") + def client(multi_test_hive_test: HiveTest, ...): + client = multi_test_hive_test.start_client(...) + yield client + # Client lifecycle managed by simulator + ``` + + """ + logger.info( + f"Creating multi-test Hive test for '{test_suite_name}' (module scope)" + ) + test: HiveTest = test_suite.start_test( + name=f"{test_suite_name}-multi-test-clients", + description=(f"Multi-test client context for {test_suite_name}"), + ) + logger.info(f"Multi-test Hive test created: {test.id}") + yield test + + # End the multi-test context at module end + # Note: Simulators should manage client lifecycle themselves + # (e.g., stop clients when done, not rely on this teardown) + logger.info(f"Ending multi-test Hive test for '{test_suite_name}'") + test.end( + result=HiveTestResult( + test_pass=True, + details="Multi-test client context completed", + ) + ) + + @pytest.fixture(scope="function") def hive_test( request: pytest.FixtureRequest, test_suite: HiveTestSuite diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/processors.py b/packages/testing/src/execution_testing/cli/pytest_commands/processors.py index c47f8990942..1e705e38293 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/processors.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/processors.py @@ -121,6 +121,13 @@ def process_args(self, args: List[str]) -> List[str]: "execution_testing.cli.pytest_commands.plugins.consume.simulators.engine.conftest", ] ) + elif self.command_name == "enginex": + modified_args.extend( + [ + "-p", + "execution_testing.cli.pytest_commands.plugins.consume.simulators.enginex.conftest", + ] + ) elif self.command_name == "sync": modified_args.extend( [ diff --git a/whitelist.txt b/whitelist.txt index 955a57adffe..17bc1be3728 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -477,6 +477,7 @@ encodings endian endianness EngineAPI +enginex enum env envvar From 87346c16b9079027308977335586b64fb51c9d1f Mon Sep 17 00:00:00 2001 From: danceratopz Date: Thu, 27 Nov 2025 09:52:28 +0100 Subject: [PATCH 02/10] feat(consume): enable dist=loadgroup by default for enginex --- .../execution_testing/cli/pytest_commands/processors.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/processors.py b/packages/testing/src/execution_testing/cli/pytest_commands/processors.py index 1e705e38293..7862140380e 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/processors.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/processors.py @@ -106,6 +106,14 @@ def process_args(self, args: List[str]) -> List[str]: ] and not self._has_parallelism_flag(args): modified_args.extend(["-n", str(hive_parallelism)]) + # For enginex: ensure xdist uses loadgroup distribution so tests with + # the same xdist_group marker (pre-alloc group) run on the same worker + if self.command_name == "enginex" and self._has_parallelism_flag( + modified_args + ): + if "--dist" not in modified_args: + modified_args.extend(["--dist", "loadgroup"]) + if os.getenv("HIVE_RANDOM_SEED") is not None: warnings.warn( "HIVE_RANDOM_SEED is not yet supported.", stacklevel=2 From 2c6dd1ca3d2d602c9739b33976a6977e0c21218e Mon Sep 17 00:00:00 2001 From: danceratopz Date: Thu, 27 Nov 2025 11:05:42 +0100 Subject: [PATCH 03/10] feat(docs): add basic enginex doc overview to running tests --- docs/running_tests/running.md | 43 +++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/docs/running_tests/running.md b/docs/running_tests/running.md index efd98c340d3..0fbe5bf87fe 100644 --- a/docs/running_tests/running.md +++ b/docs/running_tests/running.md @@ -14,6 +14,7 @@ Both `consume` and `execute` provide sub-commands which correspond to different | [`consume direct`](#direct) | Client consume tests via a `statetest` interface | EVM | None | Module test | | [`consume direct`](#direct) | Client consume tests via a `blocktest` interface | EVM, block processing | None | Module test,
Integration test | | [`consume engine`](#engine) | Client imports blocks via Engine API `EngineNewPayload` in Hive | EVM, block processing, Engine API | Staging, Hive | System test | +| [`consume enginex`](#enginex) | Client imports blocks via Engine API in Hive with, optimized by client reuse | EVM, block processing, Engine API | Staging, Hive | System test | | [`consume sync`](#sync) | Client syncs from another client using Engine API in Hive | EVM, block processing, Engine API, P2P sync | Staging, Hive | System test | | [`consume rlp`](#rlp) | Client imports RLP-encoded blocks upon start-up in Hive | EVM, block processing, RLP import (sync\*) | Staging, Hive | System test | | [`execute hive`](./execute/hive.md) | Tests executed against a client via JSON RPC `eth_sendRawTransaction` in Hive | EVM, JSON RPC, mempool | Staging, Hive | System test | @@ -62,6 +63,48 @@ The `consume engine` command: 5. **Validates responses** against expected results. 6. **Tests error conditions** and exception handling. +## EngineX + +| Nomenclature | | +| -------------- | -------------------------- | +| Command | `consume enginex` | +| Simulator | `eels/consume-enginex` | +| Fixture format | `blockchain_test_engine_x` | + +The EngineX method is a faster alternative to `consume engine` that executes multiple tests against a single client instance. This is achieved via the [Blockchain Engine X Test fixture format](./test_formats/blockchain_test_engine_x.md) which groups tests that share the same fork and EVM [Environment](./test_formats/state_test.md#fixtureenvironment) together and contains a larger, shared pre-allocation state that all tests in the group use. This allows the EngineX simulator to execute multiple tests against the same client instance, whereas the Engine Simulator starts a fresh client for each test. + +The `consume enginex` command, for each pre-allocation group: + +1. **Initializes the execution client** with the group's shared genesis state. +2. **Connects via Engine API** (port 8551). +3. **Executes all tests in the group** against the same client: + + - Submits payloads from each test using `engine_newPayload` calls. + - Validates responses against expected results. + - Tests error conditions and exception handling. + +4. **Stops the client** when all tests in the group complete. + +### Engine vs EngineX + +| | `consume engine` | `consume enginex` | +| -------------------- | ---------------------------------------------------------------------- | ------------------------------------------------------------------------ | +| **Fixture format** | [`blockchain_test_engine`](./test_formats/blockchain_test_engine.md) | [`blockchain_test_engine_x`](./test_formats/blockchain_test_engine_x.md) | +| **Client lifecycle** | New client per test | Client reused across tests with same pre-alloc | +| **Fork choice update** | FCU called for genesis and final payload | FCU for genesis and final payload skipped | +| **Execution speed** | Slower (client startup overhead) | Faster (amortized startup cost) | +| **Test isolation** | Full isolation | Shared genesis state within group | + +EngineX achieves faster execution by: + +1. **Grouping tests** by their pre-allocation state (genesis configuration). +2. **Reusing clients** across all tests in a group, avoiding repeated client startup. +3. **Skipping redundant initialization** since the client is already at the expected genesis state. + +!!! note "When to use EngineX vs Engine" + + Use `consume enginex` for faster test runs when full per-test isolation is not required. Use `consume engine` when you need complete isolation between tests or when debugging issues triggered by a single test case. + ## RLP | Nomenclature | | From 4f294f70bca9e869c0d565b7664efa26f853a0da Mon Sep 17 00:00:00 2001 From: danceratopz Date: Mon, 5 Jan 2026 09:05:33 +0100 Subject: [PATCH 04/10] chore: update vulture whitelist with enginex false positives --- vulture_whitelist.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/vulture_whitelist.py b/vulture_whitelist.py index fe0c9d48def..c69736d05e7 100644 --- a/vulture_whitelist.py +++ b/vulture_whitelist.py @@ -137,3 +137,8 @@ CommentReplaceCommand.transform_module_impl _children # unused attribute (src/ethereum_spec_tools/docc.py:751) + +# enginex/conftest.py - pytest fixtures (not direct calls) +_configure_client_manager # autouse fixture +test_suite_name # hive test suite name fixture +genesis_header # genesis header fixture From 0e877518a5711b369419a2316ad2abaac4e34bf5 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Mon, 5 Jan 2026 09:28:28 +0100 Subject: [PATCH 05/10] chore(ci): add enginex dev-mode to hive-consume workflow - Add commented-out enginex dev-mode matrix entry. - No release with enginex containing the new pre-alloc builder format (cf #2140). --- .github/workflows/hive-consume.yaml | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/hive-consume.yaml b/.github/workflows/hive-consume.yaml index 13ec8a208bd..b364d282588 100644 --- a/.github/workflows/hive-consume.yaml +++ b/.github/workflows/hive-consume.yaml @@ -10,7 +10,7 @@ on: - ".gitignore" - ".vscode/**" - "whitelist.txt" - - "docs/**" + - "docs/**" - "mkdocs.yml" pull_request: paths: @@ -71,15 +71,29 @@ jobs: - name: Engine mode: simulator simulator: ethereum/eels/consume-engine + sim_limit: ".*test_block_at_rlp_limit_with_logs.*Osaka.*" + # TODO: Enable once eels/consume-enginex simulator is added to Hive + # - name: consume-enginex + # mode: simulator + # simulator: ethereum/eels/consume-enginex + # sim_limit: ".*push0.*(Shanghai|Cancun).*" - name: RLP mode: simulator simulator: ethereum/eels/consume-rlp + sim_limit: ".*test_block_at_rlp_limit_with_logs.*Osaka.*" - name: Sync mode: simulator simulator: ethereum/eels/consume-sync + sim_limit: ".*test_block_at_rlp_limit_with_logs.*Osaka.*" - name: Dev Mode mode: dev consume_command: engine + test_filter: "Osaka and test_block_at_rlp_limit_with_logs" + # TODO: Enable once we have a release with the pre-alloc builder format cf #2140 + # - name: dev-mode-enginex + # mode: dev + # consume_command: enginex + # test_filter: "(fork_shanghai or fork_cancun) and push0" steps: - name: Checkout execution-specs uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 @@ -122,7 +136,7 @@ jobs: --client go-ethereum \ --client-file ../execution-specs/.github/configs/hive/latest.yaml \ --sim.buildarg fixtures=${{ env.FIXTURES_URL }} \ - --sim.limit=".*test_block_at_rlp_limit_with_logs.*Osaka.*" \ + --sim.limit="${{ matrix.sim_limit }}" \ --docker.output - name: Start Hive in dev mode @@ -142,5 +156,4 @@ jobs: HIVE_SIMULATOR: ${{ steps.start-hive.outputs.hive-url }} run: | uv sync --all-extras - uv run consume ${{ matrix.consume_command }} --input ${{ env.FIXTURES_URL }} -k "Osaka and test_block_at_rlp_limit_with_logs" - + uv run consume ${{ matrix.consume_command }} --input ${{ env.FIXTURES_URL }} -k "${{ matrix.test_filter }}" From d1e1b5831d337f38326903d11b02cdf3dd490886 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Tue, 10 Mar 2026 06:51:53 +0100 Subject: [PATCH 06/10] fix(consume): assign `xdist_group` during parametrization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Build combined `test_case`×`client_type` parametrization in `pytest_generate_tests`, attaching `xdist_group` marks inline so xdist's `loadgroup` scheduler reads them natively. - Simplify enginex `pytest_collection_modifyitems` to only read existing markers for counting and sorting (largest groups first). - Add `make_group_identifier()` helper for composite key construction (`pre_hash` + `client_name`). --- .../plugins/consume/consume.py | 57 ++++++++++------ .../consume/simulators/enginex/conftest.py | 67 ++++++++----------- .../simulators/helpers/test_tracker.py | 5 ++ 3 files changed, 68 insertions(+), 61 deletions(-) diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/consume.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/consume.py index 68e8fdfcecf..21d8e99aa0d 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/consume.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/consume.py @@ -8,7 +8,7 @@ from dataclasses import dataclass from io import BytesIO from pathlib import Path -from typing import Any, List, Optional, Tuple +from typing import Any, List, Optional, Tuple, cast from urllib.parse import urlparse import platformdirs @@ -19,12 +19,19 @@ from execution_testing.cli.gen_index import ( generate_fixtures_index, ) +from execution_testing.cli.pytest_commands.plugins.consume.simulators.helpers.test_tracker import ( # noqa: E501 + make_group_identifier, +) from execution_testing.fixtures import ( BaseFixture, BlockchainEngineXFixture, FixtureFormat, ) -from execution_testing.fixtures.consume import IndexFile, TestCases +from execution_testing.fixtures.consume import ( + IndexFile, + TestCaseBase, + TestCases, +) from execution_testing.forks import ( get_forks, get_relative_fork_markers, @@ -606,16 +613,6 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: getattr(pytest.mark, test_case.format.format_name) ] - # Add xdist_group marker for engine x tests to enable - # client reuse tracking - if test_case.format is BlockchainEngineXFixture: - assert hasattr(test_case, "pre_hash") and test_case.pre_hash, ( - f"BlockchainEngineXFixture test case " - f"'{test_case.id}' missing pre_hash" - ) - group_identifier = test_case.pre_hash - marks.append(pytest.mark.xdist_group(name=group_identifier)) - param = pytest.param( test_case, id=test_case.id, @@ -623,14 +620,30 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: ) param_list.append(param) - metafunc.parametrize("test_case", param_list) + if "client_type" not in metafunc.fixturenames: + metafunc.parametrize("test_case", param_list) + return - if "client_type" in metafunc.fixturenames: - metafunc.parametrize( - "client_type", - metafunc.config.hive_execution_clients, # type: ignore[attr-defined] - ids=[ - client.name - for client in metafunc.config.hive_execution_clients # type: ignore[attr-defined] - ], - ) + clients = metafunc.config.hive_execution_clients # type: ignore[attr-defined] + is_enginex = BlockchainEngineXFixture in supported_fixture_formats + combined = [] + for param in param_list: + tc = cast(TestCaseBase, param.values[0]) + for ct in clients: + marks = list(param.marks) + if is_enginex: + assert tc.pre_hash is not None + marks.append( + pytest.mark.xdist_group( + name=make_group_identifier(tc.pre_hash, ct.name) + ) + ) + combined.append( + pytest.param( + tc, + ct, + id=f"{tc.id}-{ct.name}", + marks=marks, + ) + ) + metafunc.parametrize("test_case,client_type", combined) diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/enginex/conftest.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/enginex/conftest.py index f5384389031..293514711d6 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/enginex/conftest.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/enginex/conftest.py @@ -24,6 +24,7 @@ PreAllocGroupTestTracker, enginex_group_counts_key, format_group_identifier, + make_group_identifier, ) from ..multi_test_client import MultiTestClientManager from ..timing_data import TimingData @@ -52,16 +53,14 @@ def pytest_collection_modifyitems( session: pytest.Session, config: pytest.Config, items: list[pytest.Item] ) -> None: """ - Count tests per pre-allocation group during collection phase. + Count tests per xdist_group and sort largest groups first. - This hook analyzes all collected test items to determine how many tests - belong to each pre-alloc group, enabling automatic client cleanup when - all tests in a group complete. + The xdist_group markers are set during parametrization in + `pytest_generate_tests`. This hook reads them to count tests + per group and sort for optimal xdist scheduling. Use `trylast=True` to run after test deselection (from `-k`, `-m` filters). - Reads group identifiers from `xdist_group` markers added in - `pytest_generate_tests`. """ supported_formats = getattr(config, "supported_fixture_formats", []) if BlockchainEngineXFixture not in supported_formats: @@ -70,44 +69,32 @@ def pytest_collection_modifyitems( group_counts: dict[str, int] = {} for item in items: - # Extract group identifier from xdist_group marker - # (marker was added in pytest_generate_tests in consume.py) - group_identifier = None for marker in item.iter_markers("xdist_group"): - if hasattr(marker, "kwargs") and "name" in marker.kwargs: + if "name" in marker.kwargs: group_identifier = marker.kwargs["name"] break - - if group_identifier: - group_counts[group_identifier] = ( - group_counts.get(group_identifier, 0) + 1 - ) - - if group_counts: - # Store counts in session stash for the test tracker fixture to use - session.stash[enginex_group_counts_key] = group_counts - logger.info( - f"Counted {len(group_counts)} pre-alloc groups with " - f"{sum(group_counts.values())} total tests" + else: + continue + group_counts[group_identifier] = ( + group_counts.get(group_identifier, 0) + 1 ) - # Sort tests by group_identifier to ensure consecutive execution - # This minimizes client thrashing and enables immediate client cleanup - def get_group_key(item: pytest.Item) -> str: - """Extract group identifier from item for sorting.""" - for marker in item.iter_markers("xdist_group"): - if hasattr(marker, "kwargs") and "name" in marker.kwargs: - return marker.kwargs["name"] - raise AssertionError( - f"EngineX test '{item.nodeid}' missing xdist_group marker" - ) + session.stash[enginex_group_counts_key] = group_counts + logger.info( + f"Counted {len(group_counts)} pre-alloc groups with " + f"{sum(group_counts.values())} total tests" + ) - items.sort(key=get_group_key) - logger.info( - "Sorted tests by pre-alloc group for consecutive execution" - ) - else: - logger.warning("No enginex test groups found during collection") + def sort_key(item: pytest.Item) -> tuple[int, str]: + """Return sort key: largest group first, then by group id.""" + for marker in item.iter_markers("xdist_group"): + if "name" in marker.kwargs: + gid = marker.kwargs["name"] + return (-group_counts[gid], gid) + return (0, "") + + items.sort(key=sort_key) + logger.info("Sorted tests by pre-alloc group (largest first)") @pytest.fixture(scope="session", autouse=True) @@ -151,7 +138,9 @@ def client( Called for each test, but reuses clients across tests that share the same pre-allocation group. """ - group_identifier = fixture.pre_hash + group_identifier = make_group_identifier( + fixture.pre_hash, client_type.name + ) test_id = request.node.nodeid # Check for existing client diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/helpers/test_tracker.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/helpers/test_tracker.py index 48f85680266..61fc3c43db8 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/helpers/test_tracker.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/helpers/test_tracker.py @@ -11,6 +11,11 @@ enginex_group_counts_key: StashKey[dict[str, int]] = StashKey() +def make_group_identifier(pre_hash: str, client_name: str) -> str: + """Build xdist group key from pre-alloc hash and client name.""" + return f"{pre_hash}-{client_name}" + + def format_group_identifier(group_identifier: str, max_len: int = 16) -> str: """ Safely format group identifier for logging. From 7174921f13b960147a4c89b3ba461d3886dcb9ab Mon Sep 17 00:00:00 2001 From: danceratopz Date: Tue, 10 Mar 2026 07:24:03 +0100 Subject: [PATCH 07/10] refactor(consume): remove `format_group_identifier` & dead `HIVE_GROUP_ID` - Remove `format_group_identifier()` which silently truncated the 18-char `pre_hash` (and lost the client name entirely from composite keys). - Use full group identifiers in all log messages. - Remove unused `HIVE_GROUP_ID` env var (nothing in hive reads it). - Collapse single-value f-strings where possible. --- .../consume/simulators/enginex/conftest.py | 12 ++--- .../simulators/helpers/test_tracker.py | 26 ++-------- .../consume/simulators/multi_test_client.py | 52 +++++-------------- 3 files changed, 21 insertions(+), 69 deletions(-) diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/enginex/conftest.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/enginex/conftest.py index 293514711d6..ad6108cb7f0 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/enginex/conftest.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/enginex/conftest.py @@ -23,7 +23,6 @@ from ..helpers.test_tracker import ( PreAllocGroupTestTracker, enginex_group_counts_key, - format_group_identifier, make_group_identifier, ) from ..multi_test_client import MultiTestClientManager @@ -146,10 +145,7 @@ def client( # Check for existing client existing_client = multi_test_client_manager.get_client(group_identifier) if existing_client is not None: - logger.info( - f"♻️ Reusing client for group " - f"{format_group_identifier(group_identifier)}" - ) + logger.info(f"♻️ Reusing client for group {group_identifier}") try: yield existing_client finally: @@ -165,8 +161,7 @@ def client( ) logger.info( - f"🚀 Starting client ({client_type.name}) for group " - f"{format_group_identifier(group_identifier)}" + f"🚀 Starting client ({client_type.name}) for group {group_identifier}" ) with total_timing_data.time("Start client"): @@ -182,8 +177,7 @@ def client( ) logger.info( - f"Client ({client_type.name}) ready for group " - f"{format_group_identifier(group_identifier)}" + f"Client ({client_type.name}) ready for group {group_identifier}" ) multi_test_client_manager.register_client(group_identifier, client) diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/helpers/test_tracker.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/helpers/test_tracker.py index 61fc3c43db8..21c4a35d22e 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/helpers/test_tracker.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/helpers/test_tracker.py @@ -16,16 +16,6 @@ def make_group_identifier(pre_hash: str, client_name: str) -> str: return f"{pre_hash}-{client_name}" -def format_group_identifier(group_identifier: str, max_len: int = 16) -> str: - """ - Safely format group identifier for logging. - - """ - if len(group_identifier) <= max_len: - return group_identifier - return group_identifier[:max_len] - - class PreAllocGroupTestTracker: """ Track test completion per pre-allocation group. @@ -55,9 +45,7 @@ def set_group_test_count(self, group_identifier: str, count: int) -> None: self.expected_counts[group_identifier] = count self.completed_tests[group_identifier] = set() logger.debug( - f"Set expected test count for group " - f"{format_group_identifier(group_identifier)}: " - f"{count}" + f"Set expected test count for group {group_identifier}: {count}" ) def mark_test_completed(self, group_identifier: str, test_id: str) -> bool: @@ -67,9 +55,8 @@ def mark_test_completed(self, group_identifier: str, test_id: str) -> bool: """ if group_identifier not in self.completed_tests: logger.warning( - "Marking test complete for unknown group " - f"{format_group_identifier(group_identifier)}" - ", initializing" + f"Marking test complete for unknown group " + f"{group_identifier}, initializing" ) self.completed_tests[group_identifier] = set() @@ -78,17 +65,14 @@ def mark_test_completed(self, group_identifier: str, test_id: str) -> bool: expected = self.expected_counts.get(group_identifier, 0) logger.debug( - f"Group " - f"{format_group_identifier(group_identifier)}: " - f"{completed}/{expected} tests completed" + f"Group {group_identifier}: {completed}/{expected} tests completed" ) # Check if group is complete is_complete = completed >= expected and expected > 0 if is_complete: logger.info( - f"✓ Pre-alloc group " - f"{format_group_identifier(group_identifier)}" + f"✓ Pre-alloc group {group_identifier}" f" complete ({completed}/{expected} tests)" ) diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/multi_test_client.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/multi_test_client.py index dcd866b7ba5..003a685536a 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/multi_test_client.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/multi_test_client.py @@ -12,10 +12,7 @@ from ..consume import FixturesSource from .helpers.ruleset import ruleset -from .helpers.test_tracker import ( - PreAllocGroupTestTracker, - format_group_identifier, -) +from .helpers.test_tracker import PreAllocGroupTestTracker logger = logging.getLogger(__name__) @@ -49,16 +46,10 @@ def get_client(self, group_identifier: str) -> Client | None: """ if group_identifier in self.clients: - logger.debug( - f"Found existing client for group " - f"{format_group_identifier(group_identifier)}" - ) + logger.debug(f"Found existing client for group {group_identifier}") return self.clients[group_identifier] - logger.debug( - f"No existing client for group " - f"{format_group_identifier(group_identifier)}" - ) + logger.debug(f"No existing client for group {group_identifier}") return None def register_client(self, group_identifier: str, client: Client) -> None: @@ -68,15 +59,11 @@ def register_client(self, group_identifier: str, client: Client) -> None: """ if group_identifier in self.clients: raise RuntimeError( - f"Client already exists for group " - f"{format_group_identifier(group_identifier)}" + f"Client already exists for group {group_identifier}" ) self.clients[group_identifier] = client - logger.info( - f"Registered client for group " - f"{format_group_identifier(group_identifier)}" - ) + logger.info(f"Registered client for group {group_identifier}") def mark_test_completed(self, group_identifier: str, test_id: str) -> None: """ @@ -95,21 +82,18 @@ def mark_test_completed(self, group_identifier: str, test_id: str) -> None: # Stop the client immediately when all tests in the group are complete if is_group_complete: - logger.info( - f"✓ Group {format_group_identifier(group_identifier)} complete" - ) + logger.info(f"✓ Group {group_identifier} complete") if group_identifier in self.clients: client = self.clients[group_identifier] try: logger.info( - f"🛑 Stopping client for group " - f"{format_group_identifier(group_identifier)}" + f"🛑 Stopping client for group {group_identifier}" ) client.stop() except Exception as e: logger.error( - f"Error stopping client for group " - f"{format_group_identifier(group_identifier)}: {e}" + "Error stopping client for group " + f"{group_identifier}: {e}" ) finally: # Always remove from tracking, even if stop failed @@ -124,15 +108,11 @@ def stop_all_clients(self) -> None: logger.info(f"Stopping {len(self.clients)} remaining client(s)...") for group_identifier, client in list(self.clients.items()): try: - logger.info( - f"Stopping client for group " - f"{format_group_identifier(group_identifier)}" - ) + logger.info(f"Stopping client for group {group_identifier}") client.stop() except Exception as e: logger.error( - f"Error stopping client for group " - f"{format_group_identifier(group_identifier)}: {e}" + f"Error stopping client for group {group_identifier}: {e}" ) self.clients.clear() @@ -184,10 +164,7 @@ def pre_alloc_group( # Check cache first if pre_hash in pre_alloc_group_cache: - logger.debug( - f"Using cached pre-alloc group for " - f"{format_group_identifier(pre_hash)}" - ) + logger.debug(f"Using cached pre-alloc group for {pre_hash}") return pre_alloc_group_cache[pre_hash] # Load from disk @@ -214,9 +191,7 @@ def pre_alloc_group( pre_alloc_group_obj = PreAllocGroup.from_file(pre_alloc_path) pre_alloc_group_cache[pre_hash] = pre_alloc_group_obj - logger.info( - f"Loaded pre-alloc group for {format_group_identifier(pre_hash)}" - ) + logger.info(f"Loaded pre-alloc group for {pre_hash}") return pre_alloc_group_obj @@ -277,7 +252,6 @@ def environment( "HIVE_CHECK_LIVE_PORT": str(check_live_port), **{k: f"{v:d}" for k, v in ruleset[fork].items()}, "HIVE_FORK": pre_alloc_group.fork.name(), - "HIVE_GROUP_ID": format_group_identifier(fixture.pre_hash), } environment_cache[pre_hash] = env From fc84fda7b2ca71719d268fcfcc1f512a970cd9fd Mon Sep 17 00:00:00 2001 From: danceratopz Date: Tue, 10 Mar 2026 10:48:55 +0100 Subject: [PATCH 08/10] refactor(consume): single-yield `client` fixture in enginex - Consolidate dual-yield paths (reuse vs new) into a single resolve-or-create branch followed by one `try`/`yield`/`finally`. --- .../consume/simulators/enginex/conftest.py | 62 +++++++++---------- 1 file changed, 29 insertions(+), 33 deletions(-) diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/enginex/conftest.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/enginex/conftest.py index ad6108cb7f0..dae7af3fa50 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/enginex/conftest.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/enginex/conftest.py @@ -142,48 +142,44 @@ def client( ) test_id = request.node.nodeid - # Check for existing client - existing_client = multi_test_client_manager.get_client(group_identifier) - if existing_client is not None: + resolved_client = multi_test_client_manager.get_client(group_identifier) + if resolved_client is not None: logger.info(f"♻️ Reusing client for group {group_identifier}") - try: - yield existing_client - finally: - multi_test_client_manager.mark_test_completed( - group_identifier, test_id - ) - return + else: + # Start new client; calculate genesis + genesis_bytes = json.dumps(client_genesis).encode("utf-8") + buffered_genesis = io.BufferedReader( + cast(io.RawIOBase, io.BytesIO(genesis_bytes)) + ) - # Start new client; calculate genesis - genesis_bytes = json.dumps(client_genesis).encode("utf-8") - buffered_genesis = io.BufferedReader( - cast(io.RawIOBase, io.BytesIO(genesis_bytes)) - ) + logger.info( + f"🚀 Starting client ({client_type.name}) " + f"for group {group_identifier}" + ) - logger.info( - f"🚀 Starting client ({client_type.name}) for group {group_identifier}" - ) + with total_timing_data.time("Start client"): + resolved_client = multi_test_hive_test.start_client( + client_type=client_type, + environment=environment, + files={"/genesis.json": buffered_genesis}, + ) - with total_timing_data.time("Start client"): - client = multi_test_hive_test.start_client( - client_type=client_type, - environment=environment, - files={"/genesis.json": buffered_genesis}, + assert resolved_client is not None, ( + f"Unable to connect to client ({client_type.name}) via " + "Hive. Check the client or Hive server logs for more " + "information." ) - assert client is not None, ( - f"Unable to connect to client ({client_type.name}) via Hive. " - "Check the client or Hive server logs for more information." - ) - - logger.info( - f"Client ({client_type.name}) ready for group {group_identifier}" - ) + logger.info( + f"Client ({client_type.name}) ready for group {group_identifier}" + ) - multi_test_client_manager.register_client(group_identifier, client) + multi_test_client_manager.register_client( + group_identifier, resolved_client + ) try: - yield client + yield resolved_client finally: multi_test_client_manager.mark_test_completed( group_identifier, test_id From 0aae0925a8b9267ceecb17a8aceb971369d495b9 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Tue, 10 Mar 2026 11:27:48 +0100 Subject: [PATCH 09/10] chore(consume): fix `PytestAssertRewriteWarning` warnings The enginex conftest both imported `MultiTestClientManager` and `TimingData` at module level and listed their modules in `pytest_plugins`. Python's import ran before pytest could register the modules for assert rewriting, producing: PytestAssertRewriteWarning: Module already imported so cannot be rewritten Fix by moving the imports behind `TYPE_CHECKING` and using string annotations in fixture signatures. Pytest resolves fixtures by parameter name, not type annotation, so the string literals work at runtime. Mypy/ruff still see the types via the `TYPE_CHECKING` block. --- .../plugins/consume/simulators/enginex/conftest.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/enginex/conftest.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/enginex/conftest.py index dae7af3fa50..80ad3405f2b 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/enginex/conftest.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/simulators/enginex/conftest.py @@ -10,7 +10,7 @@ import io import json import logging -from typing import Generator, cast +from typing import TYPE_CHECKING, Generator, cast import pytest from hive.client import Client, ClientType @@ -25,8 +25,10 @@ enginex_group_counts_key, make_group_identifier, ) -from ..multi_test_client import MultiTestClientManager -from ..timing_data import TimingData + +if TYPE_CHECKING: + from ..multi_test_client import MultiTestClientManager + from ..timing_data import TimingData logger = logging.getLogger(__name__) @@ -98,7 +100,7 @@ def sort_key(item: pytest.Item) -> tuple[int, str]: @pytest.fixture(scope="session", autouse=True) def _configure_client_manager( - multi_test_client_manager: MultiTestClientManager, + multi_test_client_manager: "MultiTestClientManager", pre_alloc_group_test_tracker: PreAllocGroupTestTracker, ) -> None: """Wire the test tracker to the client manager at session start.""" @@ -123,12 +125,12 @@ def test_suite_description() -> str: @pytest.fixture(scope="function") def client( multi_test_hive_test: HiveTest, - multi_test_client_manager: MultiTestClientManager, + multi_test_client_manager: "MultiTestClientManager", fixture: BlockchainEngineXFixture, client_type: ClientType, environment: dict, client_genesis: dict, - total_timing_data: TimingData, + total_timing_data: "TimingData", request: pytest.FixtureRequest, ) -> Generator[Client, None, None]: """ From 82a06a33e8a52059064241a14a589bcb7a0a25e8 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Tue, 10 Mar 2026 17:12:37 +0100 Subject: [PATCH 10/10] docs: fix typo Co-authored-by: spencer --- docs/running_tests/running.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/running_tests/running.md b/docs/running_tests/running.md index 0fbe5bf87fe..d09a1b3003c 100644 --- a/docs/running_tests/running.md +++ b/docs/running_tests/running.md @@ -14,7 +14,7 @@ Both `consume` and `execute` provide sub-commands which correspond to different | [`consume direct`](#direct) | Client consume tests via a `statetest` interface | EVM | None | Module test | | [`consume direct`](#direct) | Client consume tests via a `blocktest` interface | EVM, block processing | None | Module test,
Integration test | | [`consume engine`](#engine) | Client imports blocks via Engine API `EngineNewPayload` in Hive | EVM, block processing, Engine API | Staging, Hive | System test | -| [`consume enginex`](#enginex) | Client imports blocks via Engine API in Hive with, optimized by client reuse | EVM, block processing, Engine API | Staging, Hive | System test | +| [`consume enginex`](#enginex) | Client imports blocks via Engine API in Hive, optimized by client reuse | EVM, block processing, Engine API | Staging, Hive | System test | | [`consume sync`](#sync) | Client syncs from another client using Engine API in Hive | EVM, block processing, Engine API, P2P sync | Staging, Hive | System test | | [`consume rlp`](#rlp) | Client imports RLP-encoded blocks upon start-up in Hive | EVM, block processing, RLP import (sync\*) | Staging, Hive | System test | | [`execute hive`](./execute/hive.md) | Tests executed against a client via JSON RPC `eth_sendRawTransaction` in Hive | EVM, JSON RPC, mempool | Staging, Hive | System test |