From aa28d01577c94c1802d2ff7c8a03070485da6e42 Mon Sep 17 00:00:00 2001 From: spencer-tb Date: Mon, 9 Mar 2026 10:24:06 +0000 Subject: [PATCH] feat(execute): report individual test results in hive UI Add per-test Hive test records so each execute test appears individually in hiveview with its own pass/fail status, instead of aggregating all results into a single entry. Closes #1550 --- .../plugins/execute/rpc/hive.py | 137 ++++++++++++++++-- 1 file changed, 125 insertions(+), 12 deletions(-) diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/execute/rpc/hive.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/execute/rpc/hive.py index 4b91dd80040..76f9b055976 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/execute/rpc/hive.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/execute/rpc/hive.py @@ -22,6 +22,7 @@ ) from execution_testing.fixtures.blockchain import FixtureHeader from execution_testing.forks import Fork +from execution_testing.logging import get_logger from execution_testing.rpc import EngineRPC, EthRPC from execution_testing.test_types import ( DETERMINISTIC_FACTORY_ADDRESS, @@ -37,6 +38,8 @@ from ...consume.simulators.helpers.ruleset import ruleset from .chain_builder_eth_rpc import ChainBuilderEthRPC, TestingRPC +logger = get_logger(__name__) + def pytest_addoption(parser: pytest.Parser) -> None: """Add command-line options to pytest.""" @@ -253,12 +256,13 @@ def test_suite_description() -> str: @pytest.fixture(autouse=True, scope="session") def base_hive_test( - request: pytest.FixtureRequest, test_suite: HiveTestSuite, session_temp_folder: Path, ) -> Generator[HiveTest, None, None]: """ - Test (base) used to deploy the main client to be used throughout all tests. + Test (base) used to deploy the main client to be used throughout + all tests. Individual test results are reported by the per-test + `execute_hive_test` fixture. """ base_name = "base_hive_test" base_file = session_temp_folder / base_name @@ -269,10 +273,10 @@ def base_hive_test( test = HiveTest(**json.load(f)) else: test = test_suite.start_test( - name="Base Hive Test", + name="Client Deployment", description=( - "Base test used to deploy the main client to be used " - "throughout all tests." + "Infrastructure test used to deploy the " + "client shared by all execute tests." ), ) with open(base_file, "w") as f: @@ -293,12 +297,6 @@ def base_hive_test( yield test - test_pass = True - test_details = "All tests have completed" - if request.session.testsfailed > 0: - test_pass = False - test_details = "One or more tests have failed" - with FileLock(users_lock_file): with open(users_file, "r") as f: users = json.load(f) @@ -308,7 +306,8 @@ def base_hive_test( if users == 0: test.end( result=HiveTestResult( - test_pass=test_pass, details=test_details + test_pass=True, + details="Client deployment completed.", ) ) base_file.unlink() @@ -423,3 +422,117 @@ def eth_rpc( max_transactions_per_batch=max_transactions_per_batch, testing_rpc=testing_rpc, ) + + +@pytest.fixture(scope="function") +def test_case_description( + request: pytest.FixtureRequest, +) -> str: + """Create a description for the current execute test case.""" + node = request.node + docstring = node.obj.__doc__ if hasattr(node, "obj") else None + if docstring: + docstring = docstring.strip() + else: + docstring = "No documentation available." + + description = ( + f"Test: {node.nodeid}

{docstring}" + ) + return description + + +@pytest.fixture(autouse=True, scope="function") +def execute_hive_test( + request: pytest.FixtureRequest, + test_suite: HiveTestSuite, + test_case_description: str, +) -> Generator[HiveTest, None, None]: + """ + Create an individual Hive test record for each execute test. + + This allows each test to appear separately in the Hive UI with + its own pass/fail status, rather than aggregating all results + into a single test entry. + """ + test: HiveTest = test_suite.start_test( + name=request.node.name, + description=test_case_description, + ) + yield test + + try: + captured: list[str] = [] + setup_out = "" + for phase in ("setup", "call", "teardown"): + report = getattr(request.node, f"result_{phase}", None) + if report: + stdout = report.capstdout or "None" + stderr = report.capstderr or "None" + + if phase == "setup": + setup_out = stdout + if phase == "call" and stdout.startswith(setup_out): + stdout = stdout.removeprefix(setup_out) + + captured.append( + f"# Captured Output from Test" + f" {phase.capitalize()}\n\n" + f"## stdout:\n{stdout}\n" + f"## stderr:\n{stderr}\n" + ) + + captured_output = "\n".join(captured) + + result_call = getattr(request.node, "result_call", None) + result_setup = getattr(request.node, "result_setup", None) + result_teardown = getattr(request.node, "result_teardown", None) + + if result_call and result_call.passed: + test_passed = True + test_result_details = "Test passed.\n\n" + captured_output + elif result_call and not result_call.passed: + test_passed = False + test_result_details = ( + result_call.longreprtext + "\n" + captured_output + ) + elif result_setup and not result_setup.passed: + test_passed = False + test_result_details = ( + "Test setup failed.\n\n" + + result_setup.longreprtext + + "\n" + + captured_output + ) + elif result_teardown and not result_teardown.passed: + test_passed = False + test_result_details = ( + "Test teardown failed.\n\n" + + result_teardown.longreprtext + + "\n" + + captured_output + ) + else: + test_passed = False + test_result_details = ( + "Test failed for unknown reason.\n\n" + captured_output + ) + + test.end( + result=HiveTestResult( + test_pass=test_passed, + details=test_result_details, + ) + ) + logger.verbose(f"Finished processing test: {request.node.nodeid}") + + except Exception as e: + logger.verbose( + f"Error processing test {request.node.nodeid}: {str(e)}" + ) + test.end( + result=HiveTestResult( + test_pass=False, + details=(f"Exception whilst processing test result: {str(e)}"), + ) + )