diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a09641cf..51c68f9e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,5 +36,5 @@ jobs: # Running our heavy tests in parallel would congest resources. # Each test still executes parallelly anyway. run: | - git submodule update --init + bash scripts/fetch-eest-fixtures.sh cargo test --release --workspace -- --test-threads=1 diff --git a/.gitignore b/.gitignore index 8535fcdc..c98f0c73 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .vscode /target perf.* +crates/pevm/tests/ethereum/fixtures/ diff --git a/.gitmodules b/.gitmodules index f14a5931..e69de29b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "tests/ethereum/tests"] - path = crates/pevm/tests/ethereum/tests - url = https://github.com/ethereum/tests diff --git a/CLAUDE.md b/CLAUDE.md index a9383ff9..5e1c1f87 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,7 +24,7 @@ cargo fmt --all taplo fmt --option reorder_keys=true # TOML formatting # Tests (must use --test-threads=1 to avoid resource contention) -git submodule update --init # Required once: pulls ethereum/tests submodule +bash scripts/fetch-eest-fixtures.sh # Required once: downloads EEST state test fixtures cargo test --workspace --release -- --test-threads=1 # Run a single test @@ -61,7 +61,7 @@ cargo bench --features global-alloc --bench gigagas ### Testing layout (`crates/pevm/tests/`) -- `ethereum/` — general state tests from the official `ethereum/tests` submodule +- `ethereum/` — EEST state test fixtures (`fixtures_stable.tar.gz` from `ethereum/execution-spec-tests` releases, downloaded via `scripts/fetch-eest-fixtures.sh`, gitignored) - `evm/` — mocked blocks (raw transfers, ERC-20, Uniswap, mixed, beneficiary edge cases) - `mainnet/` — snapshots of real Ethereum mainnet blocks - `common/` — shared test harness (`runner.rs`, `storage.rs`, `snapshot_data.rs`) diff --git a/crates/pevm/tests/ethereum/main.rs b/crates/pevm/tests/ethereum/main.rs index 1342c71d..2edc6842 100644 --- a/crates/pevm/tests/ethereum/main.rs +++ b/crates/pevm/tests/ethereum/main.rs @@ -1,13 +1,12 @@ -//! Basing on . -//! These tests may seem useless: -//! - They only have one transaction. -//! - REVM already tests them. +//! Runs the Ethereum Execution Spec Tests (EEST) state test fixtures. //! -//! Nevertheless, they are important: -//! - REVM doesn't test very tightly (not matching on expected failures, skipping tests, etc.). -//! - We must use a REVM fork (for distinguishing explicit & implicit reads, etc.). -//! - We use custom handlers (for lazy-updating the beneficiary account, etc.) that require "re-testing". -//! - Help outline the minimal state commitment logic for pevm. +//! Before running, download the fixtures once with: +//! bash scripts/fetch-eest-fixtures.sh +//! +//! Why these tests matter even though EEST already runs them upstream: +//! - We use a revm fork that distinguishes explicit vs implicit reads. +//! - We use custom handlers (lazy beneficiary updates, etc.) that need re-testing. +//! - They validate our minimal state-commitment logic against the spec. use pevm::chain::PevmEthereum; use pevm::{ @@ -26,7 +25,7 @@ use revm::primitives::eip4844::{ }; use revm::primitives::hardfork::SpecId; use revm::state::{AccountInfo, Bytecode}; -use revm_statetest_types::{Env, SpecName, Test, TestSuite, TestUnit, TransactionParts}; +use revm_statetest_types::{Env, Test, TestSuite, TestUnit, TransactionParts}; use revme::cmd::statetest::{ merkle_trie::{log_rlp_hash, state_merkle_trie_root}, utils::recover_address, @@ -106,15 +105,93 @@ fn build_tx_env(path: &Path, tx: &TransactionParts, test: &Test) -> Option bool { + match exception { + "TransactionException.INTRINSIC_GAS_TOO_LOW" => { + matches!( + error, + InvalidTransaction::CallGasCostMoreThanGasLimit { .. } + ) + } + "TransactionException.INTRINSIC_GAS_BELOW_FLOOR_GAS_COST" => { + matches!(error, InvalidTransaction::GasFloorMoreThanGasLimit { .. }) + } + // Both can indicate the sender can't cover max_fee * gas_limit. + "TransactionException.INSUFFICIENT_ACCOUNT_FUNDS" + | "TransactionException.GASLIMIT_PRICE_PRODUCT_OVERFLOW" => { + matches!( + error, + InvalidTransaction::LackOfFundForMaxFee { .. } + | InvalidTransaction::OverflowPaymentInTransaction + ) + } + "TransactionException.SENDER_NOT_EOA" => *error == InvalidTransaction::RejectCallerWithCode, + "TransactionException.NONCE_IS_MAX" => { + *error == InvalidTransaction::NonceOverflowInTransaction + } + "TransactionException.PRIORITY_GREATER_THAN_MAX_FEE_PER_GAS" => { + *error == InvalidTransaction::PriorityFeeGreaterThanMaxFee + } + "TransactionException.INSUFFICIENT_MAX_FEE_PER_GAS" => { + *error == InvalidTransaction::GasPriceLessThanBasefee + } + "TransactionException.INSUFFICIENT_MAX_FEE_PER_BLOB_GAS" => { + matches!(error, InvalidTransaction::BlobGasPriceGreaterThanMax { .. }) + } + "TransactionException.TYPE_1_TX_PRE_FORK" => { + *error == InvalidTransaction::Eip2930NotSupported + } + "TransactionException.TYPE_2_TX_PRE_FORK" => { + *error == InvalidTransaction::Eip1559NotSupported + } + "TransactionException.TYPE_3_TX_PRE_FORK" => { + *error == InvalidTransaction::Eip4844NotSupported + } + "TransactionException.TYPE_3_TX_ZERO_BLOBS" => *error == InvalidTransaction::EmptyBlobs, + "TransactionException.TYPE_3_TX_CONTRACT_CREATION" => { + *error == InvalidTransaction::BlobCreateTransaction + } + "TransactionException.TYPE_3_TX_INVALID_BLOB_VERSIONED_HASH" => { + *error == InvalidTransaction::BlobVersionNotSupported + } + // Both map to an excessive total blob count/gas. + "TransactionException.TYPE_3_TX_BLOB_COUNT_EXCEEDED" + | "TransactionException.TYPE_3_TX_MAX_BLOB_GAS_ALLOWANCE_EXCEEDED" => { + matches!(error, InvalidTransaction::TooManyBlobs { .. }) + } + "TransactionException.TYPE_4_TX_PRE_FORK" => { + *error == InvalidTransaction::Eip7702NotSupported } + "TransactionException.TYPE_4_EMPTY_AUTHORIZATION_LIST" => { + *error == InvalidTransaction::EmptyAuthorizationList + } + // revm has no dedicated variant for type-4 contract creation; accept any error. + "TransactionException.TYPE_4_TX_CONTRACT_CREATION" => true, + "TransactionException.INITCODE_SIZE_EXCEEDED" => { + *error == InvalidTransaction::CreateInitCodeSizeLimit + } + // EIP-7825 per-tx cap or the block gas limit check in legacy tests. + "TransactionException.GAS_ALLOWANCE_EXCEEDED" => { + matches!( + error, + InvalidTransaction::TxGasLimitGreaterThanCap { .. } + | InvalidTransaction::CallerGasLimitMoreThanBlock + ) + } + _ => panic!("Unknown EEST exception: {exception}"), + } +} + +// EEST exception strings can be pipe-separated: "A|B" means either A or B is acceptable. +fn exception_matches(exception: &str, error: &InvalidTransaction) -> bool { + exception + .split('|') + .any(|part| single_exception_matches(part, error)) +} +fn run_test_unit(path: &Path, unit: TestUnit) { + unit.post.into_par_iter().for_each(|(spec_name, tests)| { tests.into_par_iter().for_each(|test| { let tx_env = build_tx_env(path, &unit.transaction, &test); if test.expect_exception.is_some() && tx_env.is_none() { @@ -150,55 +227,37 @@ fn run_test_unit(path: &Path, unit: TestUnit) { test.expect_exception.as_deref(), Pevm::default().execute_revm_parallel( &PevmEthereum::mainnet(), - &InMemoryStorage::new(chain_state.clone(), Arc::new(bytecodes), Default::default()), + &InMemoryStorage::new( + chain_state.clone(), + Arc::new(bytecodes), + Default::default(), + ), spec_id, build_block_env(&unit.env, spec_id), vec![tx_env.unwrap()], NonZeroUsize::MIN, ), ) { - // Skipping special cases where REVM returns `Ok` on unsupported features. - (Some("TR_TypeNotSupported"), Ok(_)) => {} - // Remaining tests that expect execution to fail -> match error (Some(exception), Err(PevmError::ExecutionError(error))) => { let pevm::ExecutionError::Transaction(error) = error else { - panic!("Mismatched error!\nPath: {path:?}\nExpected: {exception:?}\nGot: {error:?}") - }; - - // TODO: Cleaner code would be nice.. - let res = match exception { - "TR_TypeNotSupported" => true, // REVM is yielding arbitrary errors in these cases. - "TR_NonceHasMaxValue" => error == InvalidTransaction::NonceOverflowInTransaction, - "SenderNotEOA" => error == InvalidTransaction::RejectCallerWithCode, - "TR_NoFundsX" => matches!(error, InvalidTransaction::LackOfFundForMaxFee{..} | InvalidTransaction::OverflowPaymentInTransaction), - "TransactionException.INSUFFICIENT_MAX_FEE_PER_BLOB_GAS" => matches!(error, InvalidTransaction::BlobGasPriceGreaterThanMax { .. }), - "TR_BLOBCREATE" => error == InvalidTransaction::BlobCreateTransaction, - "TR_GasLimitReached" => error == InvalidTransaction::CallerGasLimitMoreThanBlock, - "TR_TipGtFeeCap" => error == InvalidTransaction::PriorityFeeGreaterThanMaxFee, - - "TR_NoFundsOrGas" | "IntrinsicGas" | "TR_IntrinsicGas" | "TransactionException.INTRINSIC_GAS_TOO_LOW" => - matches!(error, InvalidTransaction::CallGasCostMoreThanGasLimit{..}), - "TR_FeeCapLessThanBlocks" | "TransactionException.INSUFFICIENT_MAX_FEE_PER_GAS" => error == InvalidTransaction::GasPriceLessThanBasefee, - "TR_NoFunds" | "TransactionException.INSUFFICIENT_ACCOUNT_FUNDS" => matches!(error, InvalidTransaction::LackOfFundForMaxFee {..}), - "TR_EMPTYBLOB" | "TransactionException.TYPE_3_TX_ZERO_BLOBS" => error == InvalidTransaction::EmptyBlobs, - "TR_BLOBLIST_OVERSIZE" | "TransactionException.TYPE_3_TX_BLOB_COUNT_EXCEEDED" => matches!(error, InvalidTransaction::TooManyBlobs {..}), - "TR_BLOBVERSION_INVALID" | "TransactionException.TYPE_3_TX_INVALID_BLOB_VERSIONED_HASH" => error == InvalidTransaction::BlobVersionNotSupported, - "TransactionException.TYPE_3_TX_PRE_FORK|TransactionException.TYPE_3_TX_ZERO_BLOBS" | "TransactionException.TYPE_3_TX_PRE_FORK" => error == InvalidTransaction::Eip4844NotSupported, - "TransactionException.INITCODE_SIZE_EXCEEDED" | "TR_InitCodeLimitExceeded" => error == InvalidTransaction::CreateInitCodeSizeLimit, - _ => panic!("Mismatched error!\nPath: {path:?}\nExpected: {exception:?}\nGot: {error:?}") + panic!( + "Mismatched error!\nPath: {path:?}\nExpected: {exception:?}\nGot: {error:?}" + ) }; - assert!(res); + assert!( + exception_matches(exception, &error), + "Mismatched error!\nPath: {path:?}\nExpected: {exception:?}\nGot: {error:?}" + ); } - // Tests that expect execution to succeed -> match post state root (None, Ok(exec_results)) => { assert!(exec_results.len() == 1); - let PevmTxExecutionResult {receipt, state} = exec_results[0].clone(); + let PevmTxExecutionResult { receipt, state } = exec_results[0].clone(); let logs_root = log_rlp_hash(&receipt.logs); assert_eq!(logs_root, test.logs, "Mismatched logs root for {path:?}"); - // This is a good reference for a minimal state/DB commitment logic for - // pevm/revm to meet the Ethereum specs throughout the eras. + // Apply output state on top of pre-state to compute the post-state root. + // This is the minimal state-commitment logic pevm must satisfy. for (address, account) in state { if let Some(account) = account { let chain_state_account = chain_state.entry(address).or_default(); @@ -206,31 +265,45 @@ fn run_test_unit(path: &Path, unit: TestUnit) { chain_state_account.nonce = account.nonce; chain_state_account.code_hash = account.code_hash; chain_state_account.code = account.code; - chain_state_account.storage.extend(account.storage.into_iter()); + chain_state_account + .storage + .extend(account.storage.into_iter()); } else { chain_state.remove(&address); } } - // TODO: Implement our own state root calculation function to remove - // this conversion to [PlainAccount] - let plain_chain_state = chain_state.into_iter().map(|(address, account)| { - (address, PlainAccount { - info: AccountInfo { - balance: account.balance, - nonce: account.nonce, - code_hash: account.code_hash.unwrap_or(KECCAK_EMPTY), - code: account.code.map(Bytecode::from), - account_id: None, - }, - storage: account.storage.into_iter().collect(), - })}).collect::>(); - let state_root = - state_merkle_trie_root(plain_chain_state.iter().map(|(address, account)| (*address, account))); - assert_eq!(state_root, test.hash, "Mismatched state root for {path:?}"); - } - unexpected_res => { - panic!("pevm doesn't match the test's expectation for {path:?}: {unexpected_res:?}") + // TODO: Replace this PlainAccount conversion with our own trie function. + let plain_chain_state = chain_state + .into_iter() + .map(|(address, account)| { + ( + address, + PlainAccount { + info: AccountInfo { + balance: account.balance, + nonce: account.nonce, + code_hash: account.code_hash.unwrap_or(KECCAK_EMPTY), + code: account.code.map(Bytecode::from), + account_id: None, + }, + storage: account.storage.into_iter().collect(), + }, + ) + }) + .collect::>(); + let state_root = state_merkle_trie_root( + plain_chain_state + .iter() + .map(|(address, account)| (*address, account)), + ); + assert_eq!( + state_root, test.hash, + "Mismatched state root for {path:?}" + ); } + unexpected_res => panic!( + "pevm doesn't match the test's expectation for {path:?}: {unexpected_res:?}" + ), } }); }); @@ -238,36 +311,31 @@ fn run_test_unit(path: &Path, unit: TestUnit) { #[test] fn ethereum_state_tests() { - WalkDir::new("tests/ethereum/tests/GeneralStateTests") + let fixtures = std::path::PathBuf::from("tests/ethereum/fixtures/state_tests"); + assert!( + fixtures.exists(), + "EEST fixtures not found at {fixtures:?}. Run: bash scripts/fetch-eest-fixtures.sh" + ); + + WalkDir::new(fixtures) .into_iter() .filter_map(Result::ok) .map(DirEntry::into_path) .filter(|path| path.extension() == Some("json".as_ref())) .filter(|path| { - // Skip tests `revm` cannot handle. We'll support these as we replace - // `revm` with our EVM implementation, or maybe migrate to newer state - // tests anyway. - !matches!( - path.file_name().unwrap().to_str().unwrap(), - "InitCollision.json" - | "InitCollisionParis.json" - | "create2collisionStorage.json" - | "create2collisionStorageParis.json" - | "dynamicAccountOverwriteEmpty_Paris.json" - | "RevertInCreateInInitCreate2.json" - | "dynamicAccountOverwriteEmpty.json" - | "RevertInCreateInInit.json" - | "RevertInCreateInInit_Paris.json" - | "RevertInCreateInInitCreate2Paris.json" - | "ValueOverflow.json" - | "ValueOverflowParis.json" - ) + let path_str = path.to_str().unwrap(); + let name = path.file_name().unwrap().to_str().unwrap(); + // Skip tests that exercise create-collision edge cases our revm fork handles + // incorrectly. Revisit when we replace revm with our own EVM (issue #382). + !path_str.contains("eip7610_create_collision") + && !matches!( + name, + "create2collisionStorageParis.json" + | "dynamicAccountOverwriteEmpty_Paris.json" + | "RevertInCreateInInit_Paris.json" + | "RevertInCreateInInitCreate2Paris.json" + ) }) - // For development, we can further filter to run a small set of tests, - // or filter out time-consuming tests like: - // - stTimeConsuming/** - // - vmPerformance/loopMul.json - // - stQuadraticComplexityTest/Call50000_sha256.json .collect::>() .par_iter() .for_each(|path| { diff --git a/crates/pevm/tests/ethereum/tests b/crates/pevm/tests/ethereum/tests deleted file mode 160000 index c67e485f..00000000 --- a/crates/pevm/tests/ethereum/tests +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c67e485ff8b5be9abc8ad15345ec21aa22e290d9 diff --git a/scripts/fetch-eest-fixtures.sh b/scripts/fetch-eest-fixtures.sh new file mode 100755 index 00000000..5a80625e --- /dev/null +++ b/scripts/fetch-eest-fixtures.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Downloads and extracts the stable Ethereum Execution Spec Test (EEST) fixtures +# into crates/pevm/tests/ethereum/fixtures/. Run this once before running tests: +# +# bash scripts/fetch-eest-fixtures.sh +# +# To pin to a specific release, set EEST_VERSION before running: +# +# EEST_VERSION=v5.4.0 bash scripts/fetch-eest-fixtures.sh + +set -euo pipefail + +EEST_VERSION="${EEST_VERSION:-$( + curl -sf https://api.github.com/repos/ethereum/execution-spec-tests/releases/latest \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['tag_name'])" +)}" +DEST="crates/pevm/tests/ethereum/fixtures" +URL="https://github.com/ethereum/execution-spec-tests/releases/download/${EEST_VERSION}/fixtures_stable.tar.gz" + +echo "Fetching EEST ${EEST_VERSION} -> ${DEST}/" +mkdir -p "${DEST}" +curl -fL "${URL}" | tar -xz --strip-components=1 -C "${DEST}" +echo "Done."