diff --git a/packages/testing/pyproject.toml b/packages/testing/pyproject.toml index ac0025be610..329e8b3c1c2 100644 --- a/packages/testing/pyproject.toml +++ b/packages/testing/pyproject.toml @@ -52,6 +52,7 @@ dependencies = [ "ckzg>=2.1.3,<3", "tenacity>=9.0.0,<10", "Jinja2>=3,<4", + "eth-remerkleable==0.1.31", ] [project.urls] diff --git a/packages/testing/src/execution_testing/fixtures/__init__.py b/packages/testing/src/execution_testing/fixtures/__init__.py index 18d4b3a1183..c24a312a2be 100644 --- a/packages/testing/src/execution_testing/fixtures/__init__.py +++ b/packages/testing/src/execution_testing/fixtures/__init__.py @@ -14,6 +14,15 @@ BlockchainFixture, BlockchainFixtureCommon, ) +from .blockchain_rest_ssz import ( + BlockchainRestSszFixture, + FixtureRestExecutionPayload, + FixtureRestNegative, + FixtureRestPayload, + FixtureRestWithdrawal, + PayloadStatusV2, + RestPayloadMutation, +) from .collector import ( FixtureCollector, TestInfo, @@ -37,15 +46,22 @@ "BlockchainEngineXFixture", "BlockchainFixture", "BlockchainFixtureCommon", + "BlockchainRestSszFixture", "FixtureCollector", "FixtureConsumer", "FixtureFillingPhase", "FixtureFormat", + "FixtureRestExecutionPayload", + "FixtureRestNegative", + "FixtureRestPayload", + "FixtureRestWithdrawal", "LabeledFixtureFormat", + "PayloadStatusV2", "PreAllocGroup", "PreAllocGroupBuilder", "PreAllocGroupBuilders", "PreAllocGroups", + "RestPayloadMutation", "StateFixture", "TestInfo", "TransactionFixture", diff --git a/packages/testing/src/execution_testing/fixtures/blockchain_rest_ssz.py b/packages/testing/src/execution_testing/fixtures/blockchain_rest_ssz.py new file mode 100644 index 00000000000..c83978df773 --- /dev/null +++ b/packages/testing/src/execution_testing/fixtures/blockchain_rest_ssz.py @@ -0,0 +1,273 @@ +""" +Fixture models for the REST+SSZ blockchain test format. + +These pydantic models are the on-disk contract between ``fill`` (which emits +them) and ``consume`` (which reads them) for the REST+SSZ Engine API proposed +in execution-apis PR #793. This module is *only* the data definitions: there +is no fixture-generator logic and no consumer logic here, so a reviewer can +cross-check each field against the #793 spec table in isolation. + +The single point where the pydantic world touches the SSZ world is +:meth:`FixtureRestPayload.to_ssz`, which calls the ``envelope_bytes`` adapter +re-exported from :mod:`execution_testing.ssz`. ``remerkleable`` is never +imported here; the SSZ library stays quarantined behind the ``ssz`` package. +""" + +from enum import Enum +from functools import cached_property +from typing import Any, ClassVar, List + +from pydantic import Field, computed_field, model_validator + +from execution_testing.base_types import ( + Address, + Alloc, + Bloom, + Bytes, + CamelModel, + Hash, + HexNumber, +) +from execution_testing.forks import Amsterdam, Fork +from execution_testing.ssz import envelope_bytes + +from .base import BaseFixture +from .blockchain import FixtureHeader + + +class PayloadStatusV2(str, Enum): + """ + Status returned by the REST+SSZ ``newPayload`` endpoint (#793). + + NOTE: There is deliberately no ``INVALID_BLOCK_HASH`` member. PR #793 + removes that status from the enum -- a client that previously answered + ``INVALID_BLOCK_HASH`` now answers plain ``INVALID``. Its absence here is + an assertion about that spec change, not an oversight: do not re-add it to + "match" the legacy JSON-RPC ``PayloadStatusEnum`` in + :mod:`execution_testing.rpc.rpc_types`, which still carries it for the + pre-#793 Engine API. + """ + + VALID = "VALID" + INVALID = "INVALID" + SYNCING = "SYNCING" + ACCEPTED = "ACCEPTED" + + +class RestPayloadMutation(str, Enum): + """ + Wire-level mutation applied to a valid payload for a transport negative + test. + + Each member names a way to corrupt the request *envelope* (not its + consensus contents) so the server is expected to reject it with an HTTP + error before any consensus validation runs. The mutation logic itself + lives in the consume plugin (a later PR); this enum only fixes the set of + mutations the fixtures may reference. + """ + + TRUNCATE_BODY = "TRUNCATE_BODY" + NONMONOTONIC_OFFSET = "NONMONOTONIC_OFFSET" + WRONG_CONTENT_TYPE = "WRONG_CONTENT_TYPE" + FORK_MISMATCH = "FORK_MISMATCH" + + +class FixtureRestWithdrawal(CamelModel): + """A validator withdrawal inside a REST+SSZ execution payload.""" + + index: HexNumber + validator_index: HexNumber + address: Address + amount: HexNumber + + +class FixtureRestExecutionPayload(CamelModel): + """ + A REST+SSZ ``ExecutionPayloadEnvelope`` in human-readable fixture form, + fork-parameterized. + + ``fork`` (a registry key such as ``"Amsterdam"`` / ``"Cancun"``) selects + the per-fork SSZ shape this represents; it is fixture metadata, not an SSZ + field. The remaining fields are the union of every fork's payload and + envelope fields, in canonical order, so a single model serves all forks. + Fields a fork does not carry are left ``None``: + + * base fields (``parent_hash`` .. ``transactions``) -- every fork, + * ``withdrawals`` -- Shanghai+, + * ``blob_gas_used`` / ``excess_blob_gas`` -- Cancun+, + * ``block_access_list`` / ``slot_number`` -- Amsterdam+, + * ``parent_beacon_block_root`` (envelope) -- Cancun+, + * ``execution_requests`` (envelope) -- Prague+. + + The container actually built for ``fork`` is chosen from the registries in + :mod:`execution_testing.ssz`, which include only the fields that fork + declares; supplying a field a fork lacks is harmless, omitting one it needs + raises. The drift round-trip test fails loudly if this model and the + remerkleable container diverge for a fork. + """ + + fork: str + parent_hash: Hash + fee_recipient: Address + state_root: Hash + receipts_root: Hash + logs_bloom: Bloom + prev_randao: Hash + block_number: HexNumber + gas_limit: HexNumber + gas_used: HexNumber + timestamp: HexNumber + extra_data: Bytes + base_fee_per_gas: HexNumber + block_hash: Hash + transactions: List[Bytes] + withdrawals: List[FixtureRestWithdrawal] | None = None + blob_gas_used: HexNumber | None = None + excess_blob_gas: HexNumber | None = None + block_access_list: Bytes | None = Field( + None, description="RLP-serialized EIP-7928 Block Access List" + ) + slot_number: HexNumber | None = None + parent_beacon_block_root: Hash | None = None + execution_requests: List[Bytes] | None = None + + +class FixtureRestPayload(CamelModel): + """ + A single ``newPayload`` directive in a REST+SSZ blockchain test. + + Carries the execution payload, the expected :class:`PayloadStatusV2` + response, and -- for ``INVALID`` cases -- an optional ``validationError`` + string. It deliberately does *not* carry ``expectedBlobVersionedHashes`` + (removed in #793) and there is no field for an ``INVALID_BLOCK_HASH`` + verdict (removed from the status enum). + """ + + payload: FixtureRestExecutionPayload + expected_status: PayloadStatusV2 + validation_error: str | None = Field(None) + + @model_validator(mode="before") + @classmethod + def _strip_ssz_computed_field(cls, data: Any) -> Any: + """ + Drop the ``ssz`` computed field when re-reading a fixture from disk. + + ``ssz`` is emitted for human inspection but is not an input field; + without this the ``extra="forbid"`` config rejects round-tripped + fixtures. + """ + if isinstance(data, dict): + data.pop("ssz", None) + return data + + def to_ssz(self) -> bytes: + """ + Return the SSZ-encoded ``ExecutionPayloadEnvelope`` wire bytes. + + The one place the pydantic layer crosses into SSZ: it forwards the + payload's field values (and its fork) to the fork-parameterized + ``envelope_bytes`` adapter and returns the raw bytes. Keep this thin -- + no logic beyond the call-through. The consume layer uses these bytes; + the fixture's stored ``ssz`` hex is only for inspection. + """ + p = self.payload + return envelope_bytes( + p.fork, + parent_hash=p.parent_hash, + fee_recipient=p.fee_recipient, + state_root=p.state_root, + receipts_root=p.receipts_root, + logs_bloom=p.logs_bloom, + prev_randao=p.prev_randao, + block_number=p.block_number, + gas_limit=p.gas_limit, + gas_used=p.gas_used, + timestamp=p.timestamp, + extra_data=p.extra_data, + base_fee_per_gas=p.base_fee_per_gas, + block_hash=p.block_hash, + transactions=p.transactions, + withdrawals=( + None + if p.withdrawals is None + else [w.model_dump() for w in p.withdrawals] + ), + blob_gas_used=p.blob_gas_used, + excess_blob_gas=p.excess_blob_gas, + block_access_list=p.block_access_list, + slot_number=p.slot_number, + parent_beacon_block_root=p.parent_beacon_block_root, + execution_requests=p.execution_requests, + ) + + @computed_field # type: ignore[prop-decorator] + @cached_property + def ssz(self) -> Bytes: + """SSZ wire bytes as a hex string, stored for human inspection.""" + return Bytes(self.to_ssz()) + + +class FixtureRestNegative(CamelModel): + """ + A transport-only negative directive: a wire-level error, not a consensus + verdict. + + Unlike :class:`FixtureRestPayload`, whose expected outcome is a + consensus :class:`PayloadStatusV2`, the expected outcome here is an HTTP + error code. A valid base payload is corrupted per ``mutation`` and the + server is expected to answer with ``expected_http_status`` before reaching + consensus validation. That difference in outcome -- HTTP code vs. consensus + status -- is the whole reason this is a separate model. + + Speculative until the consume plugin implements the mutation logic: the + mutation set is sketched ahead of its consumer from the known negative + vectors, since adding a mutation variant later is cheaper than reshaping + the model. + """ + + payload: FixtureRestExecutionPayload + mutation: RestPayloadMutation + expected_http_status: int + + +class BlockchainRestSszFixture(BaseFixture): + """Top-level REST+SSZ blockchain test fixture.""" + + format_name: ClassVar[str] = "blockchain_test_rest_ssz" + description: ClassVar[str] = ( + "Tests that generate a blockchain test fixture for the REST+SSZ " + "Engine API." + ) + + fork: Fork = Field(..., alias="network") + pre: Alloc + genesis: FixtureHeader = Field(..., alias="genesisBlockHeader") + payloads: List[FixtureRestPayload] = Field(..., alias="restPayloads") + last_forkchoice_head: Hash + """ + Head the forkchoice settles on after the final payload. + + Typed as a plain :class:`Hash` for now. Its semantics depend on the + still-unresolved forkchoice-atomicity question in #793: if forkchoice + stays atomic with ``newPayload`` this single hash suffices, but if it + splits into a separate payload-attributes call this field's shape will + need to change. Treat the shape as provisional until that settles. + """ + + def get_fork(self) -> Fork | None: + """Return the fixture's fork.""" + return self.fork + + @classmethod + def supports_fork(cls, fork: Fork) -> bool: + """ + Return whether the fixture can be generated for ``fork``. + + The models and the SSZ bridge are fork-parameterized (every payload + fork has a container), but the REST+SSZ Engine API itself is introduced + by PR #793 at Amsterdam, so generation is gated there. Lower this bound + if/when earlier-fork REST+SSZ endpoints become testable -- no model + change is required. + """ + return fork >= Amsterdam diff --git a/packages/testing/src/execution_testing/fixtures/tests/test_blockchain_rest_ssz.py b/packages/testing/src/execution_testing/fixtures/tests/test_blockchain_rest_ssz.py new file mode 100644 index 00000000000..238cdccc533 --- /dev/null +++ b/packages/testing/src/execution_testing/fixtures/tests/test_blockchain_rest_ssz.py @@ -0,0 +1,208 @@ +""" +Tests for the REST+SSZ blockchain fixture models. + +The load-bearing test here is the drift round-trip: ``FixtureRestPayload`` and +the remerkleable ``ExecutionPayloadEnvelope`` describe the same object in two +different type systems, and they must stay in step by hand. The round-trip +(pydantic -> ``to_ssz()`` -> remerkleable decode -> field comparison) fails +loudly the moment they diverge. +""" + +from execution_testing.base_types import ( + Address, + Bloom, + Bytes, + Hash, + HexNumber, +) +from execution_testing.fixtures import BaseFixture +from execution_testing.fixtures.blockchain_rest_ssz import ( + BlockchainRestSszFixture, + FixtureRestExecutionPayload, + FixtureRestPayload, + FixtureRestWithdrawal, + PayloadStatusV2, +) +from execution_testing.ssz import ( + ExecutionPayloadEnvelopeAmsterdam, + ExecutionPayloadEnvelopeCancun, + decode_bytes, +) + +# Transactions of differing lengths, on purpose: the two-level +# `List[ByteList, N]` only exercises its inner offset table when the elements +# differ in length. +TRANSACTIONS = [ + Bytes(bytes.fromhex("02f86b01")), + Bytes(bytes.fromhex("01" * 21)), + Bytes(bytes.fromhex("03" * 5)), +] +EXECUTION_REQUESTS = [ + Bytes(bytes.fromhex("00aa")), + Bytes(bytes.fromhex("01bbcc")), +] + + +def _execution_payload() -> FixtureRestExecutionPayload: + return FixtureRestExecutionPayload( + fork="Amsterdam", + parent_hash=Hash(bytes.fromhex("aa" * 32)), + fee_recipient=Address(bytes.fromhex("bb" * 20)), + state_root=Hash(bytes.fromhex("cc" * 32)), + receipts_root=Hash(bytes.fromhex("dd" * 32)), + logs_bloom=Bloom(bytes.fromhex("00" * 256)), + prev_randao=Hash(bytes.fromhex("ee" * 32)), + block_number=HexNumber(21_000_000), + gas_limit=HexNumber(30_000_000), + gas_used=HexNumber(21_000), + timestamp=HexNumber(1_700_000_000), + extra_data=Bytes(bytes.fromhex("dead")), + base_fee_per_gas=HexNumber(10**18), + block_hash=Hash(bytes.fromhex("ff" * 32)), + transactions=list(TRANSACTIONS), + withdrawals=[ + FixtureRestWithdrawal( + index=HexNumber(7), + validator_index=HexNumber(42), + address=Address(bytes.fromhex("11" * 20)), + amount=HexNumber(32_000_000_000), + ) + ], + blob_gas_used=HexNumber(131_072), + excess_blob_gas=HexNumber(0), + block_access_list=Bytes(bytes.fromhex("c0de")), + slot_number=HexNumber(9_999), + parent_beacon_block_root=Hash(bytes.fromhex("12" * 32)), + execution_requests=list(EXECUTION_REQUESTS), + ) + + +def _payload_directive() -> FixtureRestPayload: + return FixtureRestPayload( + payload=_execution_payload(), + expected_status=PayloadStatusV2.VALID, + ) + + +def test_to_ssz_round_trips_field_for_field() -> None: + """ + Pydantic -> ``to_ssz()`` -> remerkleable decode reconstructs every field. + + This is the drift guard between ``FixtureRestPayload`` and + ``ExecutionPayloadEnvelope``. If a fork adds a field to one type system but + not the other, the decoded value no longer matches and this test fails. + """ + directive = _payload_directive() + src = directive.payload + + decoded = decode_bytes( + ExecutionPayloadEnvelopeAmsterdam, directive.to_ssz() + ) + payload = decoded.payload + + assert bytes(payload.parent_hash) == bytes(src.parent_hash) + assert bytes(payload.fee_recipient) == bytes(src.fee_recipient) + assert bytes(payload.state_root) == bytes(src.state_root) + assert bytes(payload.receipts_root) == bytes(src.receipts_root) + assert bytes(payload.logs_bloom) == bytes(src.logs_bloom) + assert bytes(payload.prev_randao) == bytes(src.prev_randao) + assert int(payload.block_number) == int(src.block_number) + assert int(payload.gas_limit) == int(src.gas_limit) + assert int(payload.gas_used) == int(src.gas_used) + assert int(payload.timestamp) == int(src.timestamp) + assert bytes(payload.extra_data) == bytes(src.extra_data) + assert int(payload.base_fee_per_gas) == int(src.base_fee_per_gas) + assert bytes(payload.block_hash) == bytes(src.block_hash) + assert [bytes(tx) for tx in payload.transactions] == [ + bytes(tx) for tx in src.transactions + ] + # An Amsterdam payload carries every (now-optional) field; narrow them. + assert src.withdrawals is not None + assert src.blob_gas_used is not None + assert src.excess_blob_gas is not None + assert src.block_access_list is not None + assert src.slot_number is not None + assert src.parent_beacon_block_root is not None + assert src.execution_requests is not None + assert len(payload.withdrawals) == len(src.withdrawals) + decoded_w = payload.withdrawals[0] + src_w = src.withdrawals[0] + assert int(decoded_w.index) == int(src_w.index) + assert int(decoded_w.validator_index) == int(src_w.validator_index) + assert bytes(decoded_w.address) == bytes(src_w.address) + assert int(decoded_w.amount) == int(src_w.amount) + assert int(payload.blob_gas_used) == int(src.blob_gas_used) + assert int(payload.excess_blob_gas) == int(src.excess_blob_gas) + assert bytes(payload.block_access_list) == bytes(src.block_access_list) + assert int(payload.slot_number) == int(src.slot_number) + assert bytes(decoded.parent_beacon_block_root) == bytes( + src.parent_beacon_block_root + ) + assert [bytes(r) for r in decoded.execution_requests] == [ + bytes(r) for r in src.execution_requests + ] + + +def test_to_ssz_matches_stored_ssz_hex() -> None: + """The stored ``ssz`` inspection field equals the live ``to_ssz()``.""" + directive = _payload_directive() + assert bytes(directive.ssz) == directive.to_ssz() + + +def test_to_ssz_dispatches_on_fork() -> None: + """ + A non-Amsterdam directive builds that fork's envelope. + + A ``Cancun`` payload omits the Amsterdam-only fields + (``block_access_list``, ``slot_number``) and ``execution_requests`` + (Prague+); ``to_ssz`` must produce bytes that decode as a Cancun + envelope, not the Amsterdam one. + """ + payload = FixtureRestExecutionPayload( + fork="Cancun", + parent_hash=Hash(bytes.fromhex("aa" * 32)), + fee_recipient=Address(bytes.fromhex("bb" * 20)), + state_root=Hash(bytes.fromhex("cc" * 32)), + receipts_root=Hash(bytes.fromhex("dd" * 32)), + logs_bloom=Bloom(bytes.fromhex("00" * 256)), + prev_randao=Hash(bytes.fromhex("ee" * 32)), + block_number=HexNumber(21_000_000), + gas_limit=HexNumber(30_000_000), + gas_used=HexNumber(21_000), + timestamp=HexNumber(1_700_000_000), + extra_data=Bytes(bytes.fromhex("dead")), + base_fee_per_gas=HexNumber(10**18), + block_hash=Hash(bytes.fromhex("ff" * 32)), + transactions=list(TRANSACTIONS), + withdrawals=[], + blob_gas_used=HexNumber(131_072), + excess_blob_gas=HexNumber(0), + parent_beacon_block_root=Hash(bytes.fromhex("12" * 32)), + # No block_access_list / slot_number / execution_requests for Cancun. + ) + directive = FixtureRestPayload( + payload=payload, expected_status=PayloadStatusV2.VALID + ) + decoded = decode_bytes(ExecutionPayloadEnvelopeCancun, directive.to_ssz()) + assert int(decoded.payload.blob_gas_used) == 131_072 + assert bytes(decoded.parent_beacon_block_root) == bytes.fromhex("12" * 32) + assert "block_access_list" not in decoded.payload.fields() + + +def test_payload_status_v2_has_no_invalid_block_hash() -> None: + """ + ``INVALID_BLOCK_HASH`` is gone from the status enum (#793). + + Its absence is an assertion about the spec change, so guard it explicitly. + """ + members = {member.value for member in PayloadStatusV2} + assert members == {"VALID", "INVALID", "SYNCING", "ACCEPTED"} + assert "INVALID_BLOCK_HASH" not in members + + +def test_format_name_is_registered() -> None: + """The fixture format registers itself with the BaseFixture registry.""" + assert ( + BaseFixture.formats["blockchain_test_rest_ssz"] + is BlockchainRestSszFixture + ) diff --git a/packages/testing/src/execution_testing/ssz/__init__.py b/packages/testing/src/execution_testing/ssz/__init__.py new file mode 100644 index 00000000000..afb2c2d65b6 --- /dev/null +++ b/packages/testing/src/execution_testing/ssz/__init__.py @@ -0,0 +1,307 @@ +""" +SSZ container types and helpers for the REST+SSZ Engine API. +""" + +from typing import Any, Mapping, Sequence, Type, TypeVar + +from remerkleable.core import View + +from .constants import ( + MAX_BYTES_PER_EXECUTION_REQUEST, + MAX_BYTES_PER_TX, + MAX_EXECUTION_REQUESTS_PER_PAYLOAD, + MAX_EXTRA_DATA_BYTES, + MAX_TXS_PER_PAYLOAD, + MAX_WITHDRAWALS_PER_PAYLOAD, + Address, + Bloom, + Bytes4, + Bytes8, + Bytes32, + Bytes48, + Hash32, + Root, + VersionedHash, +) +from .containers import ( + BUILT_PAYLOAD_BY_FORK, + EXECUTION_PAYLOAD_BODY_BY_FORK, + EXECUTION_PAYLOAD_BY_FORK, + EXECUTION_PAYLOAD_ENVELOPE_BY_FORK, + FORKCHOICE_UPDATE_BY_FORK, + PAYLOAD_ATTRIBUTES_BY_FORK, + BlobAndProofV1, + BlobAndProofV2, + BlobCellsAndProofs, + BlobsBundleV1, + BlobsBundleV2, + BlobsV1Request, + BlobsV1Response, + BlobsV2Request, + BlobsV2Response, + BlobsV3Response, + BlobsV4Request, + BlobsV4Response, + BlobV1Entry, + BlobV2Entry, + BlobV4Entry, + BodiesByHashRequest, + BodiesResponse, + BodyEntry, + BuiltPayloadAmsterdam, + BuiltPayloadCancun, + BuiltPayloadOsaka, + BuiltPayloadParis, + BuiltPayloadPrague, + BuiltPayloadShanghai, + CapabilitiesResponse, + ClientVersion, + ExecutionPayloadAmsterdam, + ExecutionPayloadBodyAmsterdam, + ExecutionPayloadBodyCancun, + ExecutionPayloadBodyOsaka, + ExecutionPayloadBodyParis, + ExecutionPayloadBodyPrague, + ExecutionPayloadBodyShanghai, + ExecutionPayloadCancun, + ExecutionPayloadEnvelopeAmsterdam, + ExecutionPayloadEnvelopeCancun, + ExecutionPayloadEnvelopeOsaka, + ExecutionPayloadEnvelopeParis, + ExecutionPayloadEnvelopePrague, + ExecutionPayloadEnvelopeShanghai, + ExecutionPayloadOsaka, + ExecutionPayloadParis, + ExecutionPayloadPrague, + ExecutionPayloadShanghai, + ForkchoiceState, + ForkchoiceUpdateAmsterdam, + ForkchoiceUpdateCancun, + ForkchoiceUpdateOsaka, + ForkchoiceUpdateParis, + ForkchoiceUpdatePrague, + ForkchoiceUpdateResponse, + ForkchoiceUpdateShanghai, + IdentityResponse, + PayloadAttributesAmsterdam, + PayloadAttributesCancun, + PayloadAttributesOsaka, + PayloadAttributesParis, + PayloadAttributesPrague, + PayloadAttributesShanghai, + PayloadStatus, + Withdrawal, +) +from .json_mapping import from_json, to_json +from .random_value import ( + RandomizationMode, + deterministic_seed, + get_random_ssz_object, +) + +ViewT = TypeVar("ViewT", bound=View) + +# The spec the containers are pinned to (execution-apis PR #793). +REFERENCE_SPEC_GIT_PATH = "src/engine/refactor-ssz.md" +REFERENCE_SPEC_VERSION = "4e0fed12d3ebc9d1ca8829331a82b97b1d1bd154" + + +def encode_bytes(value: View) -> bytes: + """Serialize an SSZ value to its canonical byte encoding.""" + return value.encode_bytes() + + +def decode_bytes(ssz_type: Type[ViewT], data: bytes) -> ViewT: + """Deserialize ``data`` into an SSZ value of ``ssz_type``.""" + return ssz_type.decode_bytes(data) + + +def hash_tree_root(value: View) -> bytes: + """Return the 32-byte SSZ `hash_tree_root` of an SSZ value.""" + return bytes(value.hash_tree_root()) + + +def _build(cls: Any, fork: str, candidates: Mapping[str, Any]) -> View: + kwargs = {} + for name in cls.fields(): + value = candidates.get(name) + if value is None: + raise ValueError( + f"{cls.__name__} ({fork}) requires field {name!r}" + ) + kwargs[name] = value + return cls(**kwargs) + + +def envelope_bytes( + fork: str, + *, + parent_hash: bytes, + fee_recipient: bytes, + state_root: bytes, + receipts_root: bytes, + logs_bloom: bytes, + prev_randao: bytes, + block_number: int, + gas_limit: int, + gas_used: int, + timestamp: int, + extra_data: bytes, + base_fee_per_gas: int, + block_hash: bytes, + transactions: Sequence[bytes], + withdrawals: Sequence[Mapping[str, Any]] | None = None, + blob_gas_used: int | None = None, + excess_blob_gas: int | None = None, + block_access_list: bytes | None = None, + slot_number: int | None = None, + parent_beacon_block_root: bytes | None = None, + execution_requests: Sequence[bytes] | None = None, +) -> bytes: + """ + Build a fork's `newPayload` envelope from plain values and return its + canonical SSZ byte encoding. + """ + if fork not in EXECUTION_PAYLOAD_ENVELOPE_BY_FORK: + raise ValueError(f"unknown fork: {fork!r}") + + def _opt_bytes(value: bytes | None) -> bytes | None: + return None if value is None else bytes(value) + + payload_values: Mapping[str, Any] = { + "parent_hash": bytes(parent_hash), + "fee_recipient": bytes(fee_recipient), + "state_root": bytes(state_root), + "receipts_root": bytes(receipts_root), + "logs_bloom": bytes(logs_bloom), + "prev_randao": bytes(prev_randao), + "block_number": block_number, + "gas_limit": gas_limit, + "gas_used": gas_used, + "timestamp": timestamp, + "extra_data": bytes(extra_data), + "base_fee_per_gas": base_fee_per_gas, + "block_hash": bytes(block_hash), + "transactions": [bytes(tx) for tx in transactions], + "withdrawals": ( + None + if withdrawals is None + else [Withdrawal(**w) for w in withdrawals] + ), + "blob_gas_used": blob_gas_used, + "excess_blob_gas": excess_blob_gas, + "block_access_list": _opt_bytes(block_access_list), + "slot_number": slot_number, + } + payload = _build(EXECUTION_PAYLOAD_BY_FORK[fork], fork, payload_values) + + envelope_values: Mapping[str, Any] = { + "payload": payload, + "parent_beacon_block_root": _opt_bytes(parent_beacon_block_root), + "execution_requests": ( + None + if execution_requests is None + else [bytes(r) for r in execution_requests] + ), + } + envelope = _build( + EXECUTION_PAYLOAD_ENVELOPE_BY_FORK[fork], fork, envelope_values + ) + return encode_bytes(envelope) + + +__all__ = ( + "BUILT_PAYLOAD_BY_FORK", + "EXECUTION_PAYLOAD_BODY_BY_FORK", + "EXECUTION_PAYLOAD_BY_FORK", + "EXECUTION_PAYLOAD_ENVELOPE_BY_FORK", + "FORKCHOICE_UPDATE_BY_FORK", + "PAYLOAD_ATTRIBUTES_BY_FORK", + "Address", + "BlobAndProofV1", + "BlobAndProofV2", + "BlobCellsAndProofs", + "BlobsBundleV1", + "BlobsBundleV2", + "BlobsV1Request", + "BlobsV1Response", + "BlobsV2Request", + "BlobsV2Response", + "BlobsV3Response", + "BlobsV4Request", + "BlobsV4Response", + "BlobV1Entry", + "BlobV2Entry", + "BlobV4Entry", + "Bloom", + "BodiesByHashRequest", + "BodiesResponse", + "BodyEntry", + "BuiltPayloadAmsterdam", + "BuiltPayloadCancun", + "BuiltPayloadOsaka", + "BuiltPayloadParis", + "BuiltPayloadPrague", + "BuiltPayloadShanghai", + "Bytes32", + "Bytes4", + "Bytes48", + "Bytes8", + "CapabilitiesResponse", + "ClientVersion", + "ExecutionPayloadAmsterdam", + "ExecutionPayloadBodyAmsterdam", + "ExecutionPayloadBodyCancun", + "ExecutionPayloadBodyOsaka", + "ExecutionPayloadBodyParis", + "ExecutionPayloadBodyPrague", + "ExecutionPayloadBodyShanghai", + "ExecutionPayloadCancun", + "ExecutionPayloadEnvelopeAmsterdam", + "ExecutionPayloadEnvelopeCancun", + "ExecutionPayloadEnvelopeOsaka", + "ExecutionPayloadEnvelopeParis", + "ExecutionPayloadEnvelopePrague", + "ExecutionPayloadEnvelopeShanghai", + "ExecutionPayloadOsaka", + "ExecutionPayloadParis", + "ExecutionPayloadPrague", + "ExecutionPayloadShanghai", + "ForkchoiceState", + "ForkchoiceUpdateAmsterdam", + "ForkchoiceUpdateCancun", + "ForkchoiceUpdateOsaka", + "ForkchoiceUpdateParis", + "ForkchoiceUpdatePrague", + "ForkchoiceUpdateResponse", + "ForkchoiceUpdateShanghai", + "Hash32", + "IdentityResponse", + "MAX_BYTES_PER_EXECUTION_REQUEST", + "MAX_BYTES_PER_TX", + "MAX_EXECUTION_REQUESTS_PER_PAYLOAD", + "MAX_EXTRA_DATA_BYTES", + "MAX_TXS_PER_PAYLOAD", + "MAX_WITHDRAWALS_PER_PAYLOAD", + "PayloadAttributesAmsterdam", + "PayloadAttributesCancun", + "PayloadAttributesOsaka", + "PayloadAttributesParis", + "PayloadAttributesPrague", + "PayloadAttributesShanghai", + "PayloadStatus", + "REFERENCE_SPEC_GIT_PATH", + "REFERENCE_SPEC_VERSION", + "RandomizationMode", + "Root", + "VersionedHash", + "Withdrawal", + "decode_bytes", + "deterministic_seed", + "encode_bytes", + "envelope_bytes", + "from_json", + "get_random_ssz_object", + "hash_tree_root", + "to_json", +) diff --git a/packages/testing/src/execution_testing/ssz/constants.py b/packages/testing/src/execution_testing/ssz/constants.py new file mode 100644 index 00000000000..6ca67d9c3c7 --- /dev/null +++ b/packages/testing/src/execution_testing/ssz/constants.py @@ -0,0 +1,74 @@ +"""SSZ size limits and byte-vector aliases for the REST+SSZ Engine API.""" + +from remerkleable.byte_arrays import ByteVector + +# Payload / envelope limits. +MAX_BYTES_PER_TX = 2**30 +MAX_TXS_PER_PAYLOAD = 2**20 +MAX_WITHDRAWALS_PER_PAYLOAD = 2**4 +BYTES_PER_LOGS_BLOOM = 256 +MAX_EXTRA_DATA_BYTES = 2**5 +MAX_BAL_BYTES = MAX_BYTES_PER_TX +MAX_EXECUTION_REQUESTS_PER_PAYLOAD = 2**8 +MAX_BYTES_PER_EXECUTION_REQUEST = MAX_BYTES_PER_TX + +# Blob / cell limits +MAX_BLOB_COMMITMENTS_PER_BLOCK = 2**12 +FIELD_ELEMENTS_PER_BLOB = 4096 +BYTES_PER_FIELD_ELEMENT = 32 +BYTES_PER_BLOB = FIELD_ELEMENTS_PER_BLOB * BYTES_PER_FIELD_ELEMENT +CELLS_PER_EXT_BLOB = 128 +FIELD_ELEMENTS_PER_CELL = 64 +BYTES_PER_CELL = FIELD_ELEMENTS_PER_CELL * BYTES_PER_FIELD_ELEMENT + +# Blob-pool / bodies. +MAX_VERSIONED_HASHES_PER_REQUEST = 128 +MAX_BLOBS_REQUEST = MAX_VERSIONED_HASHES_PER_REQUEST +MAX_BODIES_REQUEST = 2**5 + +# Status / error limits. +MAX_ERROR_BYTES = 1024 + +# Identity / capabilities limits. +MAX_CLIENT_CODE_LENGTH = 2 +MAX_CLIENT_NAME_LENGTH = 64 +MAX_CLIENT_VERSION_LENGTH = 64 +MAX_CLIENT_VERSIONS = 4 +MAX_CAPABILITY_NAME_LENGTH = 64 +MAX_CAPABILITIES = 64 + + +class Hash32(ByteVector[32]): + """A 32-byte hash (`Hash32`, `Root` and `Bytes32` share this layout).""" + + +class Bytes32(ByteVector[32]): + """A 32-byte fixed vector.""" + + +class Root(ByteVector[32]): + """A 32-byte merkle root.""" + + +class Address(ByteVector[20]): + """A 20-byte execution-layer address.""" + + +class Bloom(ByteVector[BYTES_PER_LOGS_BLOOM]): + """A 256-byte logs bloom filter.""" + + +class VersionedHash(ByteVector[32]): + """An EIP-4844 versioned blob hash.""" + + +class Bytes8(ByteVector[8]): + """An 8-byte value (e.g. `payload_id`).""" + + +class Bytes4(ByteVector[4]): + """A 4-byte value (e.g. a client commit hash).""" + + +class Bytes48(ByteVector[48]): + """A 48-byte value (KZG commitments and proofs).""" diff --git a/packages/testing/src/execution_testing/ssz/containers.py b/packages/testing/src/execution_testing/ssz/containers.py new file mode 100644 index 00000000000..45a77378279 --- /dev/null +++ b/packages/testing/src/execution_testing/ssz/containers.py @@ -0,0 +1,700 @@ +""" +SSZ container definitions for the REST+SSZ Engine API. +""" + +from typing import Dict, Type + +from remerkleable.basic import boolean, uint8, uint64, uint256 +from remerkleable.bitfields import Bitvector +from remerkleable.byte_arrays import ByteList, ByteVector +from remerkleable.complex import Container, List + +from .constants import ( + BYTES_PER_BLOB, + BYTES_PER_CELL, + CELLS_PER_EXT_BLOB, + MAX_BAL_BYTES, + MAX_BLOB_COMMITMENTS_PER_BLOCK, + MAX_BLOBS_REQUEST, + MAX_BODIES_REQUEST, + MAX_BYTES_PER_EXECUTION_REQUEST, + MAX_BYTES_PER_TX, + MAX_CAPABILITIES, + MAX_CAPABILITY_NAME_LENGTH, + MAX_CLIENT_CODE_LENGTH, + MAX_CLIENT_NAME_LENGTH, + MAX_CLIENT_VERSION_LENGTH, + MAX_CLIENT_VERSIONS, + MAX_ERROR_BYTES, + MAX_EXECUTION_REQUESTS_PER_PAYLOAD, + MAX_EXTRA_DATA_BYTES, + MAX_TXS_PER_PAYLOAD, + MAX_WITHDRAWALS_PER_PAYLOAD, + Address, + Bloom, + Bytes4, + Bytes8, + Bytes32, + Bytes48, + Hash32, + Root, + VersionedHash, +) + + +class Withdrawal(Container): + """A validator withdrawal pushed into an execution payload.""" + + index: uint64 + validator_index: uint64 + address: Address + amount: uint64 + + +class ExecutionPayloadParis(Container): + """ + The Paris execution payload. + """ + + parent_hash: Hash32 + fee_recipient: Address + state_root: Hash32 + receipts_root: Hash32 + logs_bloom: Bloom + prev_randao: Bytes32 + block_number: uint64 + gas_limit: uint64 + gas_used: uint64 + timestamp: uint64 + extra_data: ByteList[MAX_EXTRA_DATA_BYTES] + base_fee_per_gas: uint256 + block_hash: Hash32 + transactions: List[ByteList[MAX_BYTES_PER_TX], MAX_TXS_PER_PAYLOAD] + + +class ExecutionPayloadShanghai(Container): + """ + The Shanghai execution payload. + """ + + parent_hash: Hash32 + fee_recipient: Address + state_root: Hash32 + receipts_root: Hash32 + logs_bloom: Bloom + prev_randao: Bytes32 + block_number: uint64 + gas_limit: uint64 + gas_used: uint64 + timestamp: uint64 + extra_data: ByteList[MAX_EXTRA_DATA_BYTES] + base_fee_per_gas: uint256 + block_hash: Hash32 + transactions: List[ByteList[MAX_BYTES_PER_TX], MAX_TXS_PER_PAYLOAD] + withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] + + +class ExecutionPayloadCancun(Container): + """ + The Cancun execution payload. + """ + + parent_hash: Hash32 + fee_recipient: Address + state_root: Hash32 + receipts_root: Hash32 + logs_bloom: Bloom + prev_randao: Bytes32 + block_number: uint64 + gas_limit: uint64 + gas_used: uint64 + timestamp: uint64 + extra_data: ByteList[MAX_EXTRA_DATA_BYTES] + base_fee_per_gas: uint256 + block_hash: Hash32 + transactions: List[ByteList[MAX_BYTES_PER_TX], MAX_TXS_PER_PAYLOAD] + withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] + blob_gas_used: uint64 + excess_blob_gas: uint64 + + +class ExecutionPayloadPrague(Container): + """ + The Prague execution payload. + """ + + parent_hash: Hash32 + fee_recipient: Address + state_root: Hash32 + receipts_root: Hash32 + logs_bloom: Bloom + prev_randao: Bytes32 + block_number: uint64 + gas_limit: uint64 + gas_used: uint64 + timestamp: uint64 + extra_data: ByteList[MAX_EXTRA_DATA_BYTES] + base_fee_per_gas: uint256 + block_hash: Hash32 + transactions: List[ByteList[MAX_BYTES_PER_TX], MAX_TXS_PER_PAYLOAD] + withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] + blob_gas_used: uint64 + excess_blob_gas: uint64 + + +class ExecutionPayloadOsaka(Container): + """ + The Osaka execution payload. + """ + + parent_hash: Hash32 + fee_recipient: Address + state_root: Hash32 + receipts_root: Hash32 + logs_bloom: Bloom + prev_randao: Bytes32 + block_number: uint64 + gas_limit: uint64 + gas_used: uint64 + timestamp: uint64 + extra_data: ByteList[MAX_EXTRA_DATA_BYTES] + base_fee_per_gas: uint256 + block_hash: Hash32 + transactions: List[ByteList[MAX_BYTES_PER_TX], MAX_TXS_PER_PAYLOAD] + withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] + blob_gas_used: uint64 + excess_blob_gas: uint64 + + +class ExecutionPayloadAmsterdam(Container): + """ + The Amsterdam execution payload. + """ + + parent_hash: Hash32 + fee_recipient: Address + state_root: Hash32 + receipts_root: Hash32 + logs_bloom: Bloom + prev_randao: Bytes32 + block_number: uint64 + gas_limit: uint64 + gas_used: uint64 + timestamp: uint64 + extra_data: ByteList[MAX_EXTRA_DATA_BYTES] + base_fee_per_gas: uint256 + block_hash: Hash32 + transactions: List[ByteList[MAX_BYTES_PER_TX], MAX_TXS_PER_PAYLOAD] + withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] + blob_gas_used: uint64 + excess_blob_gas: uint64 + block_access_list: ByteList[MAX_BAL_BYTES] + slot_number: uint64 + + +EXECUTION_PAYLOAD_BY_FORK: Dict[str, Type[Container]] = { + "Paris": ExecutionPayloadParis, + "Shanghai": ExecutionPayloadShanghai, + "Cancun": ExecutionPayloadCancun, + "Prague": ExecutionPayloadPrague, + "Osaka": ExecutionPayloadOsaka, + "Amsterdam": ExecutionPayloadAmsterdam, +} + + +class ExecutionPayloadEnvelopeParis(Container): + """ + The Paris `newPayload` envelope (consensus-specs Bellatrix). + """ + + payload: ExecutionPayloadParis + + +class ExecutionPayloadEnvelopeShanghai(Container): + """ + The Shanghai `newPayload` envelope (consensus-specs Capella). + """ + + payload: ExecutionPayloadShanghai + + +class ExecutionPayloadEnvelopeCancun(Container): + """ + The Cancun `newPayload` envelope. + """ + + payload: ExecutionPayloadCancun + parent_beacon_block_root: Root + + +class ExecutionPayloadEnvelopePrague(Container): + """ + The Prague `newPayload` envelope. + """ + + payload: ExecutionPayloadPrague + parent_beacon_block_root: Root + execution_requests: List[ + ByteList[MAX_BYTES_PER_EXECUTION_REQUEST], + MAX_EXECUTION_REQUESTS_PER_PAYLOAD, + ] + + +class ExecutionPayloadEnvelopeOsaka(Container): + """ + The Osaka `newPayload` envelope. + """ + + payload: ExecutionPayloadOsaka + parent_beacon_block_root: Root + execution_requests: List[ + ByteList[MAX_BYTES_PER_EXECUTION_REQUEST], + MAX_EXECUTION_REQUESTS_PER_PAYLOAD, + ] + + +class ExecutionPayloadEnvelopeAmsterdam(Container): + """ + The Amsterdam `newPayload` envelope from PR #793. + """ + + payload: ExecutionPayloadAmsterdam + parent_beacon_block_root: Root + execution_requests: List[ + ByteList[MAX_BYTES_PER_EXECUTION_REQUEST], + MAX_EXECUTION_REQUESTS_PER_PAYLOAD, + ] + + +EXECUTION_PAYLOAD_ENVELOPE_BY_FORK: Dict[str, Type[Container]] = { + "Paris": ExecutionPayloadEnvelopeParis, + "Shanghai": ExecutionPayloadEnvelopeShanghai, + "Cancun": ExecutionPayloadEnvelopeCancun, + "Prague": ExecutionPayloadEnvelopePrague, + "Osaka": ExecutionPayloadEnvelopeOsaka, + "Amsterdam": ExecutionPayloadEnvelopeAmsterdam, +} + + +class PayloadAttributesParis(Container): + """The Paris payload attributes.""" + + timestamp: uint64 + prev_randao: Bytes32 + suggested_fee_recipient: Address + + +class PayloadAttributesShanghai(Container): + """The Shanghai payload attributes (adds withdrawals).""" + + timestamp: uint64 + prev_randao: Bytes32 + suggested_fee_recipient: Address + withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] + + +class PayloadAttributesCancun(Container): + """The Cancun payload attributes (adds parent_beacon_block_root).""" + + timestamp: uint64 + prev_randao: Bytes32 + suggested_fee_recipient: Address + withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] + parent_beacon_block_root: Root + + +class PayloadAttributesPrague(Container): + """The Prague payload attributes (unchanged from Cancun).""" + + timestamp: uint64 + prev_randao: Bytes32 + suggested_fee_recipient: Address + withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] + parent_beacon_block_root: Root + + +class PayloadAttributesOsaka(Container): + """The Osaka payload attributes (unchanged from Cancun).""" + + timestamp: uint64 + prev_randao: Bytes32 + suggested_fee_recipient: Address + withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] + parent_beacon_block_root: Root + + +class PayloadAttributesAmsterdam(Container): + """The Amsterdam payload attributes (slot_number, target_gas_limit).""" + + timestamp: uint64 + prev_randao: Bytes32 + suggested_fee_recipient: Address + withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] + parent_beacon_block_root: Root + slot_number: uint64 + target_gas_limit: uint64 + + +PAYLOAD_ATTRIBUTES_BY_FORK: Dict[str, Type[Container]] = { + "Paris": PayloadAttributesParis, + "Shanghai": PayloadAttributesShanghai, + "Cancun": PayloadAttributesCancun, + "Prague": PayloadAttributesPrague, + "Osaka": PayloadAttributesOsaka, + "Amsterdam": PayloadAttributesAmsterdam, +} + + +class ForkchoiceState(Container): + """The forkchoice head/safe/finalized triple (fork-independent).""" + + head_block_hash: Hash32 + safe_block_hash: Hash32 + finalized_block_hash: Hash32 + + +class PayloadStatus(Container): + """ + A newPayload/forkchoice status. + + ``status`` is a uint8 enum (0=VALID, 1=INVALID, 2=SYNCING, 3=ACCEPTED). + ``latest_valid_hash`` is ``Optional[Hash32]`` and ``validation_error`` is + ``Optional[String]``. + """ + + status: uint8 + latest_valid_hash: List[Hash32, 1] + validation_error: List[ByteList[MAX_ERROR_BYTES], 1] + + +class ForkchoiceUpdateParis(Container): + """The Paris forkchoice update.""" + + forkchoice_state: ForkchoiceState + payload_attributes: List[PayloadAttributesParis, 1] + + +class ForkchoiceUpdateShanghai(Container): + """The Shanghai forkchoice update.""" + + forkchoice_state: ForkchoiceState + payload_attributes: List[PayloadAttributesShanghai, 1] + + +class ForkchoiceUpdateCancun(Container): + """The Cancun forkchoice update.""" + + forkchoice_state: ForkchoiceState + payload_attributes: List[PayloadAttributesCancun, 1] + + +class ForkchoiceUpdatePrague(Container): + """The Prague forkchoice update.""" + + forkchoice_state: ForkchoiceState + payload_attributes: List[PayloadAttributesPrague, 1] + + +class ForkchoiceUpdateOsaka(Container): + """The Osaka forkchoice update.""" + + forkchoice_state: ForkchoiceState + payload_attributes: List[PayloadAttributesOsaka, 1] + + +class ForkchoiceUpdateAmsterdam(Container): + """The Amsterdam forkchoice update.""" + + forkchoice_state: ForkchoiceState + payload_attributes: List[PayloadAttributesAmsterdam, 1] + custody_columns: List[Bitvector[CELLS_PER_EXT_BLOB], 1] + + +FORKCHOICE_UPDATE_BY_FORK: Dict[str, Type[Container]] = { + "Paris": ForkchoiceUpdateParis, + "Shanghai": ForkchoiceUpdateShanghai, + "Cancun": ForkchoiceUpdateCancun, + "Prague": ForkchoiceUpdatePrague, + "Osaka": ForkchoiceUpdateOsaka, + "Amsterdam": ForkchoiceUpdateAmsterdam, +} + + +class ForkchoiceUpdateResponse(Container): + """The forkchoice response: a status and an optional payload id.""" + + payload_status: PayloadStatus + payload_id: List[Bytes8, 1] + + +class ExecutionPayloadBodyParis(Container): + """The Paris execution payload body.""" + + transactions: List[ByteList[MAX_BYTES_PER_TX], MAX_TXS_PER_PAYLOAD] + + +class ExecutionPayloadBodyShanghai(Container): + """The Shanghai execution payload body.""" + + transactions: List[ByteList[MAX_BYTES_PER_TX], MAX_TXS_PER_PAYLOAD] + withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] + + +class ExecutionPayloadBodyCancun(Container): + """The Cancun execution payload body (unchanged from Shanghai).""" + + transactions: List[ByteList[MAX_BYTES_PER_TX], MAX_TXS_PER_PAYLOAD] + withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] + + +class ExecutionPayloadBodyPrague(Container): + """The Prague execution payload body (unchanged from Shanghai).""" + + transactions: List[ByteList[MAX_BYTES_PER_TX], MAX_TXS_PER_PAYLOAD] + withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] + + +class ExecutionPayloadBodyOsaka(Container): + """The Osaka execution payload body (unchanged from Shanghai).""" + + transactions: List[ByteList[MAX_BYTES_PER_TX], MAX_TXS_PER_PAYLOAD] + withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] + + +class ExecutionPayloadBodyAmsterdam(Container): + """The Amsterdam execution payload body (adds block_access_list).""" + + transactions: List[ByteList[MAX_BYTES_PER_TX], MAX_TXS_PER_PAYLOAD] + withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] + block_access_list: ByteList[MAX_BAL_BYTES] + + +EXECUTION_PAYLOAD_BODY_BY_FORK: Dict[str, Type[Container]] = { + "Paris": ExecutionPayloadBodyParis, + "Shanghai": ExecutionPayloadBodyShanghai, + "Cancun": ExecutionPayloadBodyCancun, + "Prague": ExecutionPayloadBodyPrague, + "Osaka": ExecutionPayloadBodyOsaka, + "Amsterdam": ExecutionPayloadBodyAmsterdam, +} + + +class BlobsBundleV1(Container): + """The Cancun blobs bundle.""" + + commitments: List[Bytes48, MAX_BLOB_COMMITMENTS_PER_BLOCK] + proofs: List[Bytes48, MAX_BLOB_COMMITMENTS_PER_BLOCK] + blobs: List[ByteVector[BYTES_PER_BLOB], MAX_BLOB_COMMITMENTS_PER_BLOCK] + + +class BlobsBundleV2(Container): + """The Osaka+ blobs bundle.""" + + commitments: List[Bytes48, MAX_BLOB_COMMITMENTS_PER_BLOCK] + proofs: List[Bytes48, MAX_BLOB_COMMITMENTS_PER_BLOCK * CELLS_PER_EXT_BLOB] + blobs: List[ByteVector[BYTES_PER_BLOB], MAX_BLOB_COMMITMENTS_PER_BLOCK] + + +class BuiltPayloadParis(Container): + """The Paris getPayload response.""" + + payload: ExecutionPayloadParis + block_value: uint256 + + +class BuiltPayloadShanghai(Container): + """The Shanghai getPayload response.""" + + payload: ExecutionPayloadShanghai + block_value: uint256 + should_override_builder: boolean + + +class BuiltPayloadCancun(Container): + """The Cancun getPayload response.""" + + payload: ExecutionPayloadCancun + block_value: uint256 + blobs_bundle: BlobsBundleV1 + should_override_builder: boolean + + +class BuiltPayloadPrague(Container): + """ + The Prague getPayload response. + """ + + payload: ExecutionPayloadPrague + block_value: uint256 + blobs_bundle: BlobsBundleV1 + execution_requests: List[ + ByteList[MAX_BYTES_PER_EXECUTION_REQUEST], + MAX_EXECUTION_REQUESTS_PER_PAYLOAD, + ] + should_override_builder: boolean + + +class BuiltPayloadOsaka(Container): + """The Osaka getPayload response (blobs_bundle is V2).""" + + payload: ExecutionPayloadOsaka + block_value: uint256 + blobs_bundle: BlobsBundleV2 + execution_requests: List[ + ByteList[MAX_BYTES_PER_EXECUTION_REQUEST], + MAX_EXECUTION_REQUESTS_PER_PAYLOAD, + ] + should_override_builder: boolean + + +class BuiltPayloadAmsterdam(Container): + """The Amsterdam getPayload response.""" + + payload: ExecutionPayloadAmsterdam + block_value: uint256 + blobs_bundle: BlobsBundleV2 + execution_requests: List[ + ByteList[MAX_BYTES_PER_EXECUTION_REQUEST], + MAX_EXECUTION_REQUESTS_PER_PAYLOAD, + ] + should_override_builder: boolean + + +BUILT_PAYLOAD_BY_FORK: Dict[str, Type[Container]] = { + "Paris": BuiltPayloadParis, + "Shanghai": BuiltPayloadShanghai, + "Cancun": BuiltPayloadCancun, + "Prague": BuiltPayloadPrague, + "Osaka": BuiltPayloadOsaka, + "Amsterdam": BuiltPayloadAmsterdam, +} + + +class BlobAndProofV1(Container): + """A whole blob with a single proof.""" + + blob: ByteVector[BYTES_PER_BLOB] + proof: Bytes48 + + +class BlobAndProofV2(Container): + """A whole blob with cell proofs.""" + + blob: ByteVector[BYTES_PER_BLOB] + proofs: List[Bytes48, CELLS_PER_EXT_BLOB] + + +class BlobCellsAndProofs(Container): + """ + Cell-range blob contents (/blobs/v4, Amsterdam). + Each cell and proof is ``Optional``. + """ + + blob_cells: List[List[ByteVector[BYTES_PER_CELL], 1], CELLS_PER_EXT_BLOB] + proofs: List[List[Bytes48, 1], CELLS_PER_EXT_BLOB] + + +class BodiesByHashRequest(Container): + """The /bodies/hash request body.""" + + block_hashes: List[Hash32, MAX_BODIES_REQUEST] + + +class BodyEntry(Container): + """ + One bodies-response entry. + """ + + available: boolean + body: ExecutionPayloadBodyAmsterdam + + +class BodiesResponse(Container): + """The /bodies response.""" + + entries: List[BodyEntry, MAX_BODIES_REQUEST] + + +class BlobsV1Request(Container): + """The /blobs/v1 request.""" + + versioned_hashes: List[VersionedHash, MAX_BLOBS_REQUEST] + + +class BlobV1Entry(Container): + """One /blobs/v1 response entry.""" + + available: boolean + contents: BlobAndProofV1 + + +class BlobsV1Response(Container): + """The /blobs/v1 response.""" + + entries: List[BlobV1Entry, MAX_BLOBS_REQUEST] + + +class BlobsV2Request(Container): + """The /blobs/v2 request (same shape as v1).""" + + versioned_hashes: List[VersionedHash, MAX_BLOBS_REQUEST] + + +class BlobV2Entry(Container): + """One /blobs/v2 response entry (reused verbatim by /v3).""" + + available: boolean + contents: BlobAndProofV2 + + +class BlobsV2Response(Container): + """The /blobs/v2 response (all-or-nothing).""" + + entries: List[BlobV2Entry, MAX_BLOBS_REQUEST] + + +class BlobsV3Response(Container): + """The /blobs/v3 response (reuses BlobV2Entry; only semantics differ).""" + + entries: List[BlobV2Entry, MAX_BLOBS_REQUEST] + + +class BlobsV4Request(Container): + """The /blobs/v4 request (adds a cell-selection bitarray).""" + + versioned_hashes: List[VersionedHash, MAX_BLOBS_REQUEST] + indices_bitarray: Bitvector[CELLS_PER_EXT_BLOB] + + +class BlobV4Entry(Container): + """One /blobs/v4 response entry.""" + + available: boolean + contents: BlobCellsAndProofs + + +class BlobsV4Response(Container): + """The /blobs/v4 response.""" + + entries: List[BlobV4Entry, MAX_BLOBS_REQUEST] + + +class ClientVersion(Container): + """A client identity entry (JSON on the wire; SSZ shape).""" + + code: ByteList[MAX_CLIENT_CODE_LENGTH] + name: ByteList[MAX_CLIENT_NAME_LENGTH] + version: ByteList[MAX_CLIENT_VERSION_LENGTH] + commit: Bytes4 + + +class IdentityResponse(Container): + """The /identity response.""" + + versions: List[ClientVersion, MAX_CLIENT_VERSIONS] + + +class CapabilitiesResponse(Container): + """The /capabilities response (capability-name list).""" + + capabilities: List[ByteList[MAX_CAPABILITY_NAME_LENGTH], MAX_CAPABILITIES] diff --git a/packages/testing/src/execution_testing/ssz/json_mapping.py b/packages/testing/src/execution_testing/ssz/json_mapping.py new file mode 100644 index 00000000000..1df7ef7d5ee --- /dev/null +++ b/packages/testing/src/execution_testing/ssz/json_mapping.py @@ -0,0 +1,56 @@ +"""Consensus-spec canonical JSON mapping for SSZ values.""" + +from typing import Any + +from remerkleable.basic import boolean, uint +from remerkleable.byte_arrays import ByteList, ByteVector +from remerkleable.complex import Container, List + + +def to_json(value: Any) -> Any: + """ + Convert an SSZ value to its canonical JSON representation. + """ + if isinstance(value, Container): + return { + field: to_json(getattr(value, field)) + for field in type(value).fields() + } + if isinstance(value, List): + return [to_json(element) for element in value] + if isinstance(value, (ByteVector, ByteList)): + return "0x" + bytes(value).hex() + if isinstance(value, boolean): + return bool(value) + if isinstance(value, uint): + return str(int(value)) + raise TypeError(f"unsupported SSZ value type: {type(value)!r}") + + +def from_json(ssz_type: Any, obj: Any) -> Any: + """ + Parse consensus-spec canonical JSON into an SSZ value of ``ssz_type``. + """ + if issubclass(ssz_type, Container): + fields = ssz_type.fields() + return ssz_type( + **{ + field: from_json(field_type, obj[field]) + for field, field_type in fields.items() + } + ) + if issubclass(ssz_type, List): + element_type = ssz_type.element_cls() + return ssz_type(*(from_json(element_type, item) for item in obj)) + if issubclass(ssz_type, (ByteVector, ByteList)): + return ssz_type(bytes.fromhex(_strip_0x(obj))) + if issubclass(ssz_type, boolean): + return ssz_type(bool(obj)) + if issubclass(ssz_type, uint): + return ssz_type(int(obj)) + raise TypeError(f"unsupported SSZ type: {ssz_type!r}") + + +def _strip_0x(value: str) -> str: + """Return ``value`` without a leading ``0x`` prefix.""" + return value[2:] if value.startswith("0x") else value diff --git a/packages/testing/src/execution_testing/ssz/random_value.py b/packages/testing/src/execution_testing/ssz/random_value.py new file mode 100644 index 00000000000..2e8ed737e28 --- /dev/null +++ b/packages/testing/src/execution_testing/ssz/random_value.py @@ -0,0 +1,183 @@ +""" +Deterministic random SSZ value generation for static test vectors. +""" + +import hashlib +from enum import Enum +from random import Random +from typing import Any + +from remerkleable.basic import boolean, uint +from remerkleable.byte_arrays import ByteList, ByteVector +from remerkleable.complex import Container, List +from remerkleable.core import View + +_MODE_NAMES = ( + "random", + "zero", + "max", + "nil_count", + "one_count", + "max_count", +) + + +class RandomizationMode(Enum): + """ + How a value's scalar and collection fields are filled. + Mirrors the consensus-specs ``RandomizationMode``. + """ + + mode_random = 0 + mode_zero = 1 + mode_max = 2 + mode_nil_count = 3 + mode_one_count = 4 + mode_max_count = 5 + + def is_changing(self) -> bool: + """ + Return whether the mode yields varying values across cases. + + True for ``random``, ``one_count`` and ``max_count`` -- those randomize + content, so several cases are worth generating; the rest are fully + determined by a single case. + """ + return self in ( + RandomizationMode.mode_random, + RandomizationMode.mode_one_count, + RandomizationMode.mode_max_count, + ) + + def to_name(self) -> str: + """Return the canonical short name for this mode.""" + return _MODE_NAMES[self.value] + + +def deterministic_seed(*parts: object) -> int: + """ + Return a stable integer seed derived from ``parts``. + Uses SHA-256 over the slash-joined string parts. + """ + joined = "/".join(str(part) for part in parts) + digest = hashlib.sha256(joined.encode("utf-8")).digest() + return int.from_bytes(digest, "big") + + +def get_random_ssz_object( + rng: Random, + typ: Any, + max_bytes_length: int, + max_list_length: int, + mode: RandomizationMode, + chaos: bool, +) -> View: + """ + Build a value of ``typ`` filled with random data per ``mode``. + """ + if chaos: + mode = rng.choice(list(RandomizationMode)) + if issubclass(typ, ByteList): + if mode == RandomizationMode.mode_nil_count: + return typ(b"") + elif mode == RandomizationMode.mode_max_count: + return typ(_random_bytes(rng, min(max_bytes_length, typ.limit()))) + elif mode == RandomizationMode.mode_one_count: + return typ(_random_bytes(rng, min(1, typ.limit()))) + elif mode == RandomizationMode.mode_zero: + return typ(b"\x00" * min(1, typ.limit())) + elif mode == RandomizationMode.mode_max: + return typ(b"\xff" * min(1, typ.limit())) + else: + return typ( + _random_bytes( + rng, rng.randint(0, min(max_bytes_length, typ.limit())) + ) + ) + if issubclass(typ, ByteVector): + # Byte vectors are fixed length; no max-bytes cap applies. + if mode == RandomizationMode.mode_zero: + return typ(b"\x00" * typ.type_byte_length()) + elif mode == RandomizationMode.mode_max: + return typ(b"\xff" * typ.type_byte_length()) + else: + return typ(_random_bytes(rng, typ.type_byte_length())) + elif issubclass(typ, (boolean, uint)): + if mode == RandomizationMode.mode_zero: + return _min_basic_value(typ) + elif mode == RandomizationMode.mode_max: + return _max_basic_value(typ) + else: + return _random_basic_value(rng, typ) + elif issubclass(typ, List): + limit = max_list_length + if typ.limit() < limit: + limit = typ.limit() + length = rng.randint(0, limit) + if mode == RandomizationMode.mode_one_count: + length = 1 + elif mode == RandomizationMode.mode_max_count: + length = limit + elif mode == RandomizationMode.mode_nil_count: + length = 0 + element_type = typ.element_cls() + max_list_length = 1 << (max_list_length.bit_length() >> 1) + return typ( + get_random_ssz_object( + rng, + element_type, + max_bytes_length, + max_list_length, + mode, + chaos, + ) + for _ in range(length) + ) + elif issubclass(typ, Container): + return typ( + **{ + field_name: get_random_ssz_object( + rng, + field_type, + max_bytes_length, + max_list_length, + mode, + chaos, + ) + for field_name, field_type in typ.fields().items() + } + ) + else: + raise TypeError(f"unsupported SSZ type: {typ!r}") + + +def _random_bytes(rng: Random, length: int) -> bytes: + """Return ``length`` random bytes.""" + return bytes(rng.getrandbits(8) for _ in range(length)) + + +def _random_basic_value(rng: Random, typ: Any) -> View: + """Return a random ``boolean`` or ``uint`` value.""" + if issubclass(typ, boolean): + return typ(rng.choice((True, False))) + if issubclass(typ, uint): + return typ(rng.randint(0, 2 ** (typ.type_byte_length() * 8) - 1)) + raise TypeError(f"not a basic type: {typ!r}") + + +def _min_basic_value(typ: Any) -> View: + """Return the minimum (zero / False) value of a basic type.""" + if issubclass(typ, boolean): + return typ(False) + if issubclass(typ, uint): + return typ(0) + raise TypeError(f"not a basic type: {typ!r}") + + +def _max_basic_value(typ: Any) -> View: + """Return the maximum (all-ones / True) value of a basic type.""" + if issubclass(typ, boolean): + return typ(True) + if issubclass(typ, uint): + return typ(2 ** (typ.type_byte_length() * 8) - 1) + raise TypeError(f"not a basic type: {typ!r}") diff --git a/packages/testing/src/execution_testing/ssz/tests/__init__.py b/packages/testing/src/execution_testing/ssz/tests/__init__.py new file mode 100644 index 00000000000..14f702a167d --- /dev/null +++ b/packages/testing/src/execution_testing/ssz/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for the `execution_testing.ssz` package.""" diff --git a/packages/testing/src/execution_testing/ssz/tests/test_containers.py b/packages/testing/src/execution_testing/ssz/tests/test_containers.py new file mode 100644 index 00000000000..4a809ec16c9 --- /dev/null +++ b/packages/testing/src/execution_testing/ssz/tests/test_containers.py @@ -0,0 +1,260 @@ +"""Round-trip and structural tests for the SSZ containers.""" + +from .. import decode_bytes, encode_bytes, hash_tree_root +from ..containers import ( + EXECUTION_PAYLOAD_BY_FORK, + EXECUTION_PAYLOAD_ENVELOPE_BY_FORK, + ExecutionPayloadAmsterdam, + ExecutionPayloadEnvelopeAmsterdam, + Withdrawal, +) +from ..json_mapping import from_json, to_json + +# A spread of transaction lengths, on purpose: a two-level `List[ByteList, N]` +# only exercises its inner offset table when the elements differ in length. +TRANSACTIONS = [ + bytes.fromhex("02f86b01"), + bytes.fromhex("01" * 21), + bytes.fromhex("03" * 5), +] + + +def _withdrawal() -> Withdrawal: + return Withdrawal( + index=7, + validator_index=42, + address=bytes.fromhex("11" * 20), + amount=32_000_000_000, + ) + + +def _payload() -> ExecutionPayloadAmsterdam: + return ExecutionPayloadAmsterdam( + parent_hash=bytes.fromhex("aa" * 32), + fee_recipient=bytes.fromhex("bb" * 20), + state_root=bytes.fromhex("cc" * 32), + receipts_root=bytes.fromhex("dd" * 32), + logs_bloom=bytes.fromhex("00" * 256), + prev_randao=bytes.fromhex("ee" * 32), + block_number=21_000_000, + gas_limit=30_000_000, + gas_used=21_000, + timestamp=1_700_000_000, + extra_data=bytes.fromhex("dead"), + base_fee_per_gas=10**18, + block_hash=bytes.fromhex("ff" * 32), + transactions=list(TRANSACTIONS), + withdrawals=[_withdrawal()], + blob_gas_used=131_072, + excess_blob_gas=0, + block_access_list=bytes.fromhex("c0de"), + slot_number=9_999, + ) + + +def _envelope() -> ExecutionPayloadEnvelopeAmsterdam: + return ExecutionPayloadEnvelopeAmsterdam( + payload=_payload(), + parent_beacon_block_root=bytes.fromhex("12" * 32), + execution_requests=[bytes.fromhex("00aa"), bytes.fromhex("01bbcc")], + ) + + +def test_withdrawal_round_trip() -> None: + """A withdrawal survives encode -> decode unchanged.""" + value = _withdrawal() + raw = encode_bytes(value) + assert decode_bytes(Withdrawal, raw) == value + + +def test_payload_round_trip() -> None: + """An execution payload survives encode -> decode unchanged.""" + value = _payload() + raw = encode_bytes(value) + assert decode_bytes(ExecutionPayloadAmsterdam, raw) == value + + +def test_envelope_round_trip() -> None: + """An envelope survives encode -> decode unchanged.""" + value = _envelope() + raw = encode_bytes(value) + assert decode_bytes(ExecutionPayloadEnvelopeAmsterdam, raw) == value + + +def test_hash_tree_root_is_32_bytes_and_deterministic() -> None: + """`hash_tree_root` returns a stable 32-byte digest.""" + root = hash_tree_root(_envelope()) + assert isinstance(root, bytes) + assert len(root) == 32 + assert root == hash_tree_root(_envelope()) + + +def test_transactions_two_level_offsets() -> None: + """ + Transactions of differing lengths round-trip exactly, and reordering them + changes the root -- the inner offset table is order- and length-sensitive. + """ + value = _payload() + decoded = decode_bytes(ExecutionPayloadAmsterdam, encode_bytes(value)) + assert [bytes(tx) for tx in decoded.transactions] == TRANSACTIONS + + reordered = _payload() + reordered.transactions = list(reversed(TRANSACTIONS)) + assert hash_tree_root(reordered) != hash_tree_root(value) + + +def test_execution_payload_field_order() -> None: + """ + Guard the load-bearing field order against silent reordering. + + SSZ is order-sensitive: a wrong order produces a wrong but + structurally-valid root. + """ + assert list(ExecutionPayloadAmsterdam.fields().keys()) == [ + "parent_hash", + "fee_recipient", + "state_root", + "receipts_root", + "logs_bloom", + "prev_randao", + "block_number", + "gas_limit", + "gas_used", + "timestamp", + "extra_data", + "base_fee_per_gas", + "block_hash", + "transactions", + "withdrawals", + "blob_gas_used", + "excess_blob_gas", + "block_access_list", + "slot_number", + ] + + +def test_execution_payload_registry_covers_forks_in_order() -> None: + """The registry enumerates every payload fork, oldest-first.""" + assert list(EXECUTION_PAYLOAD_BY_FORK) == [ + "Paris", + "Shanghai", + "Cancun", + "Prague", + "Osaka", + "Amsterdam", + ] + + +def test_per_fork_field_deltas_match_consensus_specs() -> None: + """ + Each fork's payload is its predecessor's fields plus exactly the fields the + corresponding consensus fork added. + + This encodes the consensus-specs evolution as per-fork deltas (Paris is the + Bellatrix base; Capella adds withdrawals; Deneb adds the blob-gas fields; + Electra and Fulu add nothing; Amsterdam adds the two EL fields). SSZ is + order-sensitive, so it guards both the additions and that nothing earlier + was reordered or dropped. + """ + expected = [ + "parent_hash", + "fee_recipient", + "state_root", + "receipts_root", + "logs_bloom", + "prev_randao", + "block_number", + "gas_limit", + "gas_used", + "timestamp", + "extra_data", + "base_fee_per_gas", + "block_hash", + "transactions", + ] + additions = { + "Paris": [], + "Shanghai": ["withdrawals"], + "Cancun": ["blob_gas_used", "excess_blob_gas"], + "Prague": [], + "Osaka": [], + "Amsterdam": ["block_access_list", "slot_number"], + } + for fork, cls in EXECUTION_PAYLOAD_BY_FORK.items(): + expected = expected + additions[fork] + assert list(cls.fields().keys()) == expected, fork + + +def test_every_fork_payload_round_trips() -> None: + """Every modelled payload survives a zero-value encode -> decode.""" + for fork, cls in EXECUTION_PAYLOAD_BY_FORK.items(): + value = cls() + assert decode_bytes(cls, encode_bytes(value)) == value, fork + + +def test_envelope_field_order() -> None: + """Guard the envelope field order from PR #793.""" + assert list(ExecutionPayloadEnvelopeAmsterdam.fields().keys()) == [ + "payload", + "parent_beacon_block_root", + "execution_requests", + ] + + +def test_json_round_trip_preserves_value() -> None: + """to_json -> from_json reconstructs an equal envelope.""" + value = _envelope() + assert ( + from_json(ExecutionPayloadEnvelopeAmsterdam, to_json(value)) == value + ) + + +def test_envelope_registry_covers_forks_in_order() -> None: + """The envelope registry spans every payload fork, oldest-first.""" + assert list(EXECUTION_PAYLOAD_ENVELOPE_BY_FORK) == [ + "Paris", + "Shanghai", + "Cancun", + "Prague", + "Osaka", + "Amsterdam", + ] + + +def test_per_fork_envelope_field_deltas() -> None: + """ + Each envelope is its predecessor's fields plus exactly the arguments the + corresponding ``newPayloadVX`` gained. + + Paris/Shanghai wrap only the payload (V1/V2); Cancun introduces + ``parent_beacon_block_root``; Prague adds ``execution_requests``; the field + set is then stable through Amsterdam (later forks differ only by the inner + payload type). ``expectedBlobVersionedHashes`` is omitted throughout, + matching PR #793. + """ + expected = ["payload"] + additions = { + "Paris": [], + "Shanghai": [], + "Cancun": ["parent_beacon_block_root"], + "Prague": ["execution_requests"], + "Osaka": [], + "Amsterdam": [], + } + for fork, cls in EXECUTION_PAYLOAD_ENVELOPE_BY_FORK.items(): + expected = expected + additions[fork] + assert list(cls.fields().keys()) == expected, fork + + +def test_every_fork_envelope_round_trips() -> None: + """Every modelled envelope survives a zero-value encode -> decode.""" + for fork, cls in EXECUTION_PAYLOAD_ENVELOPE_BY_FORK.items(): + value = cls() + assert decode_bytes(cls, encode_bytes(value)) == value, fork + + +def test_per_fork_envelope_wraps_matching_payload() -> None: + """Each envelope's ``payload`` field is its own fork's payload class.""" + for fork, cls in EXECUTION_PAYLOAD_ENVELOPE_BY_FORK.items(): + payload_type = dict(cls.fields())["payload"] + assert payload_type is EXECUTION_PAYLOAD_BY_FORK[fork], fork diff --git a/packages/testing/src/execution_testing/ssz/tests/test_engine_containers.py b/packages/testing/src/execution_testing/ssz/tests/test_engine_containers.py new file mode 100644 index 00000000000..a3b231ab1e0 --- /dev/null +++ b/packages/testing/src/execution_testing/ssz/tests/test_engine_containers.py @@ -0,0 +1,194 @@ +""" +Structural tests for the broader Engine API SSZ containers from PR #793. + +Guards the load-bearing field order of every container beyond the core +payload/envelope set (those live in ``test_containers.py``), plus the subtle +``Optional``/``String`` encoding in ``PayloadStatus``. +""" + +from .. import decode_bytes, encode_bytes +from ..containers import ( + BUILT_PAYLOAD_BY_FORK, + EXECUTION_PAYLOAD_BODY_BY_FORK, + FORKCHOICE_UPDATE_BY_FORK, + PAYLOAD_ATTRIBUTES_BY_FORK, + BlobAndProofV1, + BlobAndProofV2, + BlobCellsAndProofs, + BlobsBundleV1, + BlobsBundleV2, + BlobsV1Request, + BlobsV1Response, + BlobsV4Request, + BlobV1Entry, + BlobV4Entry, + BodiesByHashRequest, + BodiesResponse, + BodyEntry, + CapabilitiesResponse, + ClientVersion, + ForkchoiceState, + ForkchoiceUpdateResponse, + IdentityResponse, + PayloadStatus, +) + +_FORKS = ["Paris", "Shanghai", "Cancun", "Prague", "Osaka", "Amsterdam"] + + +def _fields(cls: object) -> list: + return list(cls.fields().keys()) # type: ignore[attr-defined] + + +def test_new_registries_cover_forks_in_order() -> None: + """Every per-fork registry spans Paris..Amsterdam, oldest-first.""" + for reg in ( + PAYLOAD_ATTRIBUTES_BY_FORK, + FORKCHOICE_UPDATE_BY_FORK, + EXECUTION_PAYLOAD_BODY_BY_FORK, + BUILT_PAYLOAD_BY_FORK, + ): + assert list(reg) == _FORKS + + +def test_payload_attributes_field_deltas() -> None: + """PayloadAttributes grows by the fields each fork added.""" + expected = ["timestamp", "prev_randao", "suggested_fee_recipient"] + additions = { + "Paris": [], + "Shanghai": ["withdrawals"], + "Cancun": ["parent_beacon_block_root"], + "Prague": [], + "Osaka": [], + "Amsterdam": ["slot_number", "target_gas_limit"], + } + for fork, cls in PAYLOAD_ATTRIBUTES_BY_FORK.items(): + expected = expected + additions[fork] + assert _fields(cls) == expected, fork + + +def test_execution_payload_body_field_deltas() -> None: + """ExecutionPayloadBody grows transactions -> withdrawals -> BAL.""" + expected = ["transactions"] + additions = { + "Paris": [], + "Shanghai": ["withdrawals"], + "Cancun": [], + "Prague": [], + "Osaka": [], + "Amsterdam": ["block_access_list"], + } + for fork, cls in EXECUTION_PAYLOAD_BODY_BY_FORK.items(): + expected = expected + additions[fork] + assert _fields(cls) == expected, fork + + +def test_built_payload_field_order() -> None: + """ + BuiltPayload field order per fork. + + Not a simple prefix growth: ``execution_requests`` is inserted *before* + ``should_override_builder`` (normative in #793). + """ + expected = { + "Paris": ["payload", "block_value"], + "Shanghai": ["payload", "block_value", "should_override_builder"], + "Cancun": [ + "payload", + "block_value", + "blobs_bundle", + "should_override_builder", + ], + "Prague": [ + "payload", + "block_value", + "blobs_bundle", + "execution_requests", + "should_override_builder", + ], + } + expected["Osaka"] = expected["Prague"] + expected["Amsterdam"] = expected["Prague"] + for fork, cls in BUILT_PAYLOAD_BY_FORK.items(): + assert _fields(cls) == expected[fork], fork + + +def test_forkchoice_update_field_order() -> None: + """ForkchoiceUpdate gains custody_columns only at Amsterdam.""" + for fork, cls in FORKCHOICE_UPDATE_BY_FORK.items(): + base = ["forkchoice_state", "payload_attributes"] + if fork == "Amsterdam": + base = base + ["custody_columns"] + assert _fields(cls) == base, fork + + +def test_single_container_field_orders() -> None: + """Exact field order for the fork-independent containers.""" + assert _fields(ForkchoiceState) == [ + "head_block_hash", + "safe_block_hash", + "finalized_block_hash", + ] + assert _fields(PayloadStatus) == [ + "status", + "latest_valid_hash", + "validation_error", + ] + assert _fields(ForkchoiceUpdateResponse) == [ + "payload_status", + "payload_id", + ] + assert _fields(BlobsBundleV1) == ["commitments", "proofs", "blobs"] + assert _fields(BlobsBundleV2) == ["commitments", "proofs", "blobs"] + assert _fields(BlobAndProofV1) == ["blob", "proof"] + assert _fields(BlobAndProofV2) == ["blob", "proofs"] + assert _fields(BlobCellsAndProofs) == ["blob_cells", "proofs"] + assert _fields(BodiesByHashRequest) == ["block_hashes"] + assert _fields(BodyEntry) == ["available", "body"] + assert _fields(BodiesResponse) == ["entries"] + assert _fields(BlobsV1Request) == ["versioned_hashes"] + assert _fields(BlobV1Entry) == ["available", "contents"] + assert _fields(BlobsV1Response) == ["entries"] + assert _fields(BlobsV4Request) == [ + "versioned_hashes", + "indices_bitarray", + ] + assert _fields(BlobV4Entry) == ["available", "contents"] + assert _fields(ClientVersion) == ["code", "name", "version", "commit"] + assert _fields(IdentityResponse) == ["versions"] + assert _fields(CapabilitiesResponse) == ["capabilities"] + + +def test_payload_status_optional_encoding_matches_spec_examples() -> None: + """ + The ``Optional``/``String`` wire shape matches PR #793's worked examples. + + Example A: VALID with a present ``latest_valid_hash`` and absent error is + 41 bytes. Example B: INVALID with a 14-byte error and absent hash is 27. + """ + valid = PayloadStatus( + status=0, latest_valid_hash=[b"\xaa" * 32], validation_error=[] + ) + assert len(encode_bytes(valid)) == 41 + assert decode_bytes(PayloadStatus, encode_bytes(valid)) == valid + + invalid = PayloadStatus( + status=1, + latest_valid_hash=[], + validation_error=[b"bad state root"], + ) + assert len(encode_bytes(invalid)) == 27 + assert decode_bytes(PayloadStatus, encode_bytes(invalid)) == invalid + + +def test_per_fork_engine_containers_round_trip() -> None: + """Zero-value encode -> decode for every per-fork engine container.""" + for reg in ( + PAYLOAD_ATTRIBUTES_BY_FORK, + FORKCHOICE_UPDATE_BY_FORK, + EXECUTION_PAYLOAD_BODY_BY_FORK, + BUILT_PAYLOAD_BY_FORK, + ): + for fork, cls in reg.items(): + value = cls() + assert decode_bytes(cls, encode_bytes(value)) == value, fork diff --git a/packages/testing/src/execution_testing/ssz/tests/test_json_mapping.py b/packages/testing/src/execution_testing/ssz/tests/test_json_mapping.py new file mode 100644 index 00000000000..93a782399cc --- /dev/null +++ b/packages/testing/src/execution_testing/ssz/tests/test_json_mapping.py @@ -0,0 +1,52 @@ +"""Tests for the consensus-spec canonical JSON mapping.""" + +from ..containers import ExecutionPayloadAmsterdam, Withdrawal +from ..json_mapping import from_json, to_json +from .test_containers import _payload, _withdrawal + + +def test_integers_serialize_as_decimal_strings() -> None: + """ + Unsigned integers become decimal strings, not hex. + + This is the consensus-spec convention and differs from eth JSON-RPC, where + quantities are hex. + """ + obj = to_json(_payload()) + assert obj["block_number"] == "21000000" + assert obj["gas_limit"] == "30000000" + # `base_fee_per_gas` is a uint256; it must also be a plain decimal string, + # not the little-endian hex that `remerkleable.to_obj()` would emit. + assert obj["base_fee_per_gas"] == str(10**18) + + +def test_byte_fields_serialize_as_lowercase_0x_hex() -> None: + """Byte vectors and lists become lowercase ``0x``-prefixed hex.""" + obj = to_json(_payload()) + assert obj["parent_hash"] == "0x" + "aa" * 32 + assert obj["fee_recipient"] == "0x" + "bb" * 20 + assert obj["extra_data"] == "0xdead" + assert obj["transactions"] == [ + "0x02f86b01", + "0x" + "01" * 21, + "0x" + "03" * 5, + ] + + +def test_field_names_are_snake_case() -> None: + """Field names match the container definitions verbatim (snake_case).""" + obj = to_json(_payload()) + assert "block_number" in obj + assert "blockNumber" not in obj + + +def test_withdrawal_json_round_trip() -> None: + """to_json -> from_json reconstructs an equal withdrawal.""" + value = _withdrawal() + assert from_json(Withdrawal, to_json(value)) == value + + +def test_payload_json_round_trip() -> None: + """to_json -> from_json reconstructs an equal payload.""" + value = _payload() + assert from_json(ExecutionPayloadAmsterdam, to_json(value)) == value diff --git a/packages/testing/src/execution_testing/ssz/tests/test_random_value.py b/packages/testing/src/execution_testing/ssz/tests/test_random_value.py new file mode 100644 index 00000000000..8a3e1c68739 --- /dev/null +++ b/packages/testing/src/execution_testing/ssz/tests/test_random_value.py @@ -0,0 +1,156 @@ +"""Property tests for the deterministic random SSZ value generator.""" + +from random import Random +from typing import Any + +import pytest + +from .. import decode_bytes, encode_bytes +from ..containers import ( + ExecutionPayloadAmsterdam, + ExecutionPayloadEnvelopeAmsterdam, + Withdrawal, +) +from ..random_value import ( + RandomizationMode, + deterministic_seed, + get_random_ssz_object, +) + +CONTAINERS = [ + Withdrawal, + ExecutionPayloadAmsterdam, + ExecutionPayloadEnvelopeAmsterdam, +] + +MAX_BYTES_LENGTH = 48 +MAX_LIST_LENGTH = 4 + + +def _value( + ssz_type: Any, + mode: RandomizationMode, + seed: int = 0, + chaos: bool = False, +) -> Any: + return get_random_ssz_object( + Random(seed), + ssz_type, + MAX_BYTES_LENGTH, + MAX_LIST_LENGTH, + mode, + chaos, + ) + + +def test_deterministic_seed_is_stable_and_not_pythons_hash() -> None: + """The seed depends only on its parts, not on process-salted hashing.""" + assert deterministic_seed("a", "b", 1) == deterministic_seed("a", "b", 1) + assert deterministic_seed("a", "b", 1) != deterministic_seed("a", "b", 2) + + +def test_is_changing_matches_upstream() -> None: + """``random``/``one_count``/``max_count`` change; the rest are fixed.""" + changing = { + RandomizationMode.mode_random, + RandomizationMode.mode_one_count, + RandomizationMode.mode_max_count, + } + for mode in RandomizationMode: + assert mode.is_changing() == (mode in changing), mode + + +@pytest.mark.parametrize("ssz_type", CONTAINERS) +@pytest.mark.parametrize("mode", list(RandomizationMode)) +def test_same_seed_yields_identical_value( + ssz_type: Any, mode: RandomizationMode +) -> None: + """A fixed seed reproduces the exact same value (the core guarantee).""" + assert encode_bytes(_value(ssz_type, mode, seed=1234)) == encode_bytes( + _value(ssz_type, mode, seed=1234) + ) + + +@pytest.mark.parametrize("ssz_type", CONTAINERS) +@pytest.mark.parametrize("mode", list(RandomizationMode)) +def test_generated_value_round_trips( + ssz_type: Any, mode: RandomizationMode +) -> None: + """Every generated value survives encode -> decode unchanged.""" + value = _value(ssz_type, mode, seed=7) + assert decode_bytes(ssz_type, encode_bytes(value)) == value + + +def test_zero_mode_zeroes_scalars_and_byte_vectors() -> None: + """``zero`` mode zeroes scalars and byte vectors.""" + payload = _value(ExecutionPayloadAmsterdam, RandomizationMode.mode_zero) + assert int(payload.block_number) == 0 + assert bytes(payload.parent_hash) == b"\x00" * 32 + # A zero-mode ``ByteList`` is a single zero byte (``min(1, limit)``), not + # empty -- emptiness is the job of ``nil_count``. + assert bytes(payload.extra_data) == b"\x00" + + +def test_max_mode_saturates_scalars_and_byte_vectors() -> None: + """``max`` mode maxes scalars and byte vectors.""" + payload = _value(ExecutionPayloadAmsterdam, RandomizationMode.mode_max) + assert int(payload.block_number) == 2**64 - 1 + assert bytes(payload.parent_hash) == b"\xff" * 32 + # A max-mode ``ByteList`` is a single ``0xff`` byte (``min(1, limit)``). + assert bytes(payload.extra_data) == b"\xff" + + +def test_nil_count_empties_collections() -> None: + """``nil_count`` empties every variable-length collection.""" + payload = _value( + ExecutionPayloadAmsterdam, RandomizationMode.mode_nil_count + ) + assert len(payload.transactions) == 0 + assert len(payload.withdrawals) == 0 + assert bytes(payload.extra_data) == b"" + + +def test_one_count_yields_single_element_collections() -> None: + """``one_count`` puts exactly one element in each collection.""" + payload = _value( + ExecutionPayloadAmsterdam, RandomizationMode.mode_one_count + ) + assert len(payload.transactions) == 1 + assert len(payload.withdrawals) == 1 + assert len(bytes(payload.extra_data)) == 1 + + +def test_max_count_fills_collections_to_cap() -> None: + """``max_count`` fills collections to the cap.""" + payload = _value( + ExecutionPayloadAmsterdam, RandomizationMode.mode_max_count + ) + assert len(payload.transactions) == MAX_LIST_LENGTH + assert len(payload.withdrawals) == MAX_LIST_LENGTH + # ``extra_data`` capacity (32) is below the byte cap (48), so it fills to + # its own smaller limit. + assert len(bytes(payload.extra_data)) == 32 + + +def test_random_mode_changes_with_seed() -> None: + """``random`` mode produces different values for different seeds.""" + a = encode_bytes( + _value(ExecutionPayloadAmsterdam, RandomizationMode.mode_random, 1) + ) + b = encode_bytes( + _value(ExecutionPayloadAmsterdam, RandomizationMode.mode_random, 2) + ) + assert a != b + + +def test_chaos_runs_and_round_trips() -> None: + """``chaos`` re-rolls the mode per node; the result still round-trips.""" + value = _value( + ExecutionPayloadAmsterdam, + RandomizationMode.mode_random, + seed=99, + chaos=True, + ) + assert ( + decode_bytes(ExecutionPayloadAmsterdam, encode_bytes(value)) == value + ) diff --git a/packages/testing/src/execution_testing/ssz/tests/test_vectors.py b/packages/testing/src/execution_testing/ssz/tests/test_vectors.py new file mode 100644 index 00000000000..e3be9e7a69b --- /dev/null +++ b/packages/testing/src/execution_testing/ssz/tests/test_vectors.py @@ -0,0 +1,52 @@ +""" +Self-tests for the SSZ vector generator. + +The vectors themselves are *not* committed -- like the consensus-specs SSZ +static vectors and EEST's own fixtures, they are generated artifacts produced +on demand (and released), not checked into the source tree. So this guards the +generator instead of any committed files: it is deterministic, and every case +it emits is internally consistent (``value``/``serialized``/``root`` agree). +To write vectors to disk for a release, run:: + + python -m execution_testing.ssz.vectors.generate +""" + +from .. import decode_bytes, encode_bytes, from_json, hash_tree_root +from ..vectors.generate import CONTAINERS, iter_cases, serialize_case + +CASES = list(iter_cases()) + + +def test_generator_produces_cases() -> None: + """Sanity: the generator actually produces cases.""" + assert CASES + + +def test_generation_is_deterministic() -> None: + """Two independent generations are byte-identical (seeded determinism).""" + again = [serialize_case(case) for _, _, _, case in iter_cases()] + assert again == [serialize_case(case) for _, _, _, case in CASES] + + +def test_each_case_is_internally_consistent() -> None: + """ + Every emitted case's three views agree. + + ``serialized`` decodes and re-encodes to itself, its ``hash_tree_root`` + equals ``root``, and the canonical-JSON ``value`` parses back to the same + bytes. This is a self-test of the generator, not a check against an + independent oracle (that would be circular; see ``vectors/README.md``). + """ + for container_name, _, _, case in CASES: + ssz_type = CONTAINERS[container_name] + serialized = bytes.fromhex(case["serialized"][2:]) + decoded = decode_bytes(ssz_type, serialized) + assert encode_bytes(decoded) == serialized, container_name + assert "0x" + hash_tree_root(decoded).hex() == case["root"] + assert encode_bytes(from_json(ssz_type, case["value"])) == serialized + + +def test_case_has_expected_keys() -> None: + """Each generated case carries the documented top-level keys.""" + for _, _, _, case in CASES: + assert {"meta", "value", "serialized", "root"} <= case.keys() diff --git a/packages/testing/src/execution_testing/ssz/vectors/.gitignore b/packages/testing/src/execution_testing/ssz/vectors/.gitignore new file mode 100644 index 00000000000..66144fe0188 --- /dev/null +++ b/packages/testing/src/execution_testing/ssz/vectors/.gitignore @@ -0,0 +1,2 @@ +# Generated SSZ vectors are a release artifact, never committed (see README.md). +**/case_*.json diff --git a/packages/testing/src/execution_testing/ssz/vectors/README.md b/packages/testing/src/execution_testing/ssz/vectors/README.md new file mode 100644 index 00000000000..adeaf27ce5f --- /dev/null +++ b/packages/testing/src/execution_testing/ssz/vectors/README.md @@ -0,0 +1,119 @@ +# SSZ test vectors + +This directory holds SSZ static test vectors for the containers in +`execution_testing.ssz`. There are **two kinds** of vector, with different +purposes and different rules about who may produce them. Keeping them straight +is important — confusing the two re-introduces a circular test. + +## 1. Cross-client reference vectors + +These are **generated artifacts, not committed** — exactly like the +consensus-specs SSZ static vectors (the pyspec repo ships only the generator; +the vectors are built and released as tarballs) and EEST's own `fill` fixtures. +[`generate.py`](./generate.py) is the producer; running it writes +`///case_.json` files for a release (the payload and +envelope are emitted per fork from the `*_BY_FORK` registries; fork-independent +containers like `Withdrawal` omit the `` segment), but the source tree +keeps none of them. Each case has the same shape the consensus-specs SSZ static +tests use: + +```json +{ + "meta": { "container": "...", "mode": "...", "seed": "0x...", ... }, + "value": { ...consensus-spec canonical JSON... }, + "serialized": "0x...", + "root": "0x..." +} +``` + +**Producer:** `remerkleable`, via this package's adapters. **Consumers:** +independent client SSZ implementations (Nimbus, Lighthouse, Prysm, Teku, +Lodestar, the EL clients, …). This is the same arrangement as the +consensus-specs generator, whose `spec` module is the producing reference and +whose consumers are separate client implementations. Producer and consumer are +different implementations, so this is **not** circular. + +For exactly that reason, this package's own correctness tests **must not** +consume these vectors as ground truth — that would be `remerkleable` checking +`remerkleable`. [`test_vectors.py`](../tests/test_vectors.py) only self-tests the +*generator*: that it is deterministic and that each emitted case's +`value`/`serialized`/`root` agree. That is a property of the generator, not a +correctness proof of the encoding. + +### Determinism + +Every case is seeded with `sha256(spec_version ‖ preset ‖ container ‖ mode ‖ +case_index)` — never Python's `hash()`, which is salted per process. The same +tuple always yields the same value, so regeneration is reproducible across runs +and machines. The seed is recorded in each case's `meta` for auditing. + +### Randomization modes + +These mirror the consensus-specs `RandomizationMode`. Scalar *content* is set by +`zero`/`max`; collection *length* is set by the `*_count` modes. Modes that +randomize content emit several cases; the rest are deterministic (1 case). + +- `zero` — scalars and byte vectors zeroed; collection lengths still random, + elements zero-valued (1 case). +- `max` — scalars and byte vectors saturated; collection lengths still random, + elements maxed (1 case). +- `nil_count` — every variable-length collection empty (1 case). +- `one_count` — exactly one element per collection, random content (several + cases). +- `max_count` — collections filled to the cap, random content (several cases). +- `random` — random content and random collection lengths (several cases). + +Note `zero`/`max` pin scalar *content* but leave collection *length* random — +emptiness and saturation of lengths are the job of `nil_count`/`max_count`. + +Variable-length fields are capped (`MAX_LIST_LENGTH`, `MAX_BYTES_LENGTH` in +`generate.py`); the declared SSZ capacities — e.g. `2**30` bytes per +transaction — cannot be materialized. + +### Generating + +```sh +python -m execution_testing.ssz.vectors.generate +``` + +This writes the vector files for a release/distribution. **Do not commit the +output** — it is a generated artifact. `test_vectors.py` guards the generator +itself (determinism + per-case consistency), so it needs no committed files. + +## 2. Ground-truth self-test vectors (still deferred) + +Separately, the package would benefit from `*.ssz` / `*.root` vectors used to +verify **this package's own** encoding — i.e. tests where `execution_testing.ssz` +is the thing under test. Those are **still deferred**, and the rule from the +original PR stands: + +> Reference vectors used to verify our own implementation are **ground truth**: +> they must be produced by an **independent** implementation, not by +> EEST/`remerkleable`, otherwise the test that consumes them is circular. + +The Amsterdam `ExecutionPayload` adds two fields — `block_access_list` +(EIP-7928) and `slot_number` — that are **not yet present in the +consensus-specs reference implementation**. Until an independent oracle covers +the full Amsterdam container, canonical ground-truth vectors for it cannot be +generated. That work lands when such an oracle exists. + +### Generation requirements (for the ground-truth follow-up) + +When those vectors are generated, they must: + +1. Be produced by an **independent** SSZ implementation (consensus-specs Python + reference, or another non-`remerkleable` implementation), run on a known, + committed input. +2. Pin the exact source commit(s): + - execution-apis: `src/engine/refactor-ssz.md` @ + `4e0fed12d3ebc9d1ca8829331a82b97b1d1bd154`. + - consensus-specs: `a84880a47a88700d8dfa451c2a7cd4b3f309bd0d` (for inherited + Deneb/Electra fields). +3. Cover both the `minimal` and `mainnet` presets. NOTE: for these + execution-payload size limits the two presets are identical, so the + `ExecutionPayload` bytes and root are **byte-identical** across presets; both + are committed regardless, matching the consensus-specs layout. +4. Exercise **multiple `transactions` of differing lengths** (≥ 3, distinct + lengths). `transactions` is a `List[ByteList, N]`, a two-level offset + encoding; degenerate inputs (one transaction, or all the same length) hide + bugs in the inner offset table. diff --git a/packages/testing/src/execution_testing/ssz/vectors/__init__.py b/packages/testing/src/execution_testing/ssz/vectors/__init__.py new file mode 100644 index 00000000000..13478203541 --- /dev/null +++ b/packages/testing/src/execution_testing/ssz/vectors/__init__.py @@ -0,0 +1 @@ +"""Cross-client SSZ static test vectors and their generator.""" diff --git a/packages/testing/src/execution_testing/ssz/vectors/generate.py b/packages/testing/src/execution_testing/ssz/vectors/generate.py new file mode 100644 index 00000000000..f46b57712c0 --- /dev/null +++ b/packages/testing/src/execution_testing/ssz/vectors/generate.py @@ -0,0 +1,179 @@ +""" +Generate cross-client SSZ static test vectors for this package's containers. + +For every registered container, for every :class:`RandomizationMode`, this +emits one or more cases of the form ``{value, serialized, root}`` -- the same +shape the consensus-specs SSZ static tests use: + +* ``value`` -- the consensus-spec canonical JSON of the object, +* ``serialized`` -- its canonical SSZ byte encoding, ``0x``-hex, +* ``root`` -- its ``hash_tree_root``, ``0x``-hex. + +These are *reference* vectors: ``remerkleable`` is the producing reference, +exactly as the consensus-specs ``spec`` module is for the beacon vectors. They +are meant for independent client implementations to check their own SSZ +encoders against, and are deliberately **not** consumed by this package's own +correctness tests -- doing so would be circular (see ``README.md``). + +The vectors are a generated artifact (released, not committed), mirroring the +consensus-specs SSZ static vectors and EEST's own fixtures. Run as a module to +write them to disk for a release; do not commit the output:: + + python -m execution_testing.ssz.vectors.generate +""" + +import json +from pathlib import Path +from random import Random +from typing import Any, Dict, Iterator, Tuple, Type + +from remerkleable.core import View + +from .. import ( + EXECUTION_PAYLOAD_BY_FORK, + EXECUTION_PAYLOAD_ENVELOPE_BY_FORK, + REFERENCE_SPEC_VERSION, + RandomizationMode, + deterministic_seed, + encode_bytes, + get_random_ssz_object, + hash_tree_root, + to_json, +) +from ..containers import Withdrawal + +# One generated vector: ``(container, mode, case_index, case)``. +Case = Tuple[str, RandomizationMode, int, Dict[str, Any]] + +# The SSZ size-limit preset. The consensus-specs ``minimal`` and ``mainnet`` +# presets are byte-identical for these execution-payload limits (see +# ``constants.py``), so a single preset axis is emitted. The *fork* is a +# separate axis, encoded in the on-disk container name below. +PRESET = "mainnet" + +# Caps for variable-length fields. The declared SSZ capacities (e.g. 2**30 +# bytes per transaction) cannot be materialized; these keep cases small while +# still exercising offset tables and partial/empty/full collections. +MAX_LIST_LENGTH = 4 +MAX_BYTES_LENGTH = 48 + +# Cases per changing (non-deterministic) mode. Deterministic modes get one. +RANDOM_CASE_COUNT = 3 + +# The containers to generate vectors for, keyed by the on-disk path. The +# payload and envelope are emitted per fork from the registries (one directory +# per fork); ``Withdrawal`` is fork-independent. +CONTAINERS: Dict[str, Type[View]] = { + "Withdrawal": Withdrawal, + **{ + f"ExecutionPayload/{fork}": cls + for fork, cls in EXECUTION_PAYLOAD_BY_FORK.items() + }, + **{ + f"ExecutionPayloadEnvelope/{fork}": cls + for fork, cls in EXECUTION_PAYLOAD_ENVELOPE_BY_FORK.items() + }, +} + + +def generate_case( + container_name: str, + ssz_type: Type[View], + mode: RandomizationMode, + case_index: int, +) -> Dict[str, Any]: + """ + Build a single ``{meta, value, serialized, root}`` vector case. + + The seed is derived from ``(spec version, preset, container, mode, case)`` + so the case is byte-for-byte reproducible. ``meta`` records the inputs so a + consumer can regenerate and audit the case independently. + """ + seed = deterministic_seed( + REFERENCE_SPEC_VERSION, + PRESET, + container_name, + mode.to_name(), + case_index, + ) + value = get_random_ssz_object( + Random(seed), + ssz_type, + MAX_BYTES_LENGTH, + MAX_LIST_LENGTH, + mode, + chaos=False, + ) + return { + "meta": { + "container": container_name, + "preset": PRESET, + "spec_version": REFERENCE_SPEC_VERSION, + "mode": mode.to_name(), + "case": case_index, + "max_list_length": MAX_LIST_LENGTH, + "max_bytes_length": MAX_BYTES_LENGTH, + "seed": f"0x{seed:064x}", + }, + "value": to_json(value), + "serialized": "0x" + encode_bytes(value).hex(), + "root": "0x" + hash_tree_root(value).hex(), + } + + +def iter_cases() -> Iterator[Case]: + """Yield ``(container, mode, case_index, case)`` for every vector.""" + for container_name, ssz_type in CONTAINERS.items(): + for mode in RandomizationMode: + count = RANDOM_CASE_COUNT if mode.is_changing() else 1 + for case_index in range(count): + yield ( + container_name, + mode, + case_index, + generate_case(container_name, ssz_type, mode, case_index), + ) + + +def case_path( + output_dir: Path, + container_name: str, + mode: RandomizationMode, + case_index: int, +) -> Path: + """Return the on-disk path for a given case.""" + return ( + output_dir + / container_name + / mode.to_name() + / f"case_{case_index}.json" + ) + + +def serialize_case(case: Dict[str, Any]) -> str: + """Return the canonical on-disk JSON text for a case.""" + return json.dumps(case, indent=2) + "\n" + + +def generate_all(output_dir: Path) -> int: + """ + Write every vector case under ``output_dir`` and return the case count. + """ + count = 0 + for container_name, mode, case_index, case in iter_cases(): + path = case_path(output_dir, container_name, mode, case_index) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(serialize_case(case)) + count += 1 + return count + + +def main() -> None: + """Regenerate the committed vectors in this directory.""" + output_dir = Path(__file__).parent + written = generate_all(output_dir) + print(f"wrote {written} vector cases to {output_dir}") + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index a6e7a5870c1..a15989524cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -481,6 +481,12 @@ exclude = [ ] plugins = ["pydantic.mypy"] +# `eth-remerkleable` ships no type information (no py.typed); treat its imports +# as untyped instead of vendoring stubs. +[[tool.mypy.overrides]] +module = "remerkleable.*" +ignore_missing_imports = true + [tool.uv] required-version = ">=0.7.0" diff --git a/uv.lock b/uv.lock index e2429416d77..e1996198b43 100644 --- a/uv.lock +++ b/uv.lock @@ -765,6 +765,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/eb/db/f8775490669d28aca24871c67dd56b3e72105cb3bcae9a4ec65dd70859b3/eth_hash-0.7.1-py3-none-any.whl", hash = "sha256:0fb1add2adf99ef28883fd6228eb447ef519ea72933535ad1a0b28c6f65f868a", size = 8028, upload-time = "2025-01-13T21:29:19.365Z" }, ] +[[package]] +name = "eth-remerkleable" +version = "0.1.31" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/51/ac/40fde655f67fd02f07b28e3fe4b9bb4af521388ca8aa48462d622ce4fa03/eth_remerkleable-0.1.31.tar.gz", hash = "sha256:94df4b18a50dfc46f55b2e790cfece384e64032e9cb82d185cff09224682ad1d", size = 49525, upload-time = "2026-06-11T13:23:38.552Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/a1/87f7c996f344c5f213c2bca657e579c3696bb99737238c0cf9f04867386a/eth_remerkleable-0.1.31-py3-none-any.whl", hash = "sha256:7e48ea1b80935977effc02300f3f9b1db19153ab87ffbcea14a6e6429bbdc56b", size = 57192, upload-time = "2026-06-11T13:23:37.208Z" }, +] + [[package]] name = "eth-typing" version = "5.2.1" @@ -1044,6 +1053,7 @@ dependencies = [ { name = "coincurve" }, { name = "colorlog" }, { name = "eth-abi" }, + { name = "eth-remerkleable" }, { name = "ethereum-execution" }, { name = "ethereum-hive" }, { name = "ethereum-rlp" }, @@ -1097,6 +1107,7 @@ requires-dist = [ { name = "coincurve", specifier = ">=20.0.0,<21" }, { name = "colorlog", specifier = ">=6.7.0,<7" }, { name = "eth-abi", specifier = ">=5.2.0" }, + { name = "eth-remerkleable", specifier = "==0.1.31" }, { name = "ethereum-execution", editable = "." }, { name = "ethereum-hive", specifier = ">=0.1.0a1,<1.0.0" }, { name = "ethereum-rlp", specifier = ">=0.1.3,<0.2" },