diff --git a/src/Nethermind/Nethermind.Consensus/Processing/BlockAccessListManager.TxProcessorPool.cs b/src/Nethermind/Nethermind.Consensus/Processing/BlockAccessListManager.TxProcessorPool.cs index 4bf3e07c0835..d9ecea40915f 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/BlockAccessListManager.TxProcessorPool.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/BlockAccessListManager.TxProcessorPool.cs @@ -20,6 +20,7 @@ using Nethermind.Int256; using Nethermind.Logging; using Nethermind.State; +using Nethermind.Consensus.Stateless; namespace Nethermind.Consensus.Processing; @@ -86,7 +87,7 @@ static ParallelTxProcessorWithWorldStateManager() private readonly ILogManager _logManager; private readonly ObjectPool? _parentReaderEnvPool; private int _processorCount; - private readonly bool _witnessMode; + private readonly Func? _isWitnessExecution; public ParallelTxProcessorWithWorldStateManager( IBlockhashProvider blockHashProvider, @@ -96,13 +97,13 @@ public ParallelTxProcessorWithWorldStateManager( PrewarmerEnvFactory? prewarmerEnvFactory, PreBlockCaches? preBlockCaches, IReadOnlyTxProcessingEnvFactory? readOnlyTxProcessingEnvFactory, - bool witnessMode) + Func? isWitnessExecution) { _blockHashProvider = blockHashProvider; _specProvider = specProvider; _stateProvider = stateProvider; _logManager = logManager; - _witnessMode = witnessMode; + _isWitnessExecution = isWitnessExecution; _parentReaderEnvPool = CreateParentReaderEnvPool(prewarmerEnvFactory, preBlockCaches, readOnlyTxProcessingEnvFactory); for (int i = 0; i < ProcessorPoolSize; i++) { @@ -226,7 +227,7 @@ private int ClampBalIndex(uint balIndex) => (int)uint.Min(balIndex, (uint)_lastBalIndex); private TxProcessorWithWorldState NewProcessor() - => new(true, _blockHashProvider, _specProvider, _stateProvider, _logManager, _witnessMode); + => new(true, _blockHashProvider, _specProvider, _stateProvider, _logManager, _isWitnessExecution); private TxProcessorWithWorldState RentProcessor() { @@ -333,9 +334,9 @@ public SequentialTxProcessorWithWorldStateManager( ISpecProvider specProvider, IWorldState stateProvider, ILogManager logManager, - bool witnessMode) + Func? isWitnessExecution) { - _txProcessorWithWorldState = new(false, blockHashProvider, specProvider, stateProvider, logManager, witnessMode); + _txProcessorWithWorldState = new(false, blockHashProvider, specProvider, stateProvider, logManager, isWitnessExecution); _txProcessorWithWorldState.WorldState.SetGeneratingBlockAccessList(new()); } @@ -377,7 +378,7 @@ public TxProcessorWithWorldState( ISpecProvider specProvider, IWorldState stateProvider, ILogManager logManager, - bool witnessMode) + Func? isWitnessExecution) { VirtualMachine virtualMachine = new(blockHashProvider, specProvider, logManager); @@ -388,12 +389,14 @@ public TxProcessorWithWorldState( worldState = _balWorldState; } WorldState = new TracedAccessWorldState(worldState, parallel); - // Witness mode must record every code access, so it uses the non-caching CodeInfoRepository. - // EthereumCodeInfoRepository wraps a CacheCodeInfoRepository whose process-wide static code - // cache serves hits without reading through the (traced) WorldState, so cached code accesses - // would be missing from the generated witness. - ICodeInfoRepository codeInfoRepository = witnessMode - ? new CodeInfoRepository(WorldState, new EthereumPrecompileProvider()) + // On witness-capable managers, route code lookups through CodeInfoRepositoryProxy: while a + // witness is being executed (predicate true) it uses the non-caching CodeInfoRepository so + // every code access flows through the (traced) WorldState; otherwise it uses the cached repo. + // The process-wide static code cache would otherwise serve hits without reading through the + // WorldState, dropping those accesses from the witness. Non-witness managers use the cached + // repo directly with no per-call indirection. + ICodeInfoRepository codeInfoRepository = isWitnessExecution is not null + ? new CodeInfoRepositoryProxy(new EthereumCodeInfoRepository(WorldState), WorldState, new EthereumPrecompileProvider(), isWitnessExecution) : new EthereumCodeInfoRepository(WorldState); TxProcessor = new(BlobBaseFeeCalculator.Instance, specProvider, WorldState, virtualMachine, codeInfoRepository, logManager, parallel); TxProcessorAdapter = new(TxProcessor); diff --git a/src/Nethermind/Nethermind.Consensus/Processing/BlockAccessListManager.cs b/src/Nethermind/Nethermind.Consensus/Processing/BlockAccessListManager.cs index 8bdcea2ec21e..c06ef0729b88 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/BlockAccessListManager.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/BlockAccessListManager.cs @@ -47,7 +47,7 @@ public partial class BlockAccessListManager( PrewarmerEnvFactory? prewarmerEnvFactory = null, PreBlockCaches? preBlockCaches = null, IReadOnlyTxProcessingEnvFactory? readOnlyTxProcessingEnvFactory = null, - bool witnessMode = false) + Func? isWitnessExecution = null) : IBlockAccessListManager, IDisposable { private readonly ILogger _logger = logManager.GetClassLogger(); @@ -55,9 +55,9 @@ public partial class BlockAccessListManager( private ITxProcessorWithWorldStateManager? _txProcessorWithWorldStateManager; private Task? _balWarmupTask; private readonly Lazy _parallelTxProcessorWithWorldStateManager = - new(() => new(blockHashProvider, specProvider, stateProvider, logManager, prewarmerEnvFactory, preBlockCaches, readOnlyTxProcessingEnvFactory, witnessMode)); + new(() => new(blockHashProvider, specProvider, stateProvider, logManager, prewarmerEnvFactory, preBlockCaches, readOnlyTxProcessingEnvFactory, isWitnessExecution)); private readonly Lazy _sequentialTxProcessorWithWorldStateManager = - new(() => new(blockHashProvider, specProvider, stateProvider, logManager, witnessMode)); + new(() => new(blockHashProvider, specProvider, stateProvider, logManager, isWitnessExecution)); private const int GasValidationChunkSize = 8; private long? _gasRemaining; private bool _isBuilding; @@ -124,11 +124,17 @@ public void PrepareForProcessing(Block suggestedBlock, IReleaseSpec spec, Proces // Parallel execution needs the decoded BAL body (RLP fixtures only carry the hash) // and an active state scope (so we can capture the parent state root for workers). + // + // Witness execution forces sequential: parallel workers read pre-state through pooled + // parent-reader snapshots that bypass the capturing world-state proxy, so their accesses + // would be missing from the witness. Evaluated per block via the predicate so regular + // blocks on EIP-7928 chains keep parallel execution. ParallelExecutionEnabled = Enabled && blocksConfig.ParallelExecution && !_isBuilding && suggestedBlock.BlockAccessList is not null - && stateProvider.IsInScope; + && stateProvider.IsInScope + && isWitnessExecution?.Invoke() is not true; // BAL-driven read warming: mirrors BlockCachePreWarmer.IsBalReadWarmingEnabled so // HintBal honours the same opt-in config as the prewarmer path. diff --git a/src/Nethermind/Nethermind.Consensus/Stateless/CodeInfoRepositoryProxy.cs b/src/Nethermind/Nethermind.Consensus/Stateless/CodeInfoRepositoryProxy.cs new file mode 100644 index 000000000000..c2dd3f181f8b --- /dev/null +++ b/src/Nethermind/Nethermind.Consensus/Stateless/CodeInfoRepositoryProxy.cs @@ -0,0 +1,49 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Diagnostics.CodeAnalysis; +using Nethermind.Core; +using Nethermind.Core.Specs; +using Nethermind.Evm; +using Nethermind.Evm.CodeAnalysis; +using Nethermind.Evm.State; + +namespace Nethermind.Consensus.Stateless; + +/// +/// Thin decorator that routes every call to a non-caching +/// it owns when returns true, +/// and to the wrapped cached repository otherwise. The predicate is evaluated per call so the choice +/// tracks the live witness-execution state rather than being fixed at construction. +/// +/// +/// Witness execution requires every bytecode/code-hash lookup to flow through +/// (so a recording world state observes it, or so stateless execution stays isolated to the witness); +/// the process-wide static code cache used by the inner repository would short-circuit those reads. +/// The non-caching repository is built inside this decorator (rather than resolved from DI) so no other +/// DI consumer can pick it up and accidentally bypass the cache for non-witness blocks. +/// +public sealed class CodeInfoRepositoryProxy( + ICodeInfoRepository inner, + IWorldState worldState, + IPrecompileProvider precompileProvider, + Func useNonCaching) : ICodeInfoRepository +{ + private readonly ICodeInfoRepository _inner = inner; + private readonly CodeInfoRepository _nonCached = new(worldState, precompileProvider); + + private ICodeInfoRepository Current => useNonCaching() ? _nonCached : _inner; + + public CodeInfo GetCachedCodeInfo(Address codeSource, bool followDelegation, IReleaseSpec vmSpec, out Address? delegationAddress) + => Current.GetCachedCodeInfo(codeSource, followDelegation, vmSpec, out delegationAddress); + + public void InsertCode(ReadOnlyMemory code, Address codeOwner, IReleaseSpec spec) + => Current.InsertCode(code, codeOwner, spec); + + public void SetDelegation(Address codeSource, Address authority, IReleaseSpec spec) + => Current.SetDelegation(codeSource, authority, spec); + + public bool TryGetDelegation(Address address, IReleaseSpec spec, [NotNullWhen(true)] out Address? delegatedAddress) + => Current.TryGetDelegation(address, spec, out delegatedAddress); +} diff --git a/src/Nethermind/Nethermind.Consensus/Stateless/StatelessBlockProcessingEnv.cs b/src/Nethermind/Nethermind.Consensus/Stateless/StatelessBlockProcessingEnv.cs index 96439cd003ac..8d873bc8ce5b 100644 --- a/src/Nethermind/Nethermind.Consensus/Stateless/StatelessBlockProcessingEnv.cs +++ b/src/Nethermind/Nethermind.Consensus/Stateless/StatelessBlockProcessingEnv.cs @@ -60,7 +60,9 @@ private BlockProcessor GetProcessor() ParallelExecutionBatchRead = false }, new WithdrawalProcessorFactory(logManager), - witnessMode: true + // Stateless execution must resolve code only from the witness-backed state, never the + // shared cache — so it always runs in witness mode (non-caching code reads). + isWitnessExecution: static () => true ); BlockProcessor.ParallelBlockValidationTransactionsExecutor txExecutor = new( new BlockProcessor.BlockValidationTransactionsExecutor( diff --git a/src/Nethermind/Nethermind.Consensus/Stateless/WitnessCaptureSession.cs b/src/Nethermind/Nethermind.Consensus/Stateless/WitnessCaptureSession.cs new file mode 100644 index 000000000000..3fd1f6acb107 --- /dev/null +++ b/src/Nethermind/Nethermind.Consensus/Stateless/WitnessCaptureSession.cs @@ -0,0 +1,66 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Threading; + +namespace Nethermind.Consensus.Stateless; + +/// +/// Per-main-processing-scope arming point for witness capture. Holds nullable pointers to the +/// recorders that are active during a single ProcessOne call; the decorators +/// (, , +/// ) consult these pointers on every call and forward +/// straight through to the inner component when null. +/// +/// +/// +/// Single arm/disarm point in replaces the previous +/// per-decorator TryActivate/Deactivate ceremony: all three decorators become dumb +/// passthroughs that share one source of truth here. +/// +/// +/// Thread-safety: uses CAS on the world-state pointer so concurrent armers +/// see at most one winner; clears in reverse so any consumer that still sees +/// set also sees the other recorders set. The main processing +/// pipeline drives blocks serially so contention is theoretical, but the volatile reads remain +/// safe under any caller. +/// +/// +public sealed class WitnessCaptureSession +{ + private WitnessGeneratingWorldState? _worldStateRecorder; + private WitnessHeaderRecorder? _headerRecorder; + private WitnessTrieStoreRecorder? _trieRecorder; + + public WitnessGeneratingWorldState? WorldStateRecorder => Volatile.Read(ref _worldStateRecorder); + public WitnessHeaderRecorder? HeaderRecorder => Volatile.Read(ref _headerRecorder); + public WitnessTrieStoreRecorder? TrieRecorder => Volatile.Read(ref _trieRecorder); + + public bool IsActive => WorldStateRecorder is not null; + + /// + /// Atomically installs the three recorders for a single capture pass. Returns false + /// when a capture is already in progress on this session. + /// + /// + /// The world-state recorder is the primary slot — the CAS on it gates the operation; the other + /// two are written under the post-CAS happens-before, so any reader that observes the + /// world-state recorder also observes the header and trie recorders. + /// + public bool TryArm( + WitnessGeneratingWorldState worldStateRecorder, + WitnessHeaderRecorder headerRecorder, + WitnessTrieStoreRecorder trieRecorder) + { + Volatile.Write(ref _trieRecorder, trieRecorder); + Volatile.Write(ref _headerRecorder, headerRecorder); + return Interlocked.CompareExchange(ref _worldStateRecorder, worldStateRecorder, null) is null; + } + + public void Disarm() + { + Volatile.Write(ref _worldStateRecorder, null); + Volatile.Write(ref _headerRecorder, null); + Volatile.Write(ref _trieRecorder, null); + } +} diff --git a/src/Nethermind/Nethermind.Consensus/Stateless/WitnessCapturingBlockProcessor.cs b/src/Nethermind/Nethermind.Consensus/Stateless/WitnessCapturingBlockProcessor.cs new file mode 100644 index 000000000000..9eb78d1978d1 --- /dev/null +++ b/src/Nethermind/Nethermind.Consensus/Stateless/WitnessCapturingBlockProcessor.cs @@ -0,0 +1,150 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Threading; +using System.Threading.Tasks; +using Nethermind.Consensus.Processing; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Specs; +using Nethermind.Evm.Tracing; +using Nethermind.Evm.State; +using Nethermind.Logging; +using Nethermind.State; + +namespace Nethermind.Consensus.Stateless; + +/// +/// decorator that, when a witness has been requested for the block +/// being processed, arms the with fresh per-block recorders for +/// the duration of one call, then projects the recorded set into a +/// and publishes it via . +/// +/// +/// +/// Three complementary capture surfaces are active during each witnessed ProcessOne call, +/// all gated by the session: +/// +/// +/// +/// / +/// — records every account/slot/bytecode access via call hooks. +/// Drives , which runs a tree visitor over +/// the recorded keys to produce Merkle proofs. +/// +/// +/// / +/// — catches header lookups from the EVM (e.g. BLOCKHASH) and the rest of the processing +/// pipeline so the witness header chain extends back to whatever the block touched. +/// +/// +/// / +/// — intercepts raw trie node reads at the storage layer for the case where branch nodes +/// collapse during state-root recomputation and siblings are read that never surface at the +/// level. +/// +/// +/// +/// All capture state lives on per-call instances installed onto the session — there is no global +/// armed/disarmed flag, no shared mutable dictionaries, and the session's atomic +/// rejects nested or concurrent capture attempts. +/// Blocks with no pending request bypass the capture machinery entirely. +/// +/// +public sealed class WitnessCapturingBlockProcessor( + IBlockProcessor inner, + WitnessCapturingWorldStateProxy proxy, + WitnessCapturingHeaderFinder headerFinder, + WitnessCaptureSession session, + WitnessRendezvous rendezvous, + IStateReader stateReader, + IWorldStateManager worldStateManager, + ILogManager? logManager = null) : IBlockProcessor +{ + private readonly ILogger _logger = (logManager ?? LimboLogs.Instance).GetClassLogger(); + + public event Action? TransactionsExecuted + { + add => inner.TransactionsExecuted += value; + remove => inner.TransactionsExecuted -= value; + } + + public (Block Block, TxReceipt[] Receipts) ProcessOne( + Block suggestedBlock, + ProcessingOptions options, + IBlockTracer blockTracer, + IReleaseSpec spec, + CancellationToken token = default) + { + Hash256? blockHash = suggestedBlock.Hash; + Hash256? parentHash = suggestedBlock.ParentHash; + + bool shouldCapture = + blockHash is not null + && parentHash is not null + && !options.ContainsFlag(ProcessingOptions.ReadOnlyChain) + && rendezvous.HasPendingRequest(blockHash); + + if (!shouldCapture) + return inner.ProcessOne(suggestedBlock, options, blockTracer, spec, token); + + long parentBlockNumber = suggestedBlock.Number - 1; + BlockHeader parent = headerFinder.Inner.Get(parentHash, parentBlockNumber) + ?? throw new ArgumentException($"Unable to find parent for block {parentBlockNumber} with hash {parentHash}"); + + WitnessTrieStoreRecorder trieRecorder = new(); + WitnessHeaderRecorder headerRecorder = new(); + // Backend-agnostic fallback store: CreateReadOnlyTrieStore returns the backend-appropriate + // read-only store (patricia ReadOnlyTrieStore or flat FlatReadOnlyTrieStore), so this builds + // no patricia machinery on a flat node. Live trie-node capture during execution happens + // elsewhere — the main-world ITrieStore decorator on patricia, the trie-read observer on + // flat — both feeding the same recorder; this instance backs only GetWitness's rarely-hit + // root-resolution fallback. + WitnessCapturingTrieStore fallbackTrieStore = new(worldStateManager.CreateReadOnlyTrieStore(), session); + WitnessGeneratingWorldState recorder = new( + proxy.InnerState, + stateReader, + fallbackTrieStore, + trieRecorder, + headerRecorder, + headerFinder.Inner); + + if (!session.TryArm(recorder, headerRecorder, trieRecorder)) + { + // Another capture is in progress for some other block on this session. Skip capture + // for this one rather than risking interleaved recording. + if (_logger.IsWarn) _logger.Warn($"{nameof(WitnessCapturingBlockProcessor)}: session already armed when processing {blockHash}; skipping capture."); + return inner.ProcessOne(suggestedBlock, options, blockTracer, spec, token); + } + + try + { + (Block Block, TxReceipt[] Receipts) result = inner.ProcessOne(suggestedBlock, options, blockTracer, spec, token); + + if (!rendezvous.TryClaim(blockHash!, out TaskCompletionSource? tcs)) + return result; // request was cancelled while we were processing — nothing to publish. + + Witness? witness = null; + try + { + witness = recorder.GetWitness(parent); + } + catch (Exception ex) + { + if (_logger.IsError) _logger.Error($"{nameof(WitnessCapturingBlockProcessor)}: witness build failed for block {blockHash}", ex); + } + tcs!.SetResult(witness); + return result; + } + catch + { + rendezvous.CancelWitnessRequest(blockHash!); + throw; + } + finally + { + session.Disarm(); + } + } +} diff --git a/src/Nethermind/Nethermind.Consensus/Stateless/WitnessCapturingHeaderFinder.cs b/src/Nethermind/Nethermind.Consensus/Stateless/WitnessCapturingHeaderFinder.cs new file mode 100644 index 000000000000..3be5dcec71a1 --- /dev/null +++ b/src/Nethermind/Nethermind.Consensus/Stateless/WitnessCapturingHeaderFinder.cs @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Blockchain.Headers; +using Nethermind.Core; +using Nethermind.Core.Crypto; + +namespace Nethermind.Consensus.Stateless; + +/// +/// Transparent decorator that, when a capture is armed on the +/// , side-channels every successful header lookup into the +/// session's recorder. Catches BLOCKHASH lookups +/// during EVM execution so the witness headers chain extends back to whatever the EVM touched. +/// +public sealed class WitnessCapturingHeaderFinder(IHeaderFinder inner, WitnessCaptureSession session) : IHeaderFinder +{ + /// + /// The undecorated inner header finder. Exposed so witness-build code can walk ancestor headers + /// without re-entering the capture path — see . + /// + internal IHeaderFinder Inner => inner; + + public BlockHeader? Get(Hash256 blockHash, long? blockNumber = null) + { + BlockHeader? header = inner.Get(blockHash, blockNumber); + if (header is not null && session.HeaderRecorder is { } recorder) recorder.OnHeaderRead(header); + return header; + } +} diff --git a/src/Nethermind/Nethermind.Consensus/Stateless/WitnessCapturingMainProcessingModule.cs b/src/Nethermind/Nethermind.Consensus/Stateless/WitnessCapturingMainProcessingModule.cs new file mode 100644 index 000000000000..276c8dc73418 --- /dev/null +++ b/src/Nethermind/Nethermind.Consensus/Stateless/WitnessCapturingMainProcessingModule.cs @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Autofac; +using Nethermind.Blockchain.Headers; +using Nethermind.Consensus.Processing; +using Nethermind.Core; +using Nethermind.Core.Container; +using Nethermind.Core.Specs; +using Nethermind.Evm; +using Nethermind.Evm.State; + +namespace Nethermind.Consensus.Stateless; + +/// +/// On EIP-7928 chains, wires up in-flight witness capture for the main processing pipeline: +/// installs the , +/// and decorators, and the shared +/// that they all consult for the active per-block recorders. +/// +public sealed class WitnessCapturingMainProcessingModule(ISpecProvider specProvider) : Module, IMainProcessingModule +{ + protected override void Load(ContainerBuilder builder) + { + if (!specProvider.GetFinalSpec().IsEip7928Enabled) return; + + // Note: WitnessCaptureSession is registered at root (by the merge plugin) so the main-world + // trie store's read-tap — constructed at root, before this child scope exists — shares the + // same instance the decorators below consult. Re-registering it here would shadow it. + + // Signals to this scope's BlockAccessListManager that it executes for witness capture; the + // predicate tracks the armed session, so BAL forces sequential + non-caching only for the + // block actually being witnessed. + builder.AddScoped( + session => new WitnessExecutionPredicate(() => session.IsActive)); + + builder.AddDecorator(); + // Expose the same proxy instance as a typed singleton so the block-processor decorator can + // take it directly. Cast through IWorldState because Autofac doesn't model decorator chains + // as typed singletons. + builder.AddSingleton(ctx => + (WitnessCapturingWorldStateProxy)ctx.Resolve()); + + builder.AddDecorator(); + // Same typed-singleton bridge for the header-finder decorator so the block processor can + // grab its undecorated inner via .Inner when building the per-block recorder. + builder.AddSingleton(ctx => + (WitnessCapturingHeaderFinder)ctx.Resolve()); + + // Main-pipeline components in this child scope resolve a session-aware decorator that, when + // capture is armed, routes calls to a non-caching CodeInfoRepository (so every bytecode + // lookup flows through IWorldState → proxy → recorder) and, when disarmed, routes back to + // the cached repository registered at root. Other scopes (block production, RPC simulation, + // the legacy debug_executionWitness sandbox) are untouched. + builder.AddDecorator((ctx, repository) => + { + WitnessCaptureSession session = ctx.Resolve(); + return new CodeInfoRepositoryProxy( + repository, + ctx.Resolve(), + ctx.Resolve(), + () => session.IsActive); + }); + + builder.AddDecorator(); + } +} diff --git a/src/Nethermind/Nethermind.Consensus/Stateless/WitnessCapturingTrieStore.cs b/src/Nethermind/Nethermind.Consensus/Stateless/WitnessCapturingTrieStore.cs index 8c4544643a6d..7b545d0a1be0 100644 --- a/src/Nethermind/Nethermind.Consensus/Stateless/WitnessCapturingTrieStore.cs +++ b/src/Nethermind/Nethermind.Consensus/Stateless/WitnessCapturingTrieStore.cs @@ -1,9 +1,8 @@ -// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only using System; -using System.Collections.Generic; -using System.Runtime.InteropServices; +using System.Threading; using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Trie; @@ -11,33 +10,36 @@ namespace Nethermind.Consensus.Stateless; +/// +/// decorator that, when a capture is armed on the +/// , side-channels every resolved node read into the session's +/// . +/// /// -/// Delegates all logic to base store except for writing trie nodes (readonly!) /// Adds logic for capturing trie nodes accessed during execution and state root recomputation. +/// Two commit modes: +/// +/// Read-only (default) — commits are swallowed (). Used +/// when wrapping a read-only store for re-execution sandboxes and post-hoc proof collection, +/// where writes must never reach persistence. +/// Write-through (readOnly: false) — commits forward verbatim. Used when +/// decorating the live main-world trie store, which persists state; reads are still recorded, +/// but only clean (persisted) nodes — dirty in-memory nodes have no +/// /RLP yet and represent post-state anyway. +/// /// -public class WitnessCapturingTrieStore(IReadOnlyTrieStore baseStore) : ITrieStore +public class WitnessCapturingTrieStore(ITrieStore baseStore, WitnessCaptureSession session, bool readOnly = true) : ITrieStore { - // Plain Dictionary, not ConcurrentDictionary: a rented entry is exclusive to a single synchronous - // caller, so the collector only ever sees one writer per rent. - private readonly Dictionary _rlpCollector = []; - - public IEnumerable TouchedNodesRlp => _rlpCollector.Values; - - /// Clears the captured-node set so the wrapper can be reused across pooled rents. - public void Reset() => _rlpCollector.Clear(); - - public void Dispose() => baseStore.Dispose(); + private int _disposed; public TrieNode FindCachedOrUnknown(Hash256? address, in TreePath path, Hash256 hash) { TrieNode node = baseStore.FindCachedOrUnknown(address, in path, hash); - if (node.NodeType != NodeType.Unknown) - { - // Materialise the RLP only on first capture: TryAdd would allocate node.FullRlp.ToArray() - // on every cache hit (hot in SLOAD loops touching the same branch) just to discard it. - ref byte[]? slot = ref CollectionsMarshal.GetValueRefOrAddDefault(_rlpCollector, node.Keccak, out bool exists); - if (!exists) slot = node.FullRlp.ToArray(); - } + // Pass the node, not its RLP: the recorder materialises node.FullRlp.ToArray() only on first + // capture, avoiding the allocate-then-discard on every cache hit (hot in SLOAD loops touching + // the same branch). + if (node.NodeType != NodeType.Unknown && session.TrieRecorder is { } recorder) + recorder.Record(node.Keccak, node); return node; } @@ -48,20 +50,35 @@ public TrieNode FindCachedOrUnknown(Hash256? address, in TreePath path, Hash256 public byte[]? TryLoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) { byte[]? rlp = baseStore.TryLoadRlp(address, in path, hash, flags); - if (rlp is not null) _rlpCollector.TryAdd(hash, rlp); + if (rlp is not null && session.TrieRecorder is { } recorder) recorder.Record(hash, rlp); return rlp; } public bool HasRoot(Hash256 stateRoot) => baseStore.HasRoot(stateRoot); + public bool HasRoot(Hash256 stateRoot, long blockNumber) => baseStore.HasRoot(stateRoot, blockNumber); + public IDisposable BeginScope(BlockHeader? baseBlock) => baseStore.BeginScope(baseBlock); + // Route through `this` (not baseStore.GetTrieStore) so scoped reads stay captured. public IScopedTrieStore GetTrieStore(Hash256? address) => new ScopedTrieStore(this, address); public INodeStorage.KeyScheme Scheme => baseStore.Scheme; - public IBlockCommitter BeginBlockCommit(long blockNumber) => NullCommitter.Instance; + public IBlockCommitter BeginBlockCommit(long blockNumber) => + readOnly ? NullCommitter.Instance : baseStore.BeginBlockCommit(blockNumber); - // WitnessCapturingTrieStore is read-only, so we return a no-op committer that doesn't persist any trie nodes - public ICommitter BeginCommit(Hash256? address, TrieNode? root, WriteFlags writeFlags) => NullCommitter.Instance; + public ICommitter BeginCommit(Hash256? address, TrieNode? root, WriteFlags writeFlags) => + readOnly ? NullCommitter.Instance : baseStore.BeginCommit(address, root, writeFlags); + + /// + /// Dispose-once guard: in write-through mode the dispose stack owns the store's shutdown (cache + /// persistence), but Autofac also disposes decorator instances at container teardown. The second + /// call must not reach the inner store — TrieStore.Dispose re-runs PersistOnShutdown against + /// closed DBs. + /// + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) == 0) baseStore.Dispose(); + } } diff --git a/src/Nethermind/Nethermind.Consensus/Stateless/WitnessCapturingWorldStateProxy.cs b/src/Nethermind/Nethermind.Consensus/Stateless/WitnessCapturingWorldStateProxy.cs new file mode 100644 index 000000000000..ccc4c108a224 --- /dev/null +++ b/src/Nethermind/Nethermind.Consensus/Stateless/WitnessCapturingWorldStateProxy.cs @@ -0,0 +1,108 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Threading.Tasks; +using Nethermind.Core; +using Nethermind.Core.BlockAccessLists; +using Nethermind.Core.Collections; +using Nethermind.Core.Crypto; +using Nethermind.Core.Eip2930; +using Nethermind.Core.Specs; +using Nethermind.Evm.State; +using Nethermind.Evm.Tracing.State; +using Nethermind.Int256; + +namespace Nethermind.Consensus.Stateless; + +/// +/// decorator installed on the main-processing pipeline that routes every +/// call to either the per-block recorder published on or +/// straight through to the inner world state when no capture is armed. +/// +/// +/// Holds no recording state of its own — the session ([[witness-capture-session]]) owns the active +/// recorder pointer, the recorder ([[witness-generating-world-state]]) owns the captured data, and +/// arms/disarms the session for one ProcessOne +/// call. +/// +public sealed class WitnessCapturingWorldStateProxy(IWorldState inner, WitnessCaptureSession session) : IWorldState +{ + /// The undecorated inner world state. Used by the block-processor decorator to anchor a fresh recorder. + internal IWorldState InnerState => inner; + + private IWorldState Current => session.WorldStateRecorder ?? inner; + + public bool HasStateForBlock(BlockHeader? baseBlock) => Current.HasStateForBlock(baseBlock); + public void Restore(Snapshot snapshot) => Current.Restore(snapshot); + public Hash256 StateRoot => Current.StateRoot; + public bool IsInScope => Current.IsInScope; + public IWorldStateScopeProvider ScopeProvider => Current.ScopeProvider; + public IDisposable BeginScope(BlockHeader? baseBlock) => Current.BeginScope(baseBlock); + public Task HintBal(ReadOnlyBlockAccessList bal) => Current.HintBal(bal); + + public bool TryGetAccount(Address address, out AccountStruct account) => Current.TryGetAccount(address, out account); + public UInt256 GetNonce(Address address) => Current.GetNonce(address); + public bool IsStorageEmpty(Address address) => Current.IsStorageEmpty(address); + public bool HasCode(Address address) => Current.HasCode(address); + public bool IsNonZeroAccount(Address address, out bool accountExists) => Current.IsNonZeroAccount(address, out accountExists); + public bool IsDelegatedCode(Address address) => Current.IsDelegatedCode(address); + public bool IsDelegatedCode(in ValueHash256 codeHash) => Current.IsDelegatedCode(in codeHash); + public byte[]? GetCode(Address address) => Current.GetCode(address); + public byte[]? GetCode(in ValueHash256 codeHash) => Current.GetCode(in codeHash); + public bool IsContract(Address address) => Current.IsContract(address); + public bool AccountExists(Address address) => Current.AccountExists(address); + public bool IsDeadAccount(Address address) => Current.IsDeadAccount(address); + public ref readonly UInt256 GetBalance(Address address) => ref Current.GetBalance(address); + public ref readonly ValueHash256 GetCodeHash(Address address) => ref Current.GetCodeHash(address); + + public ReadOnlySpan GetOriginal(in StorageCell storageCell) => Current.GetOriginal(in storageCell); + public ReadOnlySpan Get(in StorageCell storageCell) => Current.Get(in storageCell); + public void Set(in StorageCell storageCell, byte[] newValue) => Current.Set(in storageCell, newValue); + + public ReadOnlySpan GetTransientState(in StorageCell storageCell) => Current.GetTransientState(in storageCell); + public void SetTransientState(in StorageCell storageCell, byte[] newValue) => Current.SetTransientState(in storageCell, newValue); + + public void Reset(bool resetBlockChanges = true) => Current.Reset(resetBlockChanges); + public Snapshot TakeSnapshot(bool newTransactionStart = false) => Current.TakeSnapshot(newTransactionStart); + + public void WarmUp(AccessList? accessList) => Current.WarmUp(accessList); + public void WarmUp(Address address) => Current.WarmUp(address); + + public void ClearStorage(Address address) => Current.ClearStorage(address); + public void RecalculateStateRoot() => Current.RecalculateStateRoot(); + + public void DeleteAccount(Address address) => Current.DeleteAccount(address); + public void CreateAccount(Address address, in UInt256 balance, in UInt256 nonce = default) => Current.CreateAccount(address, in balance, in nonce); + public void CreateAccountIfNotExists(Address address, in UInt256 balance, in UInt256 nonce = default) => Current.CreateAccountIfNotExists(address, in balance, in nonce); + + public bool InsertCode(Address address, in ValueHash256 codeHash, ReadOnlyMemory code, IReleaseSpec spec, bool isGenesis = false) => + Current.InsertCode(address, in codeHash, code, spec, isGenesis); + + public void AddToBalance(Address address, in UInt256 balanceChange, IReleaseSpec spec, out UInt256 oldBalance) => + Current.AddToBalance(address, in balanceChange, spec, out oldBalance); + + public bool AddToBalanceAndCreateIfNotExists(Address address, in UInt256 balanceChange, IReleaseSpec spec, out UInt256 oldBalance) => + Current.AddToBalanceAndCreateIfNotExists(address, in balanceChange, spec, out oldBalance); + + public void SubtractFromBalance(Address address, in UInt256 balanceChange, IReleaseSpec spec, out UInt256 oldBalance) => + Current.SubtractFromBalance(address, in balanceChange, spec, out oldBalance); + + public void IncrementNonce(Address address, UInt256 delta, out UInt256 oldNonce) => Current.IncrementNonce(address, delta, out oldNonce); + public void DecrementNonce(Address address, UInt256 delta) => Current.DecrementNonce(address, delta); + public void SetNonce(Address address, in UInt256 nonce) => Current.SetNonce(address, in nonce); + + public void Commit(IReleaseSpec releaseSpec, IWorldStateTracer tracer, bool isGenesis = false, bool commitRoots = true) => + Current.Commit(releaseSpec, tracer, isGenesis, commitRoots); + + public void CommitTree(long blockNumber) => Current.CommitTree(blockNumber); + public ArrayPoolList? GetAccountChanges() => Current.GetAccountChanges(); + public void ResetTransient() => Current.ResetTransient(); + + public void CreateEmptyAccountIfDeleted(Address address) => Current.CreateEmptyAccountIfDeleted(address); + public void AddAccountRead(Address address) => Current.AddAccountRead(address); + public IDisposable? BeginSystemAccountReadSuppression() => Current.BeginSystemAccountReadSuppression(); + + public void RecordBytecodeAccess(Address address) + => Current.RecordBytecodeAccess(address); +} diff --git a/src/Nethermind/Nethermind.Consensus/Stateless/WitnessExecutionPredicate.cs b/src/Nethermind/Nethermind.Consensus/Stateless/WitnessExecutionPredicate.cs new file mode 100644 index 000000000000..d259701cfb41 --- /dev/null +++ b/src/Nethermind/Nethermind.Consensus/Stateless/WitnessExecutionPredicate.cs @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; + +namespace Nethermind.Consensus.Stateless; + +/// +/// Per-scope signal that block execution in this scope serves witness purposes — recording on the +/// main pipeline ( tracks the armed ), +/// recording in the legacy debug_executionWitness sandbox, or stateless verification. +/// +/// +/// Drives BlockAccessListManager to force sequential execution and bypass the shared code +/// cache while returns true. Registered only in witness-capable scopes; +/// its absence (the common case) leaves BAL on the fast parallel + cached path. Evaluated per block, +/// so on the main pipeline it is active only for the specific block being witnessed. +/// +public sealed record WitnessExecutionPredicate(Func IsActive); diff --git a/src/Nethermind/Nethermind.Consensus/Stateless/WitnessGeneratingBlockProcessingEnvFactory.cs b/src/Nethermind/Nethermind.Consensus/Stateless/WitnessGeneratingBlockProcessingEnvFactory.cs index 2db71ad18ede..2e8c3fbe5e90 100644 --- a/src/Nethermind/Nethermind.Consensus/Stateless/WitnessGeneratingBlockProcessingEnvFactory.cs +++ b/src/Nethermind/Nethermind.Consensus/Stateless/WitnessGeneratingBlockProcessingEnvFactory.cs @@ -32,11 +32,11 @@ public interface IWitnessGeneratingBlockProcessingEnvFactory /// Builds a on demand and pools entries for reuse. /// /// -/// Each rent returns a fully-wired env (own WorldState stack, trie-store wrapper, header finder, Autofac -/// child scope). The first rent on an empty pool pays full construction cost; subsequent rents reuse a -/// pooled entry. Entries are reset on return (so a pooled entry never pins its last call's witness -/// buffers) and the pool is soft-capped — surplus and poisoned entries are disposed rather than pooled. -/// Disposing the factory drains the pool. +/// Each rent returns a fully-wired env (own WorldState stack, capturing trie-store wrapper, header +/// finder, per-entry , Autofac child scope). The first rent on an +/// empty pool pays full construction cost; subsequent rents reuse a pooled entry. Entries are reset on +/// return (so a pooled entry never pins its last call's witness buffers) and the pool is soft-capped — +/// surplus and poisoned entries are disposed rather than pooled. Disposing the factory drains the pool. /// public class WitnessGeneratingBlockProcessingEnvFactory( ILifetimeScope rootLifetimeScope, @@ -72,31 +72,45 @@ public IWitnessGeneratingBlockProcessingEnvScope CreateScope() private PooledEntry BuildEntry() { IReadOnlyDbProvider readOnlyDbProvider = new ReadOnlyDbProvider(dbProvider, true); - WitnessCapturingTrieStore trieStore = new(worldStateManager.CreateReadOnlyTrieStore()); + + // Per-entry session + recorders. The session is armed once for the entry's lifetime (the env's + // components are wired directly, not via the main-pipeline proxy); Reset() clears the recorder + // data between rents while leaving the session armed at the same recorder instances. + WitnessCaptureSession session = new(); + WitnessTrieStoreRecorder trieRecorder = new(); + WitnessHeaderRecorder headerRecorder = new(); + + WitnessCapturingTrieStore trieStore = new(worldStateManager.CreateReadOnlyTrieStore(), session); IStateReader stateReader = new StateReader(trieStore, readOnlyDbProvider.CodeDb, logManager); IWorldState baseWorldState = new WorldState( new TrieStoreScopeProvider(trieStore, readOnlyDbProvider.CodeDb, logManager), logManager); IHeaderStore headerStore = rootLifetimeScope.Resolve(); - WitnessGeneratingHeaderFinder headerFinder = new(headerStore); - // Proof-collection walks go through the global (non-capturing) reader; the capturing - // stateReader serves execution-path reads. - WitnessGeneratingWorldState witnessWorldState = new(baseWorldState, worldStateManager.GlobalStateReader, trieStore, headerFinder); + WitnessCapturingHeaderFinder capturingHeaderFinder = new(headerStore, session); + // Proof-collection walks go through the global (non-capturing) reader; the capturing trieStore + // serves execution-path reads (not account proof collection). headerStore is the undecorated source BuildHeaders walks. + WitnessGeneratingWorldState witnessWorldState = new( + baseWorldState, worldStateManager.GlobalStateReader, trieStore, trieRecorder, headerRecorder, headerStore); + + session.TryArm(witnessWorldState, headerRecorder, trieRecorder); ILifetimeScope envLifetimeScope = rootLifetimeScope.BeginLifetimeScope(builder => builder .AddScoped(stateReader) .AddScoped(witnessWorldState) .AddScoped(witnessWorldState) - .AddScoped(headerFinder) + .AddScoped(capturingHeaderFinder) .AddScoped() .AddScoped(NullReceiptStorage.Instance) .AddScoped() + // The whole sandbox re-execution records a witness, so its BlockAccessListManager runs in + // witness mode unconditionally (sequential + non-caching code reads). + .AddScoped(new WitnessExecutionPredicate(static () => true)) .AddModule(validationModules) .AddScoped()); IWitnessGeneratingBlockProcessingEnv env = envLifetimeScope.Resolve(); IBlockhashCache blockhashCache = envLifetimeScope.Resolve(); - return new PooledEntry(envLifetimeScope, readOnlyDbProvider, trieStore, witnessWorldState, headerFinder, blockhashCache, env); + return new PooledEntry(envLifetimeScope, readOnlyDbProvider, trieRecorder, headerRecorder, witnessWorldState, blockhashCache, env); } private void Return(PooledEntry entry) @@ -150,9 +164,9 @@ public void Dispose() private sealed class PooledEntry( ILifetimeScope scope, IReadOnlyDbProvider dbProvider, - WitnessCapturingTrieStore trieStore, + WitnessTrieStoreRecorder trieRecorder, + WitnessHeaderRecorder headerRecorder, WitnessGeneratingWorldState worldState, - WitnessGeneratingHeaderFinder headerFinder, IBlockhashCache blockhashCache, IWitnessGeneratingBlockProcessingEnv env) { @@ -163,13 +177,14 @@ private sealed class PooledEntry( /// /// The inner WorldState's per-call caches are already cleared by WorldState.BeginScope's /// scope-exit Reset(true); only the witness-specific accumulators are cleared here. The + /// session stays armed at the same recorder instances — clearing the recorders is enough. The /// blockhash cache is content-addressed (never stale) but grows per entry, so it's cleared too. /// public void Reset() { - trieStore.Reset(); + trieRecorder.Reset(); + headerRecorder.Reset(); worldState.Reset(); - headerFinder.Reset(); dbProvider.ClearTempChanges(); blockhashCache.Clear(); } diff --git a/src/Nethermind/Nethermind.Consensus/Stateless/WitnessGeneratingWorldState.cs b/src/Nethermind/Nethermind.Consensus/Stateless/WitnessGeneratingWorldState.cs index f33014f39984..0e7550a2a1e1 100644 --- a/src/Nethermind/Nethermind.Consensus/Stateless/WitnessGeneratingWorldState.cs +++ b/src/Nethermind/Nethermind.Consensus/Stateless/WitnessGeneratingWorldState.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Runtime.InteropServices; using Collections.Pooled; +using Nethermind.Blockchain.Headers; using Nethermind.Core; using Nethermind.Core.Collections; using Nethermind.Core.Crypto; @@ -21,12 +22,14 @@ namespace Nethermind.Consensus.Stateless; /// Serves the post-execution proof-collection walks in ; /// must be a plain (non-capturing) reader — re-traversal is proof collection, not state access, so -/// recording it into would only duplicate the witness buffers. +/// recording it into would only duplicate the witness buffers. public class WitnessGeneratingWorldState( IWorldState state, IStateReader stateReader, WitnessCapturingTrieStore trieStore, - WitnessGeneratingHeaderFinder headerFinder) + WitnessTrieStoreRecorder trieRecorder, + WitnessHeaderRecorder headerRecorder, + IHeaderFinder headerFinder) : WorldStateDecorator(state) { private readonly Dictionary> _storageSlots = []; @@ -46,17 +49,23 @@ public Witness GetWitness(BlockHeader parentHeader) // trie nodes were never captured. The walk below re-traverses the touched keys to capture // them — only needed for cross-client (e.g. geth) stateless re-execution; our own execution // wouldn't require it. - if (!trieStore.TouchedNodesRlp.Any()) + if (!trieRecorder.TouchedNodesRlp.Any()) { // No storage-slot or account reads — lazy TrieNode handling can leave the root node // uncaptured. Resolve it explicitly so the witness still includes it. + // + // BeginScope is required on flat — the fresh read-only store gathers its snapshot bundle + // per scope, keyed by (blockNumber, stateRoot); without it the resolve throws. On patricia + // it is a no-op (content-addressed store, globally available). Scoped here (not hoisted) so + // flat only gathers the bundle on the rare blocks where this fallback actually fires. + using IDisposable _ = trieStore.BeginScope(parentHeader); ITrieNodeResolver stateResolver = trieStore.GetTrieStore(null); TreePath path = TreePath.Empty; TrieNode node = stateResolver.FindCachedOrUnknown(path, parentHeader.StateRoot!); node.ResolveNode(stateResolver, path); } - using PooledSet stateNodes = new(trieStore.TouchedNodesRlp, Bytes.EqualityComparer); + using PooledSet stateNodes = new(trieRecorder.TouchedNodesRlp, Bytes.EqualityComparer); if (_storageSlots.Count > 0) { // A single walk captures both the state-trie path to every touched account and, via the @@ -103,13 +112,13 @@ public Witness GetWitness(BlockHeader parentHeader) Codes = codes, State = state, Keys = keys, - Headers = headerFinder.GetWitnessHeaders(parentHeader.Hash!) + Headers = headerRecorder.BuildHeaders(parentHeader.Hash!, headerFinder) }; } catch { // Any failure mid-build returns the rented buffers before propagating, else they leak: - // an OOM while filling a list, or GetWitnessHeaders throwing because a walked ancestor + // an OOM while filling a list, or BuildHeaders throwing because a walked ancestor // header vanished (reorg/prune between the call and the witness build). codes?.Dispose(); state?.Dispose(); diff --git a/src/Nethermind/Nethermind.Consensus/Stateless/WitnessGeneratingHeaderFinder.cs b/src/Nethermind/Nethermind.Consensus/Stateless/WitnessHeaderRecorder.cs similarity index 56% rename from src/Nethermind/Nethermind.Consensus/Stateless/WitnessGeneratingHeaderFinder.cs rename to src/Nethermind/Nethermind.Consensus/Stateless/WitnessHeaderRecorder.cs index d14d4826badb..2930d499f8c1 100644 --- a/src/Nethermind/Nethermind.Consensus/Stateless/WitnessGeneratingHeaderFinder.cs +++ b/src/Nethermind/Nethermind.Consensus/Stateless/WitnessHeaderRecorder.cs @@ -1,54 +1,62 @@ -// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only using System; using Nethermind.Blockchain.Headers; using Nethermind.Core; +using Nethermind.Core.Collections; using Nethermind.Core.Crypto; using Nethermind.Serialization.Rlp; -using Nethermind.Core.Collections; namespace Nethermind.Consensus.Stateless; -public class WitnessGeneratingHeaderFinder(IHeaderFinder inner) : IHeaderFinder +/// +/// Per-capture recorder of header reads. Lives on the while a +/// capture is armed; the IHeaderFinder decorator ([[witness-capturing-header-finder]]) reports every +/// header lookup here so can emit the contiguous header chain that the +/// stateless verifier needs. +/// +/// +/// The chain runs from _lowestRequestedHeader (a low-water mark of every header touched +/// during execution — e.g. by BLOCKHASH reaching back into the past) to the parent of the recorded +/// block, in ascending block-number order. The caller passes the undecorated +/// into so walking the chain at build time +/// does not re-enter the capture path. +/// +public sealed class WitnessHeaderRecorder { private static readonly HeaderDecoder _decoder = new(); - // Not thread-safe: a pooled rent is reset then used on a single thread, so no synchronization is - // needed. Concurrent use of one instance would require it. private long _lowestRequestedHeader = long.MaxValue; - /// Resets BLOCKHASH bookkeeping so this instance can be reused across pooled rents. + /// Resets the low-water mark so the recorder can be reused across pooled env rents. public void Reset() => _lowestRequestedHeader = long.MaxValue; - public BlockHeader? Get(Hash256 blockHash, long? blockNumber = null) + public void OnHeaderRead(BlockHeader header) { - BlockHeader? header = inner.Get(blockHash, blockNumber); - if (header is not null && header.Number < _lowestRequestedHeader) - { - _lowestRequestedHeader = header.Number; - } - return header; + if (header.Number < _lowestRequestedHeader) _lowestRequestedHeader = header.Number; } - public IOwnedReadOnlyList GetWitnessHeaders(Hash256 parentHash) + public IOwnedReadOnlyList BuildHeaders(Hash256 parentHash, IHeaderFinder finder) { Hash256 currentHash = parentHash; - BlockHeader parentHeader = inner.Get(currentHash) ?? throw new ArgumentException($"Parent {currentHash} is not found"); + BlockHeader parentHeader = finder.Get(currentHash) ?? throw new ArgumentException($"Parent {currentHash} is not found"); // BLOCKHASH can only reach below the executed block, so a recorded header above the parent // means the bookkeeping is broken — fail loudly instead of computing a non-positive count. if (_lowestRequestedHeader < long.MaxValue && _lowestRequestedHeader > parentHeader.Number) + { throw new InvalidOperationException( $"Recorded header {_lowestRequestedHeader} is above the executed-against parent {parentHeader.Number}"); + } - // Headers in ascending block-number order — any BLOCKHASH-touched ancestor first, recorded - // block last — so the chain is contiguous and replayable. _lowestRequestedHeader stays at + // Headers in ascending block-number order — any BLOCKHASH-touched ancestor first, block being + // recorded last — so the chain is contiguous and replayable. _lowestRequestedHeader stays at // long.MaxValue unless BLOCKHASH reached further back during processing. int count = _lowestRequestedHeader < long.MaxValue ? (int)(parentHeader.Number - _lowestRequestedHeader + 1) : 1; int index = count - 1; - ArrayPoolList headers = new(count, count); + ArrayPoolList headers = new(capacity: count, count); try { headers[index--] = _decoder.Encode(parentHeader).Bytes; @@ -58,7 +66,7 @@ public IOwnedReadOnlyList GetWitnessHeaders(Hash256 parentHash) for (long i = parentHeader.Number - 1; i >= _lowestRequestedHeader; i--) { currentHash = parentHeader.ParentHash!; - parentHeader = inner.Get(currentHash, i) + parentHeader = finder.Get(currentHash, i) ?? throw new ArgumentException($"Unable to get requested header at hash {currentHash} and number {i} during witness generation"); headers[index--] = _decoder.Encode(parentHeader).Bytes; } diff --git a/src/Nethermind/Nethermind.Consensus/Stateless/WitnessRendezvous.cs b/src/Nethermind/Nethermind.Consensus/Stateless/WitnessRendezvous.cs new file mode 100644 index 000000000000..368e55735958 --- /dev/null +++ b/src/Nethermind/Nethermind.Consensus/Stateless/WitnessRendezvous.cs @@ -0,0 +1,74 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Concurrent; +using System.Threading.Tasks; +using Nethermind.Core.Crypto; +using Nethermind.Logging; + +namespace Nethermind.Consensus.Stateless; + +/// +/// Cross-thread coordination between the JSON-RPC handler that requests a witness for a block +/// hash and the block-processing thread that produces one. The handler awaits a ; +/// the processor completes it once the matching ProcessOne finishes. +/// +/// +/// No state-recording concerns live here; this type is purely a hash-keyed +/// registry. The recorder side ([[witness-capturing-block-processor]]) owns the recording lifecycle and +/// calls to publish a result. +/// +public sealed class WitnessRendezvous(ILogManager? logManager = null) +{ + private readonly ILogger _logger = (logManager ?? LimboLogs.Instance).GetClassLogger(); + private readonly ConcurrentDictionary> _pending = new(); + + /// + /// Handler-side: register a pending witness request for and return + /// a that completes when the block is processed (or is cancelled). + /// A duplicate request for the same hash cancels the previous task and replaces the entry. + /// + public Task RequestWitness(Hash256 blockHash) + { + // RunContinuationsAsynchronously: completion fires from the block-processing thread; we must + // not run the handler's continuation inline there. + TaskCompletionSource tcs = new(TaskCreationOptions.RunContinuationsAsynchronously); + TaskCompletionSource effective = _pending.AddOrUpdate( + blockHash, + tcs, + (_, existing) => + { + if (_logger.IsWarn) _logger.Warn($"{nameof(WitnessRendezvous)}: duplicate RequestWitness for {blockHash}. Replacing previous entry."); + existing.TrySetCanceled(); + return tcs; + }); + return effective.Task; + } + + /// True iff a witness has been requested for . + public bool HasPendingRequest(Hash256 blockHash) => _pending.ContainsKey(blockHash); + + /// + /// Handler-side: cancel a pending request (e.g. on the exception path before the processor drains). + /// No-op when no entry exists for . + /// + public void CancelWitnessRequest(Hash256 blockHash) + { + if (_pending.TryRemove(blockHash, out TaskCompletionSource? tcs)) + { + tcs.TrySetCanceled(); + if (_logger.IsTrace) _logger.Trace($"{nameof(WitnessRendezvous)}: capture cancelled for {blockHash}"); + } + } + + /// + /// Recorder-side: atomically remove and return the pending TCS for . + /// Returns false when no request is pending or the entry was already claimed/cancelled. + /// + /// + /// Two-step (claim + complete) rather than a single Complete(hash, witness) so the recorder + /// can avoid building the witness when the request was cancelled while processing. + /// + public bool TryClaim(Hash256 blockHash, out TaskCompletionSource? tcs) + => _pending.TryRemove(blockHash, out tcs); +} diff --git a/src/Nethermind/Nethermind.Consensus/Stateless/WitnessTrieNodeReadObserver.cs b/src/Nethermind/Nethermind.Consensus/Stateless/WitnessTrieNodeReadObserver.cs new file mode 100644 index 000000000000..ea079446c11d --- /dev/null +++ b/src/Nethermind/Nethermind.Consensus/Stateless/WitnessTrieNodeReadObserver.cs @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Crypto; +using Nethermind.Trie; + +namespace Nethermind.Consensus.Stateless; + +/// +/// Bridges onto the : trie +/// node reads observed on the live processing path (e.g. flat's commit-time merkleization) land on +/// the armed session's trie recorder. Inert when no capture is armed. +/// +public sealed class WitnessTrieNodeReadObserver(WitnessCaptureSession session) : ITrieNodeReadObserver +{ + public bool IsActive => session.TrieRecorder is not null; + + public void OnTrieNodeRead(Hash256 hash, byte[] rlp) => session.TrieRecorder?.Record(hash, rlp); +} diff --git a/src/Nethermind/Nethermind.Consensus/Stateless/WitnessTrieStoreRecorder.cs b/src/Nethermind/Nethermind.Consensus/Stateless/WitnessTrieStoreRecorder.cs new file mode 100644 index 000000000000..789290c120dc --- /dev/null +++ b/src/Nethermind/Nethermind.Consensus/Stateless/WitnessTrieStoreRecorder.cs @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using Nethermind.Core.Crypto; +using Nethermind.Trie; + +namespace Nethermind.Consensus.Stateless; + +/// +/// Per-capture collector for raw trie node RLP touched while a capture is armed. Populated by +/// on every resolved node read; drained by +/// when assembling the witness state nodes. +/// +public sealed class WitnessTrieStoreRecorder +{ + private readonly ConcurrentDictionary _rlpCollector = new(); + + /// Records an already-materialised node RLP (e.g. from a TryLoadRlp that paid the read). + public void Record(Hash256 hash, byte[] rlp) => _rlpCollector.TryAdd(hash, rlp); + + /// + /// Records a resolved node, materialising its RLP only on first capture. The static factory avoids + /// a per-call closure allocation, and only + /// invokes it when the key is absent — so repeat reads of the same node (hot in SLOAD loops) skip + /// the allocate-then-discard of node.FullRlp.ToArray(). + /// + public void Record(Hash256 hash, TrieNode node) + => _rlpCollector.GetOrAdd(hash, static (_, n) => n.FullRlp.ToArray()!, node); + + public IEnumerable TouchedNodesRlp => _rlpCollector.Select(static kvp => kvp.Value); + + /// Clears the captured-node set so the recorder can be reused across pooled env rents. + public void Reset() => _rlpCollector.Clear(); +} diff --git a/src/Nethermind/Nethermind.Init/Modules/BlockProcessingModule.cs b/src/Nethermind/Nethermind.Init/Modules/BlockProcessingModule.cs index df6617d8075c..4d3277524e07 100644 --- a/src/Nethermind/Nethermind.Init/Modules/BlockProcessingModule.cs +++ b/src/Nethermind/Nethermind.Init/Modules/BlockProcessingModule.cs @@ -14,6 +14,7 @@ using Nethermind.Consensus.Processing; using Nethermind.Consensus.Producers; using Nethermind.Consensus.Rewards; +using Nethermind.Consensus.Stateless; using Nethermind.Consensus.Tracing; using Nethermind.Consensus.Validators; using Nethermind.Consensus.Withdrawals; @@ -62,7 +63,18 @@ protected override void Load(ContainerBuilder builder) .AddScoped() .AddSingleton() .AddScoped() - .AddScoped() + .AddScoped(ctx => new BlockAccessListManager( + ctx.Resolve(), + ctx.Resolve(), + ctx.Resolve(), + ctx.Resolve(), + ctx.Resolve(), + ctx.Resolve(), + ctx.ResolveOptional(), + ctx.ResolveOptional(), + ctx.ResolveOptional(), + // Present only in witness-capable scopes (main pipeline, debug_executionWitness sandbox); + isWitnessExecution: ctx.ResolveOptional()?.IsActive)) .AddScoped() .AddScoped() .AddScoped((rewardSource, txP) => rewardSource.Get(txP)) diff --git a/src/Nethermind/Nethermind.Init/Modules/MainProcessingContext.cs b/src/Nethermind/Nethermind.Init/Modules/MainProcessingContext.cs index db56e0d59997..749ebca539c7 100644 --- a/src/Nethermind/Nethermind.Init/Modules/MainProcessingContext.cs +++ b/src/Nethermind/Nethermind.Init/Modules/MainProcessingContext.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Linq; using System.Threading.Tasks; using Autofac; using Nethermind.Api; @@ -46,9 +47,14 @@ public MainProcessingContext( builder // These are main block processing specific .AddSingleton(worldState) - .AddModule(blockValidationModules) + // Dedupe by type: a module's Load runs once per instance, and re-running a module that + // registers a decorator (e.g. WitnessCapturingMainProcessingModule's IWorldState proxy) + // double-decorates and forms a self-referential cycle. Duplicate instances arise when more + // than one module tree transitively pulls in the same module (e.g. both MergePluginModule + // and AuRaMergeModule add BaseMergePluginModule in aura tests). + .AddModule([.. blockValidationModules.DistinctBy(static m => m.GetType())]) .AddSingleton(this) - .AddModule(mainProcessingModules) + .AddModule([.. mainProcessingModules.DistinctBy(static m => m.GetType())]) .AddScoped((branchProcessor, processingStats) => new BlockchainProcessor( diff --git a/src/Nethermind/Nethermind.Init/Modules/PruningTrieStoreModule.cs b/src/Nethermind/Nethermind.Init/Modules/PruningTrieStoreModule.cs index d263174c786b..2b78f8470d2c 100644 --- a/src/Nethermind/Nethermind.Init/Modules/PruningTrieStoreModule.cs +++ b/src/Nethermind/Nethermind.Init/Modules/PruningTrieStoreModule.cs @@ -21,6 +21,7 @@ using Nethermind.Synchronization.SnapSync; using Nethermind.Synchronization.Trie; using Nethermind.Trie; +using Nethermind.Trie.Pruning; namespace Nethermind.Init.Modules; @@ -85,6 +86,25 @@ dbFactory is not MemDbFactory // Most config actually done in factory. We just call `Build` and then get back components from its output. .AddSingleton() // This part is done separately so that triestore can be obtained in test. + ; + + // The main-world trie store — what GlobalWorldState reads and writes through. Registered as + // a service (rather than built inline in PruningTrieStateFactory) so plugins can decorate it, + // e.g. the witness read-tap installed by the merge plugin on EIP-7928 chains. + // ExternallyOwned: PruningTrieStateFactory pushes it onto the dispose stack, which must stay + // the sole owner — TrieStore.Dispose persists the cache on shutdown and is not idempotent. + builder.Register(ctx => + { + ITrieStore store = ctx.Resolve().PruningTrieStore; + return ctx.ResolveOptional() is { } nodeStorageCache + ? new PreCachedTrieStore(store, nodeStorageCache) + : store; + }) + .As() + .SingleInstance() + .ExternallyOwned(); + + builder .AddSingleton() .AddSingleton() .AddSingleton() diff --git a/src/Nethermind/Nethermind.Init/PruningTrieStateFactory.cs b/src/Nethermind/Nethermind.Init/PruningTrieStateFactory.cs index 9e9c21489806..40b11dcb9694 100644 --- a/src/Nethermind/Nethermind.Init/PruningTrieStateFactory.cs +++ b/src/Nethermind/Nethermind.Init/PruningTrieStateFactory.cs @@ -31,14 +31,14 @@ public class PruningTrieStateFactory( IDbProvider dbProvider, IBlockTree blockTree, MainPruningTrieStoreFactory mainPruningTrieStoreFactory, + ITrieStore mainWorldTrieStore, INodeStorage mainNodeStorage, IProcessExitSource processExit, IDisposableStack disposeStack, IFullPrunerFactory fullPrunerFactory, CompositePruningTrigger compositePruningTrigger, Lazy pathRecovery, - ILogManager logManager, - NodeStorageCache? nodeStorageCache = null + ILogManager logManager ) { private readonly ILogger _logger = logManager.GetClassLogger(); @@ -47,13 +47,6 @@ public class PruningTrieStateFactory( { IPruningTrieStore trieStore = mainPruningTrieStoreFactory.PruningTrieStore; - ITrieStore mainWorldTrieStore = trieStore; - - if (nodeStorageCache is not null) - { - mainWorldTrieStore = new PreCachedTrieStore(mainWorldTrieStore, nodeStorageCache); - } - IKeyValueStoreWithBatching codeDb = dbProvider.CodeDb; IWorldStateScopeProvider scopeProvider = syncConfig.TrieHealing ? new HealingWorldStateScopeProvider( diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V1.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V1.cs index 768ffada62a2..22933926af33 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V1.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V1.cs @@ -2173,6 +2173,7 @@ public async Task Should_warn_for_missing_capabilities() SszRestPaths.PostV4Forkchoice, SszRestPaths.PostV2PayloadBodiesByHash, SszRestPaths.GetV2PayloadBodiesByRange, + SszRestCapabilities.NewPayloadWithWitness, ]; public static IEnumerable SszRestPathsAdvertisedCases() diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs index cb088e136db2..bfc16598030a 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs @@ -407,6 +407,7 @@ public async Task NewPayloadV3_should_verify_blob_versioned_hashes_again Substitute.For?>>(), Substitute.For, IReadOnlyList>>(), Substitute.For(), + Substitute.For(), Substitute.For(), chain.SpecProvider, new GCKeeper(NoGCStrategy.Instance, chain.LogManager), diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.WitnessCapture.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.WitnessCapture.cs new file mode 100644 index 000000000000..8cb39dcc99d4 --- /dev/null +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.WitnessCapture.cs @@ -0,0 +1,661 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Autofac; +using Nethermind.Consensus.Processing; +using Nethermind.Consensus.Producers; +using Nethermind.Consensus.Stateless; +using Nethermind.Core; +using Nethermind.Core.Collections; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using Nethermind.Core.Test.Builders; +using Nethermind.Evm.Tracing; +using Nethermind.Int256; +using Nethermind.JsonRpc; +using Nethermind.Merge.Plugin.Data; +using Nethermind.Merge.Plugin.Handlers; +using Nethermind.Specs.Forks; +using NSubstitute; +using NUnit.Framework; + +namespace Nethermind.Merge.Plugin.Test; + +public partial class EngineModuleTests +{ + private static Witness MakeStubWitness() => new() + { + State = new ArrayPoolList(1) { new byte[] { 0xDE, 0xAD } }, + Codes = new ArrayPoolList(0), + Keys = new ArrayPoolList(0), + Headers = new ArrayPoolList(0), + }; + + private sealed class WitnessHandlerBuilder + { + public IEngineRpcModule EngineModule { get; set; } + = SucceedingEngineModule(new PayloadStatusV1 { Status = PayloadStatus.Valid, LatestValidHash = TestItem.KeccakA }); + + public WitnessRendezvous Rendezvous { get; set; } = new(); + + public NewPayloadWithWitnessHandler Build() => + new(new Lazy(() => EngineModule), Rendezvous); + + public static IEngineRpcModule SucceedingEngineModule(PayloadStatusV1 status) + { + IEngineRpcModule module = Substitute.For(); + module + .engine_newPayloadV5(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(ResultWrapper.Success(status)); + return module; + } + + public static IEngineRpcModule FailingEngineModule(string error, int errorCode) + { + IEngineRpcModule module = Substitute.For(); + module + .engine_newPayloadV5(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(ResultWrapper.Fail(error, errorCode)); + return module; + } + } + + [Test] + [Category("WitnessCapture")] + public void Rendezvous_RequestWitness_returns_incomplete_task_until_completed() + { + WitnessRendezvous rendezvous = new(); + + Task task = rendezvous.RequestWitness(TestItem.KeccakA); + + Assert.That(task.IsCompleted, Is.False, + "the task must remain pending until the block-processor decorator publishes a result"); + } + + [Test] + [Category("WitnessCapture")] + public void Rendezvous_CancelWitnessRequest_cancels_TCS_and_removes_entry() + { + WitnessRendezvous rendezvous = new(); + Hash256 hash = TestItem.KeccakD; + + Task captureTask = rendezvous.RequestWitness(hash); + Assert.That(rendezvous.HasPendingRequest(hash), Is.True); + + rendezvous.CancelWitnessRequest(hash); + + Assert.That(rendezvous.HasPendingRequest(hash), Is.False, + "CancelWitnessRequest must remove the entry"); + Assert.That(captureTask.IsCanceled, Is.True, + "CancelWitnessRequest must cancel the TCS so any awaiter gets OperationCanceledException"); + } + + [Test] + [Category("WitnessCapture")] + public void Rendezvous_CancelWitnessRequest_noop_when_no_entry_exists() + { + WitnessRendezvous rendezvous = new(); + Action cancel = () => rendezvous.CancelWitnessRequest(Keccak.Zero); + Assert.That(cancel, Throws.Nothing, "cancelling a non-existent request is a valid no-op"); + } + + [Test] + [Category("WitnessCapture")] + public void Rendezvous_duplicate_RequestWitness_cancels_previous_TCS() + { + WitnessRendezvous rendezvous = new(); + Hash256 hash = TestItem.KeccakE; + + Task first = rendezvous.RequestWitness(hash); + Task second = rendezvous.RequestWitness(hash); + + Assert.That(first.IsCanceled, Is.True, + "the orphaned TCS must be cancelled so any awaiter gets OperationCanceledException rather than hanging forever"); + Assert.That(second.IsCompleted, Is.False, "the replacement TCS is still pending"); + } + + [Test] + [Category("WitnessCapture")] + public async Task BlockProcessor_completes_rendezvous_task_synchronously_inside_newPayloadV5() + { + using MergeTestBlockchain chain = await CreateBlockchain(Amsterdam.Instance); + WitnessRendezvous rendezvous = chain.Container.Resolve(); + + (ExecutionPayloadV4 payload, byte[][]? requests) = await BuildAmsterdamPayload(chain); + Hash256 hash = payload.BlockHash!; + + Task captureTask = rendezvous.RequestWitness(hash); + + await chain.EngineRpcModule.engine_newPayloadV5(payload, [], TestItem.KeccakE, requests ?? []); + + Assert.That(captureTask.IsCompleted, Is.True, + "the block-processor decorator must complete the TCS synchronously inside ProcessOne, " + + "before engine_newPayloadV5 returns, so the handler's await is a non-blocking retrieval"); + + using Witness? witness = await captureTask; + Assert.That(witness, Is.Not.Null, "a VALID block must produce a non-null witness"); + } + + [Test] + [Category("WitnessCapture")] + public async Task BlockProcessor_does_not_capture_when_no_request_pending() + { + using MergeTestBlockchain chain = await CreateBlockchain(Amsterdam.Instance); + WitnessRendezvous rendezvous = chain.Container.Resolve(); + + (ExecutionPayloadV4 payload, byte[][]? requests) = await BuildAmsterdamPayload(chain); + + await chain.EngineRpcModule.engine_newPayloadV5(payload, [], TestItem.KeccakE, requests ?? []); + + Assert.That(rendezvous.HasPendingRequest(payload.BlockHash!), Is.False, + "no entry should appear in the rendezvous for a plain engine_newPayloadV5 call"); + } + + [Test] + [Category("WitnessCapture")] + public async Task BlockProcessor_multi_block_branch_captures_independent_witnesses() + { + using MergeTestBlockchain chain = await CreateBlockchain(Amsterdam.Instance); + IEngineRpcModule rpc = chain.EngineRpcModule; + WitnessRendezvous rendezvous = chain.Container.Resolve(); + + (ExecutionPayloadV4 p1, byte[][]? r1) = await BuildAmsterdamPayload(chain); + Task t1 = rendezvous.RequestWitness(p1.BlockHash!); + await rpc.engine_newPayloadV5(p1, [], TestItem.KeccakE, r1 ?? []); + await rpc.engine_forkchoiceUpdatedV4( + new ForkchoiceStateV1(p1.BlockHash!, p1.BlockHash!, p1.BlockHash!), null); + (await t1)?.Dispose(); + + (ExecutionPayloadV4 p2, byte[][]? r2) = await BuildAmsterdamPayload(chain); + Task t2 = rendezvous.RequestWitness(p2.BlockHash!); + await rpc.engine_newPayloadV5(p2, [], TestItem.KeccakE, r2 ?? []); + + Assert.That(t1.IsCompletedSuccessfully, Is.True, "block-1 task was completed during block-1"); + Assert.That(t2.IsCompletedSuccessfully, Is.True, "block-2 task must be completed during block-2"); + + using Witness? w2 = await t2; + Assert.That(w2, Is.Not.Null, "block 2 must produce a valid witness"); + } + + [Test] + [Category("WitnessCapture")] + public async Task BlockProcessor_uncaptured_block_between_two_captured_blocks_leaves_clean_state() + { + using MergeTestBlockchain chain = await CreateBlockchain(Amsterdam.Instance); + IEngineRpcModule rpc = chain.EngineRpcModule; + WitnessRendezvous rendezvous = chain.Container.Resolve(); + + (ExecutionPayloadV4 p1, byte[][]? r1) = await BuildAmsterdamPayload(chain); + Task t1 = rendezvous.RequestWitness(p1.BlockHash!); + await rpc.engine_newPayloadV5(p1, [], TestItem.KeccakE, r1 ?? []); + await rpc.engine_forkchoiceUpdatedV4(new ForkchoiceStateV1(p1.BlockHash!, p1.BlockHash!, p1.BlockHash!), null); + (await t1)?.Dispose(); + + (ExecutionPayloadV4 p2, byte[][]? r2) = await BuildAmsterdamPayload(chain); + await rpc.engine_newPayloadV5(p2, [], TestItem.KeccakE, r2 ?? []); + await rpc.engine_forkchoiceUpdatedV4(new ForkchoiceStateV1(p2.BlockHash!, p2.BlockHash!, p2.BlockHash!), null); + + (ExecutionPayloadV4 p3, byte[][]? r3) = await BuildAmsterdamPayload(chain); + Task t3 = rendezvous.RequestWitness(p3.BlockHash!); + await rpc.engine_newPayloadV5(p3, [], TestItem.KeccakE, r3 ?? []); + + Assert.That(t3.IsCompletedSuccessfully, Is.True, + "an armed capture for block 3 must succeed even after an uncaptured block 2"); + using Witness? w3 = await t3; + Assert.That(w3, Is.Not.Null, "block 3 must produce a valid witness"); + } + + /// + /// Builds an IEngineRpcModule mock whose engine_newPayloadV5 implementation simulates what the + /// WitnessCapturingBlockProcessor decorator does on the real path: claim the pending rendezvous + /// entry for the requested block hash and publish into it. + /// + private static IEngineRpcModule PublishingEngineModule(WitnessRendezvous rendezvous, Witness? witness, PayloadStatusV1 status) + { + IEngineRpcModule module = Substitute.For(); + module + .engine_newPayloadV5(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(call => + { + ExecutionPayloadV4 payload = call.Arg(); + if (rendezvous.TryClaim(payload.BlockHash!, out TaskCompletionSource? tcs)) + tcs!.SetResult(witness); + return ResultWrapper.Success(status); + }); + return module; + } + + [Test] + [Category("WitnessCapture")] + public async Task Handler_returns_witness_from_rendezvous_on_valid_status() + { + using Witness expectedWitness = MakeStubWitness(); + WitnessRendezvous rendezvous = new(); + + NewPayloadWithWitnessHandler handler = new( + new Lazy(() => PublishingEngineModule( + rendezvous, + expectedWitness, + new PayloadStatusV1 { Status = PayloadStatus.Valid, LatestValidHash = TestItem.KeccakA })), + rendezvous); + + ResultWrapper result = + await handler.HandleAsync(new ExecutionPayloadV4 { BlockHash = TestItem.KeccakA }, [], TestItem.KeccakA, []); + + Assert.That(result.Result.ResultType, Is.EqualTo(ResultType.Success)); + Assert.That(result.Data.Status, Is.EqualTo(PayloadStatus.Valid)); + Assert.That(result.Data.ExecutionWitness, Is.SameAs(expectedWitness)); + } + + [Test] + [Category("WitnessCapture")] + public async Task Handler_valid_status_with_null_witness_yields_null_witness() + { + WitnessRendezvous rendezvous = new(); + + NewPayloadWithWitnessHandler handler = new( + new Lazy(() => PublishingEngineModule( + rendezvous, + witness: null, + new PayloadStatusV1 { Status = PayloadStatus.Valid, LatestValidHash = TestItem.KeccakB })), + rendezvous); + + ResultWrapper result = + await handler.HandleAsync(new ExecutionPayloadV4 { BlockHash = TestItem.KeccakA }, [], TestItem.KeccakA, []); + + Assert.That(result.Result.ResultType, Is.EqualTo(ResultType.Success)); + Assert.That(result.Data.Status, Is.EqualTo(PayloadStatus.Valid)); + Assert.That(result.Data.ExecutionWitness, Is.Null); + } + + private static IEnumerable NonValidOutcomes() + { + yield return new TestCaseData((Func)(() => WitnessHandlerBuilder.SucceedingEngineModule( + new PayloadStatusV1 { Status = PayloadStatus.Syncing }))) + .SetName("SYNCING status"); + yield return new TestCaseData((Func)(() => WitnessHandlerBuilder.SucceedingEngineModule( + new PayloadStatusV1 { Status = PayloadStatus.Invalid, LatestValidHash = TestItem.KeccakD, ValidationError = "bad block" }))) + .SetName("INVALID status"); + yield return new TestCaseData((Func)(() => WitnessHandlerBuilder.FailingEngineModule( + "Unsupported fork", MergeErrorCodes.UnsupportedFork))) + .SetName("RPC failure"); + } + + [TestCaseSource(nameof(NonValidOutcomes))] + [Category("WitnessCapture")] + public async Task Handler_cancels_rendezvous_when_not_valid(Func moduleFactory) + { + WitnessRendezvous rendezvous = new(); + + NewPayloadWithWitnessHandler handler = new(new Lazy(moduleFactory), rendezvous); + + await handler.HandleAsync(new ExecutionPayloadV4 { BlockHash = TestItem.KeccakA }, [], TestItem.KeccakA, []); + + Assert.That(rendezvous.HasPendingRequest(TestItem.KeccakA), Is.False, + "the handler must cancel the rendezvous entry on every non-VALID outcome"); + } + + [Test] + [Category("WitnessCapture")] + public async Task Handler_rejects_null_blockHash_with_InvalidParams_and_does_not_register() + { + WitnessRendezvous rendezvous = new(); + + NewPayloadWithWitnessHandler handler = new( + new Lazy(() => WitnessHandlerBuilder.SucceedingEngineModule( + new PayloadStatusV1 { Status = PayloadStatus.Valid, LatestValidHash = TestItem.KeccakA })), + rendezvous); + + ExecutionPayloadV4 payload = new() + { + BlockHash = null! + }; + ResultWrapper result = + await handler.HandleAsync(payload, [], TestItem.KeccakA, []); + + Assert.That(result.Result.ResultType, Is.EqualTo(ResultType.Failure), + "a null blockHash is a malformed payload — return InvalidParams instead of forwarding"); + Assert.That(result.ErrorCode, Is.EqualTo(ErrorCodes.InvalidParams)); + } + + [Test] + [Category("WitnessCapture")] + public async Task E2E_empty_Amsterdam_block_produces_VALID_with_non_null_witness() + { + using MergeTestBlockchain chain = await CreateBlockchain(Amsterdam.Instance); + + (ExecutionPayloadV4 payload, byte[][]? requests) = await BuildAmsterdamPayload(chain); + + ResultWrapper result = + await chain.EngineRpcModule.engine_newPayloadWithWitness( + payload, [], TestItem.KeccakE, requests ?? []); + + Assert.That(result.Result.ResultType, Is.EqualTo(ResultType.Success)); + Assert.That(result.Data.Status, Is.EqualTo(PayloadStatus.Valid)); + + using Witness? witness = result.Data.ExecutionWitness; + Assert.That(witness, Is.Not.Null, "VALID block must include a witness"); + Assert.That(witness!.State.Count, Is.GreaterThan(0), + "witness State must contain at least the state root proof node"); + } + + [Test] + [Category("WitnessCapture")] + public async Task E2E_block_with_ETH_transfer_produces_multi_node_witness() + { + using MergeTestBlockchain chain = await CreateBlockchain(Amsterdam.Instance); + + Transaction tx = Build.A.Transaction + .WithValue(UInt256.One) + .WithTo(TestItem.AddressB) + .WithMaxFeePerGas(20.GWei) + .WithMaxPriorityFeePerGas(1.GWei) + .WithType(TxType.EIP1559) + .SignedAndResolved(chain.EthereumEcdsa, TestItem.PrivateKeyA) + .TestObject; + chain.AddTransactions(tx); + + (ExecutionPayloadV4 payload, byte[][]? requests) = await BuildAmsterdamPayload(chain); + + ResultWrapper result = + await chain.EngineRpcModule.engine_newPayloadWithWitness( + payload, [], TestItem.KeccakE, requests ?? []); + + Assert.That(result.Data.Status, Is.EqualTo(PayloadStatus.Valid)); + using Witness? witness = result.Data.ExecutionWitness; + Assert.That(witness, Is.Not.Null); + Assert.That(witness!.State.Count, Is.GreaterThan(1), + "a transfer touches sender, recipient and fee-recipient: at least 2 proof paths"); + } + + [Test] + [Category("WitnessCapture")] + public async Task E2E_witness_state_nodes_satisfy_spec_size_constraints() + { + using MergeTestBlockchain chain = await CreateBlockchain(Amsterdam.Instance); + + (ExecutionPayloadV4 payload, byte[][]? requests) = await BuildAmsterdamPayload(chain); + + ResultWrapper result = + await chain.EngineRpcModule.engine_newPayloadWithWitness( + payload, [], TestItem.KeccakE, requests ?? []); + + using Witness? witness = result.Data.ExecutionWitness; + Assert.That(witness, Is.Not.Null); + + foreach (byte[] node in witness!.State) + { + Assert.That(node, Is.Not.Empty, "every state node must be a non-empty RLP blob"); + Assert.That(node.Length, Is.LessThanOrEqualTo(1_048_576), + "each state element must not exceed MAX_WITNESS_ITEM_BYTES"); + } + + Assert.That(witness.State.Count, Is.LessThanOrEqualTo(1_048_576), + "State list must not exceed MAX_WITNESS_ITEMS"); + } + + [Test] + [Category("WitnessCapture")] + public async Task E2E_sequential_blocks_produce_independent_witness_instances() + { + using MergeTestBlockchain chain = await CreateBlockchain(Amsterdam.Instance); + IEngineRpcModule rpc = chain.EngineRpcModule; + + (ExecutionPayloadV4 p1, byte[][]? r1) = await BuildAmsterdamPayload(chain); + ResultWrapper res1 = + await rpc.engine_newPayloadWithWitness(p1, [], TestItem.KeccakE, r1 ?? []); + await rpc.engine_forkchoiceUpdatedV4( + new ForkchoiceStateV1(p1.BlockHash!, p1.BlockHash!, p1.BlockHash!), null); + + (ExecutionPayloadV4 p2, byte[][]? r2) = await BuildAmsterdamPayload(chain); + ResultWrapper res2 = + await rpc.engine_newPayloadWithWitness(p2, [], TestItem.KeccakE, r2 ?? []); + + Assert.That(res1.Data.Status, Is.EqualTo(PayloadStatus.Valid)); + Assert.That(res2.Data.Status, Is.EqualTo(PayloadStatus.Valid)); + + using Witness? w1 = res1.Data.ExecutionWitness; + using Witness? w2 = res2.Data.ExecutionWitness; + + Assert.That(w1, Is.Not.Null); + Assert.That(w2, Is.Not.Null); + Assert.That(w1, Is.Not.SameAs(w2), + "each block produces its own Witness instance; shared reference indicates a tracking bug"); + } + + [Test] + [Category("WitnessCapture")] + public async Task E2E_non_VALID_response_has_null_witness_and_no_rendezvous_leak() + { + using MergeTestBlockchain chain = await CreateBlockchain(Amsterdam.Instance); + WitnessRendezvous rendezvous = chain.Container.Resolve(); + + (ExecutionPayloadV4 good, byte[][]? requests) = await BuildAmsterdamPayload(chain); + ExecutionPayloadV4 bad = new() + { + BlockHash = Keccak.Zero, + ParentHash = good.ParentHash, + FeeRecipient = good.FeeRecipient, + StateRoot = good.StateRoot, + ReceiptsRoot = good.ReceiptsRoot, + LogsBloom = good.LogsBloom, + PrevRandao = good.PrevRandao, + BlockNumber = good.BlockNumber, + GasLimit = good.GasLimit, + GasUsed = good.GasUsed, + Timestamp = good.Timestamp, + ExtraData = good.ExtraData, + BaseFeePerGas = good.BaseFeePerGas, + Transactions = good.Transactions, + Withdrawals = good.Withdrawals, + BlobGasUsed = good.BlobGasUsed, + ExcessBlobGas = good.ExcessBlobGas, + ParentBeaconBlockRoot = good.ParentBeaconBlockRoot, + ExecutionRequests = good.ExecutionRequests, + BlockAccessList = good.BlockAccessList, + SlotNumber = good.SlotNumber, + }; + + ResultWrapper result = + await chain.EngineRpcModule.engine_newPayloadWithWitness(bad, [], TestItem.KeccakE, requests ?? []); + + Assert.That(result.Result.ResultType, Is.EqualTo(ResultType.Success), + "non-VALID status must still yield HTTP 200 / RPC success per the spec"); + Assert.That(result.Data.Status, Is.Not.EqualTo(PayloadStatus.Valid)); + Assert.That(result.Data.ExecutionWitness, Is.Null, + "spec: witness must be None when status is not VALID"); + + Assert.That(rendezvous.HasPendingRequest(Keccak.Zero), Is.False, + "the handler must cancel the rendezvous entry on non-VALID outcomes"); + } + + [Test] + [Category("WitnessCapture")] + public async Task Regression_ProcessOne_called_exactly_once_during_engine_newPayloadWithWitness() + { + int processCount = 0; + + using MergeTestBlockchain chain = await CreateBlockchain( + Amsterdam.Instance, + configurer: builder => + builder.AddDecorator((_, inner) => + new CountingBranchProcessorDecorator(inner, () => Interlocked.Increment(ref processCount)))); + + (ExecutionPayloadV4 payload, byte[][]? requests) = await BuildAmsterdamPayload(chain); + processCount = 0; + + ResultWrapper result = + await chain.EngineRpcModule.engine_newPayloadWithWitness( + payload, [], TestItem.KeccakE, requests ?? []); + + Assert.That(result.Data.Status, Is.EqualTo(PayloadStatus.Valid)); + Assert.That(result.Data.ExecutionWitness, Is.Not.Null); + + Assert.That(processCount, Is.EqualTo(1), + "Option A must execute the block exactly once; " + + "a count of 2 means the old double-execution bug has regressed"); + } + + [Test] + [Category("WitnessCapture")] + public async Task Regression_plain_engine_newPayloadV5_unaffected_by_witness_infrastructure() + { + using MergeTestBlockchain chain = await CreateBlockchain(Amsterdam.Instance); + WitnessRendezvous rendezvous = chain.Container.Resolve(); + + (ExecutionPayloadV4 payload, byte[][]? requests) = await BuildAmsterdamPayload(chain); + Hash256 hash = payload.BlockHash!; + + ResultWrapper result = + await chain.EngineRpcModule.engine_newPayloadV5(payload, [], TestItem.KeccakE, requests ?? []); + + Assert.That(result.Data.Status, Is.EqualTo(PayloadStatus.Valid), + "the witness infrastructure must be completely transparent to the normal path"); + Assert.That(rendezvous.HasPendingRequest(hash), Is.False, + "no rendezvous entry should exist for a plain engine_newPayloadV5 call"); + } + + [Test] + [Category("WitnessCapture")] + public async Task Witness_state_nodes_are_consistent_with_parent_state_root() + { + using MergeTestBlockchain chain = await CreateBlockchain(Amsterdam.Instance); + + Transaction tx = Build.A.Transaction + .WithValue(UInt256.One) + .WithTo(TestItem.AddressB) + .WithMaxFeePerGas(20.GWei) + .WithMaxPriorityFeePerGas(1.GWei) + .WithType(TxType.EIP1559) + .SignedAndResolved(chain.EthereumEcdsa, TestItem.PrivateKeyA) + .TestObject; + chain.AddTransactions(tx); + + (ExecutionPayloadV4 payload, byte[][]? requests) = await BuildAmsterdamPayload(chain); + ResultWrapper result = + await chain.EngineRpcModule.engine_newPayloadWithWitness( + payload, [], TestItem.KeccakE, requests ?? []); + + Assert.That(result.Data.Status, Is.EqualTo(PayloadStatus.Valid)); + using Witness? witness = result.Data.ExecutionWitness; + Assert.That(witness, Is.Not.Null); + + foreach (byte[] node in witness!.State) + { + Assert.That(node.Length, Is.GreaterThanOrEqualTo(1), + "an empty node indicates drain ran before CommitTree populated the trie cache"); + } + } + + [Test] + [Category("WitnessCapture")] + public async Task Witness_headers_contain_at_least_parent_header() + { + using MergeTestBlockchain chain = await CreateBlockchain(Amsterdam.Instance); + + (ExecutionPayloadV4 payload, byte[][]? requests) = await BuildAmsterdamPayload(chain); + ResultWrapper result = + await chain.EngineRpcModule.engine_newPayloadWithWitness( + payload, [], TestItem.KeccakE, requests ?? []); + + Assert.That(result.Data.Status, Is.EqualTo(PayloadStatus.Valid)); + using Witness? witness = result.Data.ExecutionWitness; + Assert.That(witness, Is.Not.Null); + + Assert.That(witness!.Headers.Count, Is.GreaterThanOrEqualTo(1), + "Witness.Headers must contain at least the parent block header " + + "(WitnessHeaderRecorder.BuildHeaders always includes parentHash)."); + } + + [Test] + [Category("WitnessCapture")] + public async Task Witness_headers_items_are_valid_RLP_encoded_block_headers() + { + using MergeTestBlockchain chain = await CreateBlockchain(Amsterdam.Instance); + + (ExecutionPayloadV4 payload, byte[][]? requests) = await BuildAmsterdamPayload(chain); + ResultWrapper result = + await chain.EngineRpcModule.engine_newPayloadWithWitness( + payload, [], TestItem.KeccakE, requests ?? []); + + using Witness? witness = result.Data.ExecutionWitness; + Assert.That(witness, Is.Not.Null); + + foreach (byte[] header in witness!.Headers) + { + Assert.That(header, Is.Not.Empty, "each header entry must be an RLP-encoded block header"); + Assert.That(header.Length, Is.LessThanOrEqualTo(1_048_576), + "each header must fit within MAX_WITNESS_ITEM_BYTES per execution-apis#773"); + } + } + + private static async Task<(ExecutionPayloadV4 Payload, byte[][]? ExecutionRequests)> + BuildAmsterdamPayload(MergeTestBlockchain chain) + { + IEngineRpcModule rpc = chain.EngineRpcModule; + Block head = chain.BlockTree.Head!; + + PayloadAttributes attributes = new() + { + Timestamp = head.Timestamp + 1, + PrevRandao = TestItem.KeccakH, + SuggestedFeeRecipient = TestItem.AddressF, + Withdrawals = [], + ParentBeaconBlockRoot = TestItem.KeccakE, + SlotNumber = (ulong?)(head.Number + 1), + }; + + Hash256 headHash = head.Hash!; + ForkchoiceStateV1 fcu = new(headHash, headHash, headHash); + + Task improvementWait = chain.WaitForImprovedBlock(headHash); + ResultWrapper fcuResult = + await rpc.engine_forkchoiceUpdatedV4(fcu, attributes); + Assert.That(fcuResult.Result.ResultType, Is.EqualTo(ResultType.Success)); + + await improvementWait; + + byte[] payloadIdBytes = Nethermind.Core.Extensions.Bytes.FromHexString(fcuResult.Data.PayloadId!); + ResultWrapper getPayload = await rpc.engine_getPayloadV6(payloadIdBytes); + Assert.That(getPayload.Data, Is.Not.Null); + + return (getPayload.Data!.ExecutionPayload, getPayload.Data!.ExecutionRequests); + } + + private sealed class CountingBranchProcessorDecorator(IBranchProcessor inner, Action onProcess) + : IBranchProcessor + { + public event EventHandler? BlockProcessed + { + add => inner.BlockProcessed += value; + remove => inner.BlockProcessed -= value; + } + + public event EventHandler? BlocksProcessing + { + add => inner.BlocksProcessing += value; + remove => inner.BlocksProcessing -= value; + } + + public event EventHandler? BlockProcessing + { + add => inner.BlockProcessing += value; + remove => inner.BlockProcessing -= value; + } + + public Block[] Process( + BlockHeader? baseBlock, + IReadOnlyList suggestedBlocks, + ProcessingOptions processingOptions, + IBlockTracer blockTracer, + CancellationToken token = default) + { + onProcess(); + return inner.Process(baseBlock, suggestedBlocks, processingOptions, blockTracer, token); + } + } +} diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/SszRest/SszCodecTests.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/SszRest/SszCodecTests.cs index 68ad10f20b8a..d4daec19da04 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/SszRest/SszCodecTests.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/SszRest/SszCodecTests.cs @@ -9,9 +9,11 @@ using Nethermind.Core.Test.Builders; using Nethermind.Int256; using Nethermind.Consensus.Producers; +using Nethermind.Consensus.Stateless; using Nethermind.Merge.Plugin.Data; using Nethermind.Merge.Plugin.SszRest; using NUnit.Framework; +using System.Buffers.Binary; namespace Nethermind.Merge.Plugin.Test.SszRest; @@ -682,4 +684,133 @@ public void SszKzgCommitment_list_roundtrip_preserves_raw_bytes() for (int i = 0; i < proofs.Length; i++) Assert.That(decoded.Commitments![i].AsSpan().ToArray(), Is.EqualTo(proofs[i]), $"commitment {i} bytes must round-trip exactly"); } + + // Witness Union is Some iff status is VALID AND witness is non-null; otherwise None. + [TestCase(PayloadStatus.Valid, true, true)] + [TestCase(PayloadStatus.Valid, false, false)] + [TestCase(PayloadStatus.Invalid, true, false)] + [TestCase(PayloadStatus.Syncing, true, false)] + [TestCase(PayloadStatus.Accepted, true, false)] + public void EncodeNewPayloadWithWitnessResponse_witness_union_presence(string status, bool hasWitness, bool expectedPresent) + { + using Witness? witness = hasWitness ? MakeMinimalWitness() : null; + PayloadStatusV1 ps = new() { Status = status }; + + byte[] encoded = Encode( + (ps, witness), + static (t, w) => SszCodec.EncodeNewPayloadWithWitnessResponse(t.Item1, t.Item2, w)); + + (_, _, bool witnessPresent) = SszCodec.DecodeNewPayloadWithWitnessResponse(encoded); + Assert.That(witnessPresent, Is.EqualTo(expectedPresent)); + } + + [Test] + public void EncodeNewPayloadWithWitnessResponse_container_header_is_13_bytes_and_offsets_are_correct() + { + PayloadStatusV1 ps = new() { Status = PayloadStatus.Valid, LatestValidHash = TestItem.KeccakA }; + + byte[] encoded = Encode( + (ps, (Witness?)null), + static (t, w) => SszCodec.EncodeNewPayloadWithWitnessResponse(t.Item1, t.Item2, w)); + + ReadOnlySpan buf = encoded; + + Assert.That(buf[0], Is.EqualTo(0), "VALID encodes as status byte 0x00"); + + int off1 = BinaryPrimitives.ReadInt32LittleEndian(buf.Slice(1, 4)); + Assert.That(off1, Is.EqualTo(13), "latest_valid_hash Union starts immediately after the 13-byte fixed header"); + + Assert.That(buf[off1], Is.EqualTo(0x01), "latest_valid_hash Union selector must be 0x01 (Some) when hash is present"); + Assert.That(buf.Slice(off1 + 1, 32).ToArray(), Is.EqualTo(TestItem.KeccakA.Bytes.ToArray()), + "latest_valid_hash bytes must follow immediately after the 0x01 selector"); + + int off2 = BinaryPrimitives.ReadInt32LittleEndian(buf.Slice(5, 4)); + Assert.That(off2, Is.EqualTo(46), "validation_error Union starts after latest_valid_hash (13 header + 33 lvh bytes)"); + Assert.That(buf[off2], Is.EqualTo(0x00), "validation_error Union selector must be 0x00 (None) when no error"); + + int off3 = BinaryPrimitives.ReadInt32LittleEndian(buf.Slice(9, 4)); + Assert.That(off3, Is.EqualTo(47), "witness Union starts after validation_error (46 + 1 None byte)"); + Assert.That(buf[off3], Is.EqualTo(0x00), "witness Union selector must be 0x00 (None) when no witness was generated"); + } + + [Test] + public void EncodeNewPayloadWithWitnessResponse_ssz_golden_byte_roundtrip() + { + // Fixture-representative witness data: two trie nodes, one code item, one header. + byte[] stateNode1 = [0xf8, 0x44, 0x01, 0x02, 0x03]; + byte[] stateNode2 = [0xe2, 0x80, 0xa0, 0xaa, 0xbb]; + byte[] codeItem = [0x60, 0x01, 0x60, 0x00, 0x52]; + byte[] headerBlob = [0xf9, 0x02, 0x18, 0x01, 0x02]; + + using Witness witness = new() + { + State = new Core.Collections.ArrayPoolList(2) { stateNode1, stateNode2 }, + Codes = new Core.Collections.ArrayPoolList(1) { codeItem }, + Headers = new Core.Collections.ArrayPoolList(1) { headerBlob }, + Keys = new Core.Collections.ArrayPoolList(0), + }; + + Hash256 latestValidHash = TestItem.KeccakB; + PayloadStatusV1 ps = new() + { + Status = PayloadStatus.Valid, + LatestValidHash = latestValidHash, + }; + + byte[] encoded = Encode( + (ps, witness), + static (t, w) => SszCodec.EncodeNewPayloadWithWitnessResponse(t.Item1, t.Item2, w)); + + ReadOnlySpan buf = encoded; + + Assert.That(buf[0], Is.EqualTo(0x00), "VALID encodes as status byte 0x00"); + + int offLvh = BinaryPrimitives.ReadInt32LittleEndian(buf.Slice(1, 4)); + Assert.That(offLvh, Is.EqualTo(13), "latest_valid_hash offset must be 13 (right after the fixed header)"); + Assert.That(buf[offLvh], Is.EqualTo(0x01), "latest_valid_hash selector must be 0x01 (Some)"); + Assert.That(buf.Slice(offLvh + 1, 32).ToArray(), Is.EqualTo(latestValidHash.Bytes.ToArray()), + "latest_valid_hash bytes must match"); + + int offVe = BinaryPrimitives.ReadInt32LittleEndian(buf.Slice(5, 4)); + Assert.That(offVe, Is.EqualTo(46), "validation_error offset must be 46 (13 header + 33 Some-hash bytes)"); + Assert.That(buf[offVe], Is.EqualTo(0x00), "validation_error selector must be 0x00 (None) when absent"); + + int offW = BinaryPrimitives.ReadInt32LittleEndian(buf.Slice(9, 4)); + Assert.That(offW, Is.EqualTo(47), "witness offset must be 47 (46 + 1 None byte for validation_error)"); + + (byte decodedStatusByte, Hash256? decodedLvh, bool witnessPresent) = + SszCodec.DecodeNewPayloadWithWitnessResponse(encoded); + + Assert.That(decodedStatusByte, Is.EqualTo(0x00), "decoded status byte must be 0x00 (VALID)"); + Assert.That(decodedLvh, Is.Not.Null, "decoded latest_valid_hash must be present"); + Assert.That(decodedLvh!.Bytes.ToArray(), Is.EqualTo(latestValidHash.Bytes.ToArray()), + "decoded hash must match the original"); + Assert.That(witnessPresent, Is.True, "VALID + witness bytes => witness union must be Some"); + } + + [Test] + public void EncodeNewPayloadWithWitnessResponse_invalid_status_suppresses_witness() + { + using Witness witness = MakeMinimalWitness(); + PayloadStatusV1 ps = new() { Status = PayloadStatus.Invalid }; + + byte[] encoded = Encode( + (ps, witness), + static (t, w) => SszCodec.EncodeNewPayloadWithWitnessResponse(t.Item1, t.Item2, w)); + + (byte decodedStatusByte, _, bool witnessPresent) = + SszCodec.DecodeNewPayloadWithWitnessResponse(encoded); + + Assert.That(decodedStatusByte, Is.EqualTo(0x01), "INVALID encodes as status byte 0x01"); + Assert.That(witnessPresent, Is.False, + "INVALID status must not carry a witness even when one was passed to the encoder"); + } + + private static Witness MakeMinimalWitness() => new() + { + State = new Core.Collections.ArrayPoolList(0), + Codes = new Core.Collections.ArrayPoolList(0), + Keys = new Core.Collections.ArrayPoolList(0), + Headers = new Core.Collections.ArrayPoolList(0), + }; } diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/SszRest/SszMiddlewareTests.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/SszRest/SszMiddlewareTests.cs index 2d2e8aecdf7f..42214992b6a0 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/SszRest/SszMiddlewareTests.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/SszRest/SszMiddlewareTests.cs @@ -10,7 +10,9 @@ using Microsoft.AspNetCore.Http; using Nethermind.Config; using Nethermind.Consensus.Producers; +using Nethermind.Consensus.Stateless; using Nethermind.Core; +using Nethermind.Core.Collections; using Nethermind.Core.Authentication; using Nethermind.Core.Crypto; using Nethermind.Core.Specs; @@ -22,6 +24,8 @@ using Nethermind.Merge.Plugin.Data; using Nethermind.Merge.Plugin.SszRest; using Nethermind.Merge.Plugin.SszRest.Handlers; +using Nethermind.Serialization.Rlp; +using Nethermind.Serialization.Rlp.Eip7928; using Nethermind.Specs.Forks; using System.Linq; using NSubstitute; @@ -114,11 +118,14 @@ private SszMiddleware BuildMiddleware(RequestDelegate? next = null) new CapabilitiesSszHandler(_specProvider), ]; + NewPayloadWithWitnessSszHandler witness = new(_engineModule); + return new SszMiddleware( passthrough, _urlCollection, _auth, handlers, + witness, _processExitSource, LimboLogs.Instance); } @@ -143,6 +150,15 @@ private static DefaultHttpContext MakePostContext(string path, byte[] body, int return ctx; } + private static DefaultHttpContext MakeJsonPostContext(string path, byte[] body, int port = AuthenticatedPort) + { + DefaultHttpContext ctx = MakeBaseContext("POST", path, port); + ctx.Request.ContentType = "application/json"; + ctx.Request.ContentLength = body.Length; + ctx.Request.Body = new MemoryStream(body); + return ctx; + } + private static DefaultHttpContext MakeGetContext(string path, int port = AuthenticatedPort) { DefaultHttpContext ctx = MakeBaseContext("GET", path, port); @@ -611,7 +627,7 @@ public async Task Encoder_returning_zero_length_for_non_null_data_yields_204() // 204 No Content rather than 200 OK with Content-Length: 0. ZeroLengthEncodeHandler handler = new(); SszMiddleware middleware = new( - _ => Task.CompletedTask, _urlCollection, _auth, [handler], _processExitSource, LimboLogs.Instance); + _ => Task.CompletedTask, _urlCollection, _auth, [handler], witnessHandler: null, _processExitSource, LimboLogs.Instance); DefaultHttpContext ctx = MakePostContext($"/engine/v2/{ParisUrl}/{ZeroLengthEncodeHandler.ResourceName}", []); @@ -1023,4 +1039,231 @@ public async Task Error_response_has_correct_RFC7807_shape_with_detail_for_non_c Assert.That(root.TryGetProperty("detail", out _), Is.True, "unsupported-fork must include 'detail'"); Assert.That(root.EnumerateObject().Count(), Is.EqualTo(2), "error body must have exactly two keys: type + detail"); } + + // The witness endpoint (EIP-7928) is served by SszMiddleware's dedicated fast-path on its own + // version-less /new-payload-with-witness path. It speaks application/json (request) and emits + // application/octet-stream SSZ on success, RFC 7807 application/problem+json on error. + + [Test] + public async Task NewPayloadWithWitness_returns_200_with_octet_stream_and_decodable_ssz_for_valid_status() + { + Witness stubWitness = new() + { + State = new ArrayPoolList(1) { new byte[] { 0xDE, 0xAD, 0xBE, 0xEF } }, + Codes = new ArrayPoolList(0), + Keys = new ArrayPoolList(0), + Headers = new ArrayPoolList(0), + }; + + NewPayloadWithWitnessV1Result witnessResult = NewPayloadWithWitnessV1Result.FromPayloadStatus( + new PayloadStatusV1 { Status = PayloadStatus.Valid, LatestValidHash = TestItem.KeccakA }, + stubWitness); + + _engineModule.engine_newPayloadWithWitness( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(ResultWrapper.Success(witnessResult)); + + byte[] body = BuildMinimalWitnessRequestBody(); + DefaultHttpContext ctx = MakeJsonPostContext("/new-payload-with-witness", body); + + await _middleware.InvokeAsync(ctx); + + await _engineModule.Received(1).engine_newPayloadWithWitness( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()); + Assert.That(ctx.Response.StatusCode, Is.EqualTo(StatusCodes.Status200OK), + "VALID with a successfully generated witness must return 200 OK"); + Assert.That(ctx.Response.ContentType, Does.Contain(OctetStream), + "successful SSZ responses must use application/octet-stream"); + + byte[] responseBody = ResponseBytes(ctx); + Assert.That(responseBody, Is.Not.Empty, "the SSZ body must contain the encoded response"); + + (byte decodedStatus, Hash256? decodedLvh, bool witnessPresent) = SszCodec.DecodeNewPayloadWithWitnessResponse(responseBody); + Assert.That(decodedStatus, Is.EqualTo(0), "decoded status byte must match VALID"); + Assert.That(decodedLvh, Is.EqualTo(TestItem.KeccakA), + "latest_valid_hash Union Some variant must round-trip the hash correctly"); + Assert.That(witnessPresent, Is.True, + "a VALID response with a generated witness must encode the witness as Union Some (selector 0x01)"); + } + + [Test] + public async Task NewPayloadWithWitness_valid_status_but_witness_generation_fails_returns_200_with_null_witness() + { + NewPayloadWithWitnessV1Result witnessResult = NewPayloadWithWitnessV1Result.FromPayloadStatus( + new PayloadStatusV1 { Status = PayloadStatus.Valid, LatestValidHash = TestItem.KeccakA }, + witness: null); + + _engineModule.engine_newPayloadWithWitness( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(ResultWrapper.Success(witnessResult)); + + byte[] body = BuildMinimalWitnessRequestBody(); + DefaultHttpContext ctx = MakeJsonPostContext("/new-payload-with-witness", body); + + await _middleware.InvokeAsync(ctx); + + Assert.That(ctx.Response.StatusCode, Is.EqualTo(StatusCodes.Status200OK), + "the block is accepted even when witness generation fails; CL must not see 500"); + Assert.That(ctx.Response.ContentType, Does.Contain(OctetStream)); + + byte[] responseBody = ResponseBytes(ctx); + Assert.That(responseBody, Is.Not.Empty); + (byte decodedStatus, _, bool witnessPresent) = SszCodec.DecodeNewPayloadWithWitnessResponse(responseBody); + Assert.That(decodedStatus, Is.EqualTo(0)); + Assert.That(witnessPresent, Is.False, + "when witness generation fails the witness Union field must be None (selector 0x00)"); + } + + [Test] + public async Task NewPayloadWithWitness_wrong_content_type_post_returns_415() + { + DefaultHttpContext ctx = MakeBaseContext("POST", "/new-payload-with-witness", AuthenticatedPort); + ctx.Request.ContentType = "text/plain"; + ctx.Request.Body = Stream.Null; + + await _middleware.InvokeAsync(ctx); + + Assert.That(ctx.Response.StatusCode, Is.EqualTo(StatusCodes.Status415UnsupportedMediaType), + "a POST with wrong Content-Type must receive 415, not fall through to 404"); + Assert.That(ctx.Response.ContentType, Does.Contain("application/problem+json")); + } + + [Test] + public async Task NewPayloadWithWitness_non_valid_status_returns_200_with_ssz_body() + { + NewPayloadWithWitnessV1Result witnessResult = NewPayloadWithWitnessV1Result.FromPayloadStatus( + new PayloadStatusV1 { Status = PayloadStatus.Syncing }); + + _engineModule.engine_newPayloadWithWitness( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(ResultWrapper.Success(witnessResult)); + + byte[] body = BuildMinimalWitnessRequestBody(); + DefaultHttpContext ctx = MakeJsonPostContext("/new-payload-with-witness", body); + + await _middleware.InvokeAsync(ctx); + + Assert.That(ctx.Response.StatusCode, Is.EqualTo(StatusCodes.Status200OK), + "SYNCING is a normal processing outcome and must return 200, not an HTTP error"); + Assert.That(ctx.Response.ContentType, Does.Contain(OctetStream)); + Assert.That(ResponseBytes(ctx), Is.Not.Empty, "the SSZ body must contain the status fields"); + } + + [Test] + public async Task NewPayloadWithWitness_malformed_json_returns_400_problem_json() + { + byte[] badBody = System.Text.Encoding.UTF8.GetBytes("not json at all"); + DefaultHttpContext ctx = MakeJsonPostContext("/new-payload-with-witness", badBody); + + await _middleware.InvokeAsync(ctx); + + Assert.That(ctx.Response.StatusCode, Is.EqualTo(StatusCodes.Status400BadRequest)); + Assert.That(ctx.Response.ContentType, Does.Contain("application/problem+json"), + "error responses must be RFC 7807 application/problem+json"); + string responseBody = System.Text.Encoding.UTF8.GetString(ResponseBytes(ctx)); + Assert.That(responseBody, Does.Contain("\"type\"")); + } + + [Test] + public async Task NewPayloadWithWitness_non_post_method_returns_405() + { + DefaultHttpContext ctx = MakeGetContext("/new-payload-with-witness"); + + await _middleware.InvokeAsync(ctx); + + Assert.That(ctx.Response.StatusCode, Is.EqualTo(StatusCodes.Status405MethodNotAllowed), + "spec mandates 405 for any method other than POST on this endpoint"); + Assert.That(ctx.Response.ContentType, Does.Contain("application/problem+json")); + } + + [Test] + public async Task NewPayloadWithWitness_unsupported_fork_returns_400_with_correct_type() + { + _engineModule.engine_newPayloadWithWitness( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(ResultWrapper.Fail("Unsupported fork", MergeErrorCodes.UnsupportedFork)); + + byte[] body = BuildMinimalWitnessRequestBody(); + DefaultHttpContext ctx = MakeJsonPostContext("/new-payload-with-witness", body); + + await _middleware.InvokeAsync(ctx); + + Assert.That(ctx.Response.StatusCode, Is.EqualTo(StatusCodes.Status400BadRequest)); + Assert.That(ctx.Response.ContentType, Does.Contain("application/problem+json")); + string responseBody = System.Text.Encoding.UTF8.GetString(ResponseBytes(ctx)); + Assert.That(responseBody, Does.Contain("/engine-api/errors/unsupported-fork"), + "the RFC 7807 type URI for unsupported-fork must be present in the error body"); + } + + [Test] + public async Task NewPayloadWithWitness_via_versioned_engine_path_returns_404() + { + byte[] body = BuildMinimalWitnessRequestBody(); + DefaultHttpContext ctx = MakePostContext("/engine/v1/new-payload-with-witness", body); + + await _middleware.InvokeAsync(ctx); + + Assert.That(ctx.Response.StatusCode, Is.EqualTo(StatusCodes.Status404NotFound), + "the witness endpoint has no versioned /engine/vN/ path; the versioned URL must return 404"); + Assert.That(ctx.Response.ContentType, Does.Contain("application/problem+json"), + "error responses must always be RFC 7807 application/problem+json"); + } + + [Test] + public async Task NewPayloadWithWitness_non_UnsupportedFork_engine_error_returns_500() + { + _engineModule.engine_newPayloadWithWitness( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(ResultWrapper.Fail("Something exploded", ErrorCodes.InternalError)); + + byte[] body = BuildMinimalWitnessRequestBody(); + DefaultHttpContext ctx = MakeJsonPostContext("/new-payload-with-witness", body); + + await _middleware.InvokeAsync(ctx); + + Assert.That(ctx.Response.StatusCode, Is.EqualTo(StatusCodes.Status500InternalServerError), + "non-UnsupportedFork engine errors must map to 500 Internal Server Error"); + Assert.That(ctx.Response.ContentType, Does.Contain("application/problem+json")); + string responseBody = System.Text.Encoding.UTF8.GetString(ResponseBytes(ctx)); + Assert.That(responseBody, Does.Contain("/engine-api/errors/internal")); + } + + private static byte[] BuildMinimalWitnessRequestBody() + { + ExecutionPayloadV4 payload = new() + { + ParentHash = TestItem.KeccakA, + FeeRecipient = TestItem.AddressA, + StateRoot = TestItem.KeccakB, + ReceiptsRoot = TestItem.KeccakC, + LogsBloom = Bloom.Empty, + PrevRandao = TestItem.KeccakD, + BlockNumber = 1, + GasLimit = 1_000_000, + GasUsed = 0, + Timestamp = 1_700_000_000, + ExtraData = [], + BaseFeePerGas = 1, + BlockHash = TestItem.KeccakE, + Transactions = [], + Withdrawals = [], + BlobGasUsed = 0, + ExcessBlobGas = 0, + ParentBeaconBlockRoot = TestItem.KeccakA, + ExecutionRequests = [], + BlockAccessList = BlockAccessListDecoder.EncodeToBytes(new BlockAccessListBuilder().TestObject) + }; + + string json = System.Text.Json.JsonSerializer.Serialize( + new object?[] + { + payload, + Array.Empty(), + TestItem.KeccakA, + Array.Empty() + }, + Serialization.Json.EthereumJsonSerializer.JsonOptions); + + return System.Text.Encoding.UTF8.GetBytes(json); + } } diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Data/EngineApiJsonContext.cs b/src/Nethermind/Nethermind.Merge.Plugin/Data/EngineApiJsonContext.cs index 0a969638c450..8d4296035a61 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Data/EngineApiJsonContext.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Data/EngineApiJsonContext.cs @@ -3,6 +3,7 @@ using System.Text.Json.Serialization; using Nethermind.Consensus.Producers; +using Nethermind.Consensus.Stateless; using Nethermind.Merge.Plugin.Handlers; using Nethermind.Serialization.Json; @@ -15,8 +16,16 @@ namespace Nethermind.Merge.Plugin.Data; DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, IncludeFields = true, Converters = new[] { typeof(ByteArrayArrayConverter) })] +[JsonSerializable(typeof(ExecutionPayload))] [JsonSerializable(typeof(ExecutionPayloadV3))] +[JsonSerializable(typeof(ExecutionPayloadV4))] +[JsonSerializable(typeof(PayloadStatusV1))] [JsonSerializable(typeof(byte[][]))] [JsonSerializable(typeof(PayloadAttributes))] [JsonSerializable(typeof(GetBlobsHandlerV2Request))] +[JsonSerializable(typeof(ExecutionPayloadBodyV1Result))] +[JsonSerializable(typeof(TransitionConfigurationV1))] +[JsonSerializable(typeof(ClientVersionV1))] +[JsonSerializable(typeof(NewPayloadWithWitnessV1Result))] +[JsonSerializable(typeof(Witness))] internal partial class EngineApiJsonContext : JsonSerializerContext; diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Data/NewPayloadWithWitnessV1Result.cs b/src/Nethermind/Nethermind.Merge.Plugin/Data/NewPayloadWithWitnessV1Result.cs new file mode 100644 index 000000000000..754d34bc57e4 --- /dev/null +++ b/src/Nethermind/Nethermind.Merge.Plugin/Data/NewPayloadWithWitnessV1Result.cs @@ -0,0 +1,41 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Text.Json.Serialization; +using Nethermind.Consensus.Stateless; +using Nethermind.Core.Crypto; + +namespace Nethermind.Merge.Plugin.Data; + +/// +/// Result of engine_newPayloadWithWitness. +/// Combines the standard fields with an optional +/// that is populated when is +/// . +/// +/// +public class NewPayloadWithWitnessV1Result : IDisposable +{ + public string Status { get; set; } = PayloadStatus.Invalid; + + [JsonIgnore(Condition = JsonIgnoreCondition.Never)] + public Hash256? LatestValidHash { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.Never)] + public string? ValidationError { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public Witness? ExecutionWitness { get; set; } + + public static NewPayloadWithWitnessV1Result FromPayloadStatus(PayloadStatusV1 status, Witness? witness = null) => + new() + { + Status = status.Status, + LatestValidHash = status.LatestValidHash, + ValidationError = status.ValidationError, + ExecutionWitness = witness + }; + + public void Dispose() => ExecutionWitness?.Dispose(); +} diff --git a/src/Nethermind/Nethermind.Merge.Plugin/EngineRpcModule.Amsterdam.cs b/src/Nethermind/Nethermind.Merge.Plugin/EngineRpcModule.Amsterdam.cs index 576b9a9c4464..96bc24fb0bf3 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/EngineRpcModule.Amsterdam.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/EngineRpcModule.Amsterdam.cs @@ -18,6 +18,7 @@ public partial class EngineRpcModule : IEngineRpcModule private readonly IAsyncHandler _getPayloadHandlerV6 = getPayloadHandlerV6; private readonly IHandler, IReadOnlyList> _executionGetPayloadBodiesByHashV2Handler = getPayloadBodiesByHashV2Handler; private readonly IGetPayloadBodiesByRangeV2Handler _executionGetPayloadBodiesByRangeV2Handler = getPayloadBodiesByRangeV2Handler; + private readonly INewPayloadWithWitnessHandler _newPayloadWithWitnessHandler = newPayloadWithWitnessHandler; private readonly IAsyncHandler?> _getBlobsHandlerV4 = getBlobsHandlerV4; @@ -27,6 +28,14 @@ public partial class EngineRpcModule : IEngineRpcModule public Task> engine_newPayloadV5(ExecutionPayloadV4 executionPayload, Hash256?[] blobVersionedHashes, Hash256? parentBeaconBlockRoot, byte[][]? executionRequests) => NewPayload(new ExecutionPayloadParams(executionPayload, blobVersionedHashes, parentBeaconBlockRoot, executionRequests), EngineApiVersions.NewPayload.V5); + public Task> engine_newPayloadWithWitness( + ExecutionPayloadV4 executionPayload, + Hash256?[] blobVersionedHashes, + Hash256? parentBeaconBlockRoot, + byte[][]? executionRequests) + => _newPayloadWithWitnessHandler.HandleAsync( + executionPayload, blobVersionedHashes, parentBeaconBlockRoot, executionRequests); + public Task> engine_forkchoiceUpdatedV4(ForkchoiceStateV1 forkchoiceState, PayloadAttributes? payloadAttributes = null, BitArray? custodyColumns = null) { // Per execution-apis #793: custody-column updates are best-effort, errors swallowed. @@ -38,10 +47,13 @@ public Task> engine_forkchoiceUpdatedV4 return ForkchoiceUpdated(forkchoiceState, payloadAttributes, EngineApiVersions.Fcu.V4); } - public Task>> engine_getPayloadBodiesByHashV2(IReadOnlyList blockHashes) + public Task>> engine_getPayloadBodiesByHashV2( + IReadOnlyList blockHashes) => _executionGetPayloadBodiesByHashV2Handler.Handle(blockHashes); - public Task>> engine_getPayloadBodiesByRangeV2(long start, long count) + public Task>> engine_getPayloadBodiesByRangeV2( + long start, + long count) => _executionGetPayloadBodiesByRangeV2Handler.Handle(start, count); public Task?>> engine_getBlobsV4(byte[][] blobVersionedHashes, System.Collections.BitArray indicesBitarray) diff --git a/src/Nethermind/Nethermind.Merge.Plugin/EngineRpcModule.cs b/src/Nethermind/Nethermind.Merge.Plugin/EngineRpcModule.cs index 4c9027783d77..4bea20581bbd 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/EngineRpcModule.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/EngineRpcModule.cs @@ -32,6 +32,7 @@ public partial class EngineRpcModule( IAsyncHandler?> getBlobsHandlerV4, IHandler, IReadOnlyList> getPayloadBodiesByHashV2Handler, IGetPayloadBodiesByRangeV2Handler getPayloadBodiesByRangeV2Handler, + INewPayloadWithWitnessHandler newPayloadWithWitnessHandler, IEngineRequestsTracker engineRequestsTracker, ISpecProvider specProvider, GCKeeper gcKeeper, diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/EngineRpcCapabilitiesProvider.cs b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/EngineRpcCapabilitiesProvider.cs index e101c2326cbc..c27ae33d3966 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/EngineRpcCapabilitiesProvider.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/EngineRpcCapabilitiesProvider.cs @@ -126,6 +126,7 @@ void Configure(string method, string path, RpcCapabilityOptions options) Configure(nameof(IEngineRpcModule.engine_forkchoiceUpdatedV4), SszRestPaths.PostV4Forkchoice, GateWithWarn(spec.IsEip7843Enabled)); Configure(nameof(IEngineRpcModule.engine_getPayloadBodiesByHashV2), SszRestPaths.PostV2PayloadBodiesByHash, GateWithWarn(spec.IsEip7928Enabled)); Configure(nameof(IEngineRpcModule.engine_getPayloadBodiesByRangeV2), SszRestPaths.GetV2PayloadBodiesByRange, GateWithWarn(spec.IsEip7928Enabled)); + Configure(nameof(IEngineRpcModule.engine_newPayloadWithWitness), SszRestCapabilities.NewPayloadWithWitness, GateWithWarn(spec.IsEip7928Enabled)); json = jsonLocal; ssz = sszLocal; diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/INewPayloadWithWitnessHandler.cs b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/INewPayloadWithWitnessHandler.cs new file mode 100644 index 000000000000..41ee0e3df175 --- /dev/null +++ b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/INewPayloadWithWitnessHandler.cs @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Threading.Tasks; +using Nethermind.Core.Crypto; +using Nethermind.JsonRpc; +using Nethermind.Merge.Plugin.Data; + +namespace Nethermind.Merge.Plugin.Handlers; + +/// +/// Handles the engine_newPayloadWithWitness RPC method. +/// +public interface INewPayloadWithWitnessHandler +{ + Task> HandleAsync( + ExecutionPayloadV4 executionPayload, + Hash256?[] blobVersionedHashes, + Hash256? parentBeaconBlockRoot, + byte[][]? executionRequests); +} diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/NewPayloadWithWitnessHandler.cs b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/NewPayloadWithWitnessHandler.cs new file mode 100644 index 000000000000..8b05780559c7 --- /dev/null +++ b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/NewPayloadWithWitnessHandler.cs @@ -0,0 +1,100 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Threading.Tasks; +using Nethermind.Consensus.Stateless; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.JsonRpc; +using Nethermind.Logging; +using Nethermind.Merge.Plugin.Data; + +namespace Nethermind.Merge.Plugin.Handlers; + +/// +/// is taken via to break the construction +/// cycle (the module composes this handler). On pre-Amsterdam chains the +/// decorator is not installed, so the rendezvous +/// TCS for any requested block hash never completes; the cancel-on-non-VALID and +/// cancel-when-not-completed branches below handle that gracefully. +/// +public sealed class NewPayloadWithWitnessHandler( + Lazy engineModule, + WitnessRendezvous rendezvous, + ILogManager? logManager = null) : INewPayloadWithWitnessHandler +{ + private readonly ILogger _logger = (logManager ?? LimboLogs.Instance).GetClassLogger(); + + public async Task> HandleAsync( + ExecutionPayloadV4 executionPayload, + Hash256?[] blobVersionedHashes, + Hash256? parentBeaconBlockRoot, + byte[][]? executionRequests) + { + Hash256? blockHash = executionPayload.BlockHash; + + if (blockHash is null) + { + if (_logger.IsWarn) _logger.Warn("engine_newPayloadWithWitness: payload BlockHash is null — rejecting as InvalidParams."); + return ResultWrapper.Fail( + "executionPayload.blockHash is required", ErrorCodes.InvalidParams); + } + + Task captureTask = rendezvous.RequestWitness(blockHash); + + ResultWrapper statusResult; + try + { + statusResult = await engineModule.Value.engine_newPayloadV5( + executionPayload, blobVersionedHashes, parentBeaconBlockRoot, executionRequests); + } + catch + { + rendezvous.CancelWitnessRequest(blockHash); + throw; + } + + using (statusResult) + { + if (statusResult.Result.ResultType != ResultType.Success) + { + rendezvous.CancelWitnessRequest(blockHash); + return ResultWrapper.Fail( + statusResult.Result.Error ?? "engine_newPayloadV5 failed", + statusResult.ErrorCode); + } + + PayloadStatusV1 payloadStatus = statusResult.Data!; + Witness? witness = null; + + if (payloadStatus.Status == PayloadStatus.Valid) + { + // BlockProcessor normally completes the TCS synchronously inside ProcessOne. + // If it didn't, the block either took an early-return path (already known, etc.) + // or the decorator isn't installed (pre-Amsterdam) — cancel so the await below + // doesn't block forever. + if (!captureTask.IsCompleted) + rendezvous.CancelWitnessRequest(blockHash); + + try + { + witness = await captureTask; + } + catch (OperationCanceledException) + { + if (_logger.IsWarn) _logger.Warn($"engine_newPayloadWithWitness: witness capture cancelled for {blockHash}. Returning VALID with no witness."); + } + } + else + { + rendezvous.CancelWitnessRequest(blockHash); + if (captureTask.IsCompletedSuccessfully) + (await captureTask)?.Dispose(); + } + + return ResultWrapper.Success( + NewPayloadWithWitnessV1Result.FromPayloadStatus(payloadStatus, witness)); + } + } +} diff --git a/src/Nethermind/Nethermind.Merge.Plugin/IEngineRpcModule.Amsterdam.cs b/src/Nethermind/Nethermind.Merge.Plugin/IEngineRpcModule.Amsterdam.cs index 2efd15d28668..dd522b749605 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/IEngineRpcModule.Amsterdam.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/IEngineRpcModule.Amsterdam.cs @@ -26,6 +26,12 @@ public partial interface IEngineRpcModule : IRpcModule IsImplemented = true)] Task> engine_newPayloadV5(ExecutionPayloadV4 executionPayload, Hash256?[] blobVersionedHashes, Hash256? parentBeaconBlockRoot, byte[][]? executionRequests); + [JsonRpcMethod( + Description = "Verifies the payload according to the execution environment rules and returns the verification status, hash of the last valid block, and the execution witness when the payload is valid.", + IsSharable = true, + IsImplemented = true)] + Task> engine_newPayloadWithWitness(ExecutionPayloadV4 executionPayload, Hash256?[] blobVersionedHashes, Hash256? parentBeaconBlockRoot, byte[][]? executionRequests); + [JsonRpcMethod( Description = "Applies fork choice and starts building a new block if payload attributes are present.", IsSharable = true, diff --git a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs index c82650155817..19c5ee432a8b 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs @@ -15,8 +15,10 @@ using Nethermind.Config; using Nethermind.Consensus; using Nethermind.Consensus.Processing; +using Nethermind.Core.Container; using Nethermind.Consensus.Producers; using Nethermind.Consensus.Rewards; +using Nethermind.Consensus.Stateless; using Nethermind.Consensus.Validators; using Nethermind.Core; using Nethermind.Core.Crypto; @@ -36,13 +38,14 @@ using Nethermind.Merge.Plugin.InvalidChainTracker; using Nethermind.Merge.Plugin.SszRest; using Nethermind.Merge.Plugin.Synchronization; +using Nethermind.Trie; +using Nethermind.Trie.Pruning; using Nethermind.Network.Contract.P2P; using Nethermind.Serialization.Json; using Nethermind.Specs.ChainSpecStyle; using Nethermind.State; using Nethermind.Synchronization; using Nethermind.Synchronization.ParallelSync; -using Nethermind.Trie.Pruning; using Nethermind.TxPool; namespace Nethermind.Merge.Plugin; @@ -280,6 +283,28 @@ protected override void Load(ContainerBuilder builder) => builder .AddSingleton() .AddDecorator() + .AddSingleton() + // Rendezvous lives in the root scope so the JSON-RPC handler can take it directly; the + // main-processing module simply consumes it when EIP-7928 is enabled. + .AddSingleton() + // The capture session also lives at root: the main-world trie store below is constructed + // at root, before the main-processing child scope exists, so its read-tap must consult a + // root-scoped session. The main-processing module's decorators resolve this same instance. + .AddSingleton() + // Read-tap on the patricia main-world trie store (registered by PruningTrieStoreModule). + // Inert — one null check per node read — until the witness-capturing block processor arms + // the session. Never constructed on flat, where no main-world ITrieStore is resolved. + // Lambda registration because the write-through flag is not container-resolvable: the + // live store persists state, so commits must forward rather than hit NullCommitter. + .AddDecorator((ctx, trieStore) => + new WitnessCapturingTrieStore(trieStore, ctx.Resolve(), readOnly: false)) + // Flat-side capture: FlatWorldStateManager threads this observer into its main world + // state's trie adapters, so commit-time merkleization reads (write paths + collapse + // siblings) land on the armed session's trie recorder. Inert when no capture is armed; + // never resolved on patricia (the optional ctor param is only on the flat manager). + // Use that as flat db doesn't have ITrieStore to wrap in WitnessCapturingTrieStore for main processing pipeline + .AddSingleton() + .AddSingleton() .ResolveOnServiceActivation() @@ -321,6 +346,7 @@ protected override void Load(ContainerBuilder builder) => builder .AddSingleton?>, GetBlobsHandlerV4>() .AddSingleton, IReadOnlyList>, GetPayloadBodiesByHashV2Handler>() .AddSingleton() + .AddSingleton() .AddSingleton() .AddSingleton((ctx) => diff --git a/src/Nethermind/Nethermind.Merge.Plugin/SszRest/Handlers/NewPayloadWithWitnessSszHandler.cs b/src/Nethermind/Nethermind.Merge.Plugin/SszRest/Handlers/NewPayloadWithWitnessSszHandler.cs new file mode 100644 index 000000000000..62e09b423da7 --- /dev/null +++ b/src/Nethermind/Nethermind.Merge.Plugin/SszRest/Handlers/NewPayloadWithWitnessSszHandler.cs @@ -0,0 +1,150 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Buffers; +using System.Text.Json; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Nethermind.Consensus.Stateless; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.JsonRpc; +using Nethermind.Merge.Plugin.Data; +using Nethermind.Serialization.Json; + +namespace Nethermind.Merge.Plugin.SszRest.Handlers; + +/// +/// Handles POST /new-payload-with-witness. Per execution-apis#773 the request is JSON +/// (same shape as engine_newPayloadV5 params) and the response is SSZ-encoded +/// NewPayloadWithWitnessResponseV1 — the only mixed-format endpoint in the SSZ-REST surface. +/// +public sealed class NewPayloadWithWitnessSszHandler( + IEngineRpcModule engineModule) : SszEndpointHandlerBase +{ + + public override string HttpMethod => "POST"; + + // Non-versioned path; SszMiddleware routes via a dedicated fast path for this resource. + public override string Resource => SszRestPaths.NewPayloadWithWitness; + public override int? Version => null; + + public override async Task HandleAsync(HttpContext ctx, int version, ReadOnlyMemory extra, ReadOnlySequence body) + { + NewPayloadV5Params? request = DeserializeRequest(body); + if (request is null) + { + await WriteErrorAsync(ctx, StatusCodes.Status400BadRequest, "Malformed JSON body or invalid parameter shapes", ErrorCodes.ParseError); + return; + } + + ResultWrapper result = await engineModule.engine_newPayloadWithWitness( + request.ExecutionPayload, + request.ExpectedBlobVersionedHashes, + request.ParentBeaconBlockRoot, + request.ExecutionRequests); + + using (result) + { + if (result.Result.ResultType != ResultType.Success) + { + int httpStatus = result.ErrorCode switch + { + MergeErrorCodes.UnsupportedFork => StatusCodes.Status400BadRequest, + _ => StatusCodes.Status500InternalServerError, + }; + int jsonRpcCode = result.ErrorCode switch + { + MergeErrorCodes.UnsupportedFork => MergeErrorCodes.UnsupportedFork, + _ => ErrorCodes.InternalError, + }; + await WriteErrorAsync(ctx, httpStatus, result.Result.Error ?? "Unknown error", jsonRpcCode); + return; + } + + NewPayloadWithWitnessV1Result witnessResult = result.Data!; + PayloadStatusV1 payloadStatus = new() + { + Status = witnessResult.Status, + LatestValidHash = witnessResult.LatestValidHash, + ValidationError = witnessResult.ValidationError + }; + await WriteSszNewPayloadWithWitnessAsync(ctx, payloadStatus, witnessResult.ExecutionWitness); + } + } + + // Witness ownership stays with the caller — the enclosing ResultWrapper disposes it. + private static async Task WriteSszNewPayloadWithWitnessAsync(HttpContext ctx, PayloadStatusV1 status, Witness? witness) + { + ArrayBufferWriter buffer = new(); + int length; + try + { + length = SszCodec.EncodeNewPayloadWithWitnessResponse(status, witness, buffer); + } + catch + { + ctx.Abort(); + throw; + } + + ctx.Response.ContentType = "application/octet-stream"; + ctx.Response.ContentLength = length; + ctx.Response.StatusCode = StatusCodes.Status200OK; + + System.IO.Pipelines.PipeWriter pipe = ctx.Response.BodyWriter; + try + { + await pipe.WriteAsync(buffer.WrittenMemory, ctx.RequestAborted); + } + catch + { + ctx.Abort(); + throw; + } + + await ctx.Response.CompleteAsync(); + } + + private static NewPayloadV5Params? DeserializeRequest(ReadOnlySequence body) + { + try + { + Utf8JsonReader reader = new(body); + + if (!reader.Read() || reader.TokenType != JsonTokenType.StartArray) return null; + + if (!reader.Read()) return null; + ExecutionPayloadV4? payload = JsonSerializer.Deserialize( + ref reader, EthereumJsonSerializer.JsonOptions); + if (payload is null) return null; + + if (!reader.Read()) return null; + Hash256?[]? blobHashes = JsonSerializer.Deserialize( + ref reader, EthereumJsonSerializer.JsonOptions); + + if (!reader.Read()) return null; + Hash256? parentBeaconBlockRoot = JsonSerializer.Deserialize( + ref reader, EthereumJsonSerializer.JsonOptions); + + if (!reader.Read()) return null; + byte[][]? executionRequests = JsonSerializer.Deserialize( + ref reader, EthereumJsonSerializer.JsonOptions); + + if (!reader.Read() || reader.TokenType != JsonTokenType.EndArray) return null; + + return new NewPayloadV5Params(payload, blobHashes ?? [], parentBeaconBlockRoot, executionRequests); + } + catch (JsonException) + { + return null; + } + } + + private sealed record NewPayloadV5Params( + ExecutionPayloadV4 ExecutionPayload, + Hash256?[] ExpectedBlobVersionedHashes, + Hash256? ParentBeaconBlockRoot, + byte[][]? ExecutionRequests); +} diff --git a/src/Nethermind/Nethermind.Merge.Plugin/SszRest/Handlers/SszRestPaths.cs b/src/Nethermind/Nethermind.Merge.Plugin/SszRest/Handlers/SszRestPaths.cs index d7f83b522910..c0bb904e3615 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/SszRest/Handlers/SszRestPaths.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/SszRest/Handlers/SszRestPaths.cs @@ -75,6 +75,10 @@ public static class SszRestPaths public const string Blobs = "blobs"; + // Witness endpoint resource segment (EIP-7928). Routed by SszMiddleware's dedicated witness + // fast-path (its own version-less /new-payload-with-witness path), not the fork-segment router. + public const string NewPayloadWithWitness = "new-payload-with-witness"; + // Documentation strings for the SSZ-REST routes — used by EngineRpcCapabilitiesProvider // (registration) and EngineModuleTests (coverage assertions). Built at static-init time from // each fork's EngineApiUrlSegment so the route docs stay in sync with the routing layer. @@ -152,3 +156,13 @@ public static class SszRestPaths return null; } } + +/// +/// Engine API capability names that are advertised by engine_exchangeCapabilities but are +/// not part of the standard POST /engine/v2/{fork}/{resource} path scheme. The EIP-7928 +/// witness endpoint has its own dedicated, version-less path and is advertised under this name. +/// +public static class SszRestCapabilities +{ + public const string NewPayloadWithWitness = "rest_engine_newPayloadWithWitness"; +} diff --git a/src/Nethermind/Nethermind.Merge.Plugin/SszRest/SszCodec.cs b/src/Nethermind/Nethermind.Merge.Plugin/SszRest/SszCodec.cs index 804f03deff16..ec31f1707461 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/SszRest/SszCodec.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/SszRest/SszCodec.cs @@ -5,21 +5,21 @@ using System.Buffers; using System.Collections.Generic; using System.Text; +using Nethermind.Consensus.Stateless; using Nethermind.Core; +using Nethermind.Core.Collections; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using Nethermind.Consensus.Producers; using Nethermind.Merge.Plugin.Data; using Nethermind.Serialization.Ssz; +using System.Buffers.Binary; namespace Nethermind.Merge.Plugin.SszRest; public static class SszCodec { - /// - /// SSZ-encodes directly into 's buffer - /// (no intermediate pooled allocation) and returns the number of bytes written. - /// + /// Encode directly into the writer's buffer (no intermediate alloc); returns bytes written. private static int EncodeToWriter(T value, IBufferWriter writer) where T : ISszCodec { int length = T.GetLength(value); @@ -32,6 +32,139 @@ private static int EncodeToWriter(T value, IBufferWriter writer) where public static int EncodePayloadStatus(PayloadStatusV1 ps, IBufferWriter writer) => EncodeToWriter(BuildPayloadStatusWire(ps), writer); + public static int EncodeNewPayloadWithWitnessResponse(PayloadStatusV1 ps, Witness? witness, IBufferWriter writer) + { + const int ValidationErrorMax = 8192; + const int FixedHeaderBytes = 1 + 4 + 4 + 4; + + bool hasLvh = ps.LatestValidHash is not null; + int lvhLen = hasLvh ? 33 : 1; + + byte[] errorBytes = ps.ValidationError is not null + ? Encoding.UTF8.GetBytes(ps.ValidationError) + : []; + if (errorBytes.Length > ValidationErrorMax) + errorBytes = TruncateUtf8(errorBytes, ValidationErrorMax); + bool hasError = ps.ValidationError is not null; + int errorLen = hasError ? 1 + errorBytes.Length : 1; + + bool hasWitness = witness is not null && ps.Status == PayloadStatus.Valid; + ExecutionWitnessV1Wire witnessWire = hasWitness ? BuildExecutionWitnessV1Wire(witness!) : default; + int witnessBodyLen = hasWitness ? ExecutionWitnessV1Wire.GetLength(witnessWire) : 0; + int witnessLen = hasWitness ? 1 + witnessBodyLen : 1; + + int totalLen = FixedHeaderBytes + lvhLen + errorLen + witnessLen; + + // No dst.Clear() — every byte in [0, totalLen) is overwritten below. + Span dst = writer.GetSpan(totalLen)[..totalLen]; + + int pos = 0; + + dst[pos++] = EngineStatusToSsz(ps.Status); + + // SSZ offsets are uint32; all three are bounded by MaxBodySize (16 MiB) << 2^31. + uint off1 = (uint)FixedHeaderBytes; + BinaryPrimitives.WriteUInt32LittleEndian(dst.Slice(pos, 4), off1); + pos += 4; + + uint off2 = off1 + (uint)lvhLen; + BinaryPrimitives.WriteUInt32LittleEndian(dst.Slice(pos, 4), off2); + pos += 4; + + uint off3 = off2 + (uint)errorLen; + BinaryPrimitives.WriteUInt32LittleEndian(dst.Slice(pos, 4), off3); + pos += 4; + + if (hasLvh) + { + dst[pos++] = 0x01; + ps.LatestValidHash!.Bytes.CopyTo(dst.Slice(pos, 32)); + pos += 32; + } + else + { + dst[pos++] = 0x00; + } + + if (hasError) + { + dst[pos++] = 0x01; + errorBytes.CopyTo(dst.Slice(pos, errorBytes.Length)); + pos += errorBytes.Length; + } + else + { + dst[pos++] = 0x00; + } + + if (hasWitness) + { + dst[pos++] = 0x01; + ExecutionWitnessV1Wire.Encode(dst.Slice(pos, witnessBodyLen), witnessWire); + pos += witnessBodyLen; + } + else + { + dst[pos++] = 0x00; + } + + if (pos != totalLen) + throw new InvalidOperationException($"NewPayloadWithWitnessResponseV1 encode length mismatch: wrote {pos} bytes but expected {totalLen}"); + writer.Advance(totalLen); + return totalLen; + } + + /// Test helper: round-trip decode of NewPayloadWithWitnessResponseV1 SSZ output. + public static (byte Status, Hash256? LatestValidHash, bool WitnessPresent) + DecodeNewPayloadWithWitnessResponse(ReadOnlySpan data) + { + const int FixedHeaderBytes = 1 + 4 + 4 + 4; + if (data.Length < FixedHeaderBytes) + throw new ArgumentException("Response too short to be a valid NewPayloadWithWitnessResponseV1"); + + byte status = data[0]; + // Spec offsets are uint32; cast to int for slicing (we're well within int range). + int off1 = (int)BinaryPrimitives.ReadUInt32LittleEndian(data.Slice(1, 4)); + int off2 = (int)BinaryPrimitives.ReadUInt32LittleEndian(data.Slice(5, 4)); + int off3 = (int)BinaryPrimitives.ReadUInt32LittleEndian(data.Slice(9, 4)); + + ReadOnlySpan lvhSlice = data.Slice(off1, off2 - off1); + Hash256? latestValidHash = null; + if (lvhSlice.Length >= 1 && lvhSlice[0] == 0x01) + latestValidHash = new Hash256(lvhSlice.Slice(1, 32)); + + ReadOnlySpan witnessSlice = data.Slice(off3); + bool witnessPresent = witnessSlice.Length >= 1 && witnessSlice[0] == 0x01; + + return (status, latestValidHash, witnessPresent); + } + + private static ExecutionWitnessV1Wire BuildExecutionWitnessV1Wire(Witness witness) + { + return new ExecutionWitnessV1Wire + { + State = ToWitnessItems(witness.State), + Codes = ToWitnessItems(witness.Codes), + Headers = ToWitnessItems(witness.Headers) + }; + + static SszWitnessItem[] ToWitnessItems(IOwnedReadOnlyList items) + { + SszWitnessItem[] result = new SszWitnessItem[items.Count]; + for (int i = 0; i < items.Count; i++) + result[i] = new SszWitnessItem { Bytes = items[i] }; + return result; + } + } + + private static byte[] TruncateUtf8(byte[] utf8, int maxBytes) + { + int i = maxBytes; + while (i > 0 && (utf8[i] & 0xC0) == 0x80) + i--; + return utf8[..i]; + } + public static int EncodeForkchoiceUpdatedResponse(ForkchoiceUpdatedV1Result resp, IBufferWriter writer) { SszPayloadId[]? pidList = null; diff --git a/src/Nethermind/Nethermind.Merge.Plugin/SszRest/SszMiddleware.cs b/src/Nethermind/Nethermind.Merge.Plugin/SszRest/SszMiddleware.cs index b77d31aae158..7d2cffa776ff 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/SszRest/SszMiddleware.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/SszRest/SszMiddleware.cs @@ -35,6 +35,11 @@ public sealed class SszMiddleware // Path: /engine/v2/{fork}/{resource}[/{extra}] private const string EnginePrefix = "/engine/v2/"; + // The witness endpoint (EIP-7928) keeps its own dedicated, version-less path and a fast-path + // dispatch — it is not part of the /engine/v2/{fork}/{resource} routing scheme, and it speaks + // application/json rather than application/octet-stream. + private const string WitnessPath = "/new-payload-with-witness"; + /// /// Maximum allowed request body size in bytes (64 MiB). /// Mirrors the payload.max_bytes example value advertised in the Engine API @@ -47,6 +52,8 @@ public sealed class SszMiddleware private readonly FrozenDictionary>.AlternateLookup> _postLookup; private readonly FrozenDictionary>.AlternateLookup> _getLookup; + private readonly ISszEndpointHandler? _witnessHandler; + private enum SszRequestKind { NotEngine, EngineWrongMediaType, EngineOk } public SszMiddleware( @@ -54,12 +61,14 @@ public SszMiddleware( IJsonRpcUrlCollection urlCollection, IRpcAuthentication auth, IEnumerable handlers, + NewPayloadWithWitnessSszHandler? witnessHandler, IProcessExitSource processExitSource, ILogManager logManager) { _next = next; _urlCollection = urlCollection; _auth = auth; + _witnessHandler = witnessHandler; _logger = logManager.GetClassLogger(); _processExitToken = processExitSource.Token; (_postRoutes, _getRoutes) = BuildRoutes(handlers); @@ -76,6 +85,9 @@ private static (FrozenDictionary> post, foreach (ISszEndpointHandler h in handlers) { + // The witness handler is injected directly and dispatched via its own fast-path. + if (h is NewPayloadWithWitnessSszHandler) continue; + // Dictionaries are keyed case-insensitively below — keep resource as-is, no lowercasing. Dictionary> dict = h.HttpMethod.Equals("GET", StringComparison.OrdinalIgnoreCase) @@ -136,6 +148,10 @@ private async Task ProcessSszRequestAsync(HttpContext ctx) await SszEndpointHandlerBase.WriteErrorAsync(ctx, StatusCodes.Status401Unauthorized, "Authentication error"); } + else if (IsWitnessPath(ctx.Request.Path.Value ?? string.Empty)) + { + await DispatchWitnessAsync(ctx); + } else if (!TryRoute(ctx.Request.Path.Value ?? string.Empty, out int version, out string? fork, out ReadOnlyMemory pathSegment, out bool unsupportedFork)) { @@ -175,70 +191,120 @@ await SszEndpointHandlerBase.WriteErrorAsync(ctx, StatusCodes.Status404NotFound, : $"SSZ-REST {ctx.Request.Method} /engine/v2/{pathSegment.Span}/{extra.Span}"); } - // Read directly from PipeReader: the buffer is a ReadOnlySequence over Kestrel's - // pooled blocks (~4 KB each), so multi-segment is the common case for blob-bearing - // payloads. The generated SSZ codecs accept ReadOnlySequence — single-segment - // is zero-copy, multi-segment consolidates once via ArrayPool. Both paths skip the - // MemoryStream + ToArray dance the previous implementation needed. - PipeReader reader = ctx.Request.BodyReader; - ReadOnlySequence body = default; - bool bodyRead = false; - try - { - body = await ReadBodyAsync(ctx, reader); - bodyRead = true; - Metrics.SszRestRequestBytesTotal += body.Length; + await DispatchAsync(ctx, handler!, version, extra); + } + } - await handler!.HandleAsync(ctx, version, extra, body); + private static bool IsWitnessPath(string path) + => path.Equals(WitnessPath, StringComparison.OrdinalIgnoreCase); - int status = ctx.Response.StatusCode; - switch (status) - { - case >= 200 and < 300: - Metrics.SszRestRequestsSuccessTotal++; - break; - case >= 400 and < 500: - Metrics.SszRestRequestsClientErrorTotal++; - break; - case >= 500: - Metrics.SszRestRequestsServerErrorTotal++; - break; - } - } - catch (InvalidOperationException ex) when (!bodyRead) - { - Metrics.SszRestRequestsClientErrorTotal++; - await SszEndpointHandlerBase.WriteErrorAsync(ctx, StatusCodes.Status413PayloadTooLarge, ex.Message); - } - catch (Exception ex) when (ex is InvalidDataException or EndOfStreamException) - { - // Per execution-apis #793 (Engine API SSZ Transport spec, "HTTP status codes" section): - // malformed SSZ encoding is 400 Bad Request with type=ssz-decode-error: canned error, - // no detail (spec verbatim). 422 Unprocessable Entity is reserved for - // "Invalid payload attributes" and is emitted by the handler chain via - // ErrorCodeToHttpStatus when the engine module returns InvalidPayloadAttributes. - Metrics.SszRestDecodeFailuresTotal++; - Metrics.SszRestRequestsClientErrorTotal++; - if (_logger.IsDebug) _logger.Debug($"SSZ-REST malformed body at {ctx.Request.Path.Value}: {ex.Message}"); - await SszEndpointHandlerBase.WriteErrorAsync(ctx, StatusCodes.Status400BadRequest, - string.Empty, SszRestErrorCodes.SszDecodeError); - } - catch (Exception ex) - { - Metrics.SszRestRequestsServerErrorTotal++; - if (_logger.IsError) _logger.Error($"SSZ-REST handler error for {ctx.Request.Path.Value}", ex); - - // If the inner code already aborted the request (e.g. encode failed mid-stream - // and called ctx.Abort), don't try to write a 500 — WriteAsync would throw - // OperationCanceledException, producing a duplicate exception in the logs. - if (!ctx.RequestAborted.IsCancellationRequested) - await SszEndpointHandlerBase.WriteErrorAsync(ctx, StatusCodes.Status500InternalServerError, "Internal server error"); - } - finally + /// + /// Fast-path dispatch for the version-less witness endpoint. Validates method (POST only) and + /// content-type (application/json) before delegating to the witness handler via . + /// + private async Task DispatchWitnessAsync(HttpContext ctx) + { + if (!HttpMethods.IsPost(ctx.Request.Method)) + { + Metrics.SszRestRequestsClientErrorTotal++; + ctx.Response.Headers.Allow = "POST"; + await SszEndpointHandlerBase.WriteErrorAsync(ctx, StatusCodes.Status405MethodNotAllowed, + $"Method '{ctx.Request.Method}' is not allowed on {WitnessPath}. Only POST is supported.", + SszRestErrorCodes.MethodNotFound); + return; + } + + string? contentType = ctx.Request.ContentType; + if (contentType is null || !contentType.Contains("application/json", StringComparison.OrdinalIgnoreCase)) + { + Metrics.SszRestRequestsClientErrorTotal++; + ctx.Response.Headers["Accept"] = "application/json"; + await SszEndpointHandlerBase.WriteErrorAsync(ctx, StatusCodes.Status415UnsupportedMediaType, + $"Content-Type must be application/json for {WitnessPath}.", + SszRestErrorCodes.UnsupportedMediaType); + return; + } + + if (_witnessHandler is null) + { + Metrics.SszRestRequestsClientErrorTotal++; + await SszEndpointHandlerBase.WriteErrorAsync(ctx, StatusCodes.Status404NotFound, + "Endpoint not available", SszRestErrorCodes.MethodNotFound); + return; + } + + if (_logger.IsTrace) _logger.Trace($"SSZ-REST POST {WitnessPath}"); + + await DispatchAsync(ctx, _witnessHandler, version: 0, extra: default); + } + + /// Shared body-read + handler invocation + metrics + error mapping for both the + /// fork-routed endpoints and the witness fast-path. + private async Task DispatchAsync(HttpContext ctx, ISszEndpointHandler handler, int version, ReadOnlyMemory extra) + { + // Read directly from PipeReader: the buffer is a ReadOnlySequence over Kestrel's + // pooled blocks (~4 KB each), so multi-segment is the common case for blob-bearing + // payloads. The generated SSZ codecs accept ReadOnlySequence — single-segment + // is zero-copy, multi-segment consolidates once via ArrayPool. Both paths skip the + // MemoryStream + ToArray dance the previous implementation needed. + PipeReader reader = ctx.Request.BodyReader; + ReadOnlySequence body = default; + bool bodyRead = false; + try + { + body = await ReadBodyAsync(ctx, reader); + bodyRead = true; + Metrics.SszRestRequestBytesTotal += body.Length; + + await handler.HandleAsync(ctx, version, extra, body); + + int status = ctx.Response.StatusCode; + switch (status) { - if (bodyRead) reader.AdvanceTo(body.End); + case >= 200 and < 300: + Metrics.SszRestRequestsSuccessTotal++; + break; + case >= 400 and < 500: + Metrics.SszRestRequestsClientErrorTotal++; + break; + case >= 500: + Metrics.SszRestRequestsServerErrorTotal++; + break; } } + catch (InvalidOperationException ex) when (!bodyRead) + { + Metrics.SszRestRequestsClientErrorTotal++; + await SszEndpointHandlerBase.WriteErrorAsync(ctx, StatusCodes.Status413PayloadTooLarge, ex.Message); + } + catch (Exception ex) when (ex is InvalidDataException or EndOfStreamException) + { + // Per execution-apis #793 (Engine API SSZ Transport spec, "HTTP status codes" section): + // malformed SSZ encoding is 400 Bad Request with type=ssz-decode-error: canned error, + // no detail (spec verbatim). 422 Unprocessable Entity is reserved for + // "Invalid payload attributes" and is emitted by the handler chain via + // ErrorCodeToHttpStatus when the engine module returns InvalidPayloadAttributes. + Metrics.SszRestDecodeFailuresTotal++; + Metrics.SszRestRequestsClientErrorTotal++; + if (_logger.IsDebug) _logger.Debug($"SSZ-REST malformed body at {ctx.Request.Path.Value}: {ex.Message}"); + await SszEndpointHandlerBase.WriteErrorAsync(ctx, StatusCodes.Status400BadRequest, + string.Empty, SszRestErrorCodes.SszDecodeError); + } + catch (Exception ex) + { + Metrics.SszRestRequestsServerErrorTotal++; + if (_logger.IsError) _logger.Error($"SSZ-REST handler error for {ctx.Request.Path.Value}", ex); + + // If the inner code already aborted the request (e.g. encode failed mid-stream + // and called ctx.Abort), don't try to write a 500 — WriteAsync would throw + // OperationCanceledException, producing a duplicate exception in the logs. + if (!ctx.RequestAborted.IsCancellationRequested) + await SszEndpointHandlerBase.WriteErrorAsync(ctx, StatusCodes.Status500InternalServerError, "Internal server error"); + } + finally + { + if (bodyRead) reader.AdvanceTo(body.End); + } } private static bool TryRoute(string path, out int version, out string? fork, @@ -406,6 +472,13 @@ private static SszRequestKind ClassifySszRequest(HttpContext ctx) { string path = ctx.Request.Path.Value ?? string.Empty; + // The witness endpoint is a dedicated, version-less path. Intercept it for ALL methods so + // non-POST / wrong content-type get a proper 405/415 from DispatchWitnessAsync rather than + // falling through to the next middleware and returning a confusing 404. The application/json + // content-type check is deferred to DispatchWitnessAsync (it is not an octet-stream route). + if (path.Equals(WitnessPath, StringComparison.OrdinalIgnoreCase)) + return SszRequestKind.EngineOk; + if (!path.StartsWith("/engine/", StringComparison.OrdinalIgnoreCase)) return SszRequestKind.NotEngine; diff --git a/src/Nethermind/Nethermind.Merge.Plugin/SszRest/SszMiddlewareConfigurer.cs b/src/Nethermind/Nethermind.Merge.Plugin/SszRest/SszMiddlewareConfigurer.cs index 9bd0facd55dc..81616cfe8f44 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/SszRest/SszMiddlewareConfigurer.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/SszRest/SszMiddlewareConfigurer.cs @@ -81,6 +81,12 @@ public void Configure(IServiceCollection services) foreach (Type handler in SingletonHandlers) services.AddSingleton(typeof(ISszEndpointHandler), handler); + + // EIP-7928 witness endpoint: registered as the concrete type so SszMiddleware can take it + // directly for its dedicated fast-path, and as ISszEndpointHandler (via the same instance) + // so DI doesn't double-construct on resolution. + services.AddSingleton(); + services.AddSingleton(static sp => sp.GetRequiredService()); } } diff --git a/src/Nethermind/Nethermind.Merge.Plugin/SszRest/SszWireTypes.cs b/src/Nethermind/Nethermind.Merge.Plugin/SszRest/SszWireTypes.cs index ac4bc73de5cf..725f54090470 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/SszRest/SszWireTypes.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/SszRest/SszWireTypes.cs @@ -455,3 +455,16 @@ public partial struct GetBlobsV4ResponseWire { [SszList(128)] public BlobV4EntryWire[]? Entries { get; set; } } +[SszContainer(isCollectionItself: true)] +public partial struct SszWitnessItem +{ + [SszList(1048576)] public byte[]? Bytes { get; set; } +} + +[SszContainer] +public partial struct ExecutionWitnessV1Wire +{ + [SszList(1048576)] public SszWitnessItem[]? State { get; set; } + [SszList(1048576)] public SszWitnessItem[]? Codes { get; set; } + [SszList(1048576)] public SszWitnessItem[]? Headers { get; set; } +} diff --git a/src/Nethermind/Nethermind.State.Flat/ScopeProvider/FlatScopeProvider.cs b/src/Nethermind/Nethermind.State.Flat/ScopeProvider/FlatScopeProvider.cs index 6d8c8834edd1..8357f8c92959 100644 --- a/src/Nethermind/Nethermind.State.Flat/ScopeProvider/FlatScopeProvider.cs +++ b/src/Nethermind/Nethermind.State.Flat/ScopeProvider/FlatScopeProvider.cs @@ -6,6 +6,7 @@ using Nethermind.Db; using Nethermind.Evm.State; using Nethermind.Logging; +using Nethermind.Trie; namespace Nethermind.State.Flat.ScopeProvider; @@ -16,7 +17,8 @@ public class FlatScopeProvider( ITrieWarmer trieWarmer, ResourcePool.Usage usage, ILogManager logManager, - bool isReadOnly) + bool isReadOnly, + ITrieNodeReadObserver? trieReadObserver = null) : IWorldStateScopeProvider, IDisposable { private readonly TrieStoreScopeProvider.KeyValueWithBatchingBackedCodeDb _codeDb = new(codeDb, isPersistent: !isReadOnly); @@ -44,7 +46,8 @@ public IWorldStateScopeProvider.IScope BeginScope(BlockHeader? baseBlock) trieWarmer, logManager, warmReadPool: _warmReadPool, - isReadOnly: isReadOnly); + isReadOnly: isReadOnly, + trieReadObserver: trieReadObserver); } public void Dispose() diff --git a/src/Nethermind/Nethermind.State.Flat/ScopeProvider/FlatStorageTree.cs b/src/Nethermind/Nethermind.State.Flat/ScopeProvider/FlatStorageTree.cs index c7df70ef2c99..5c0853870d9c 100644 --- a/src/Nethermind/Nethermind.State.Flat/ScopeProvider/FlatStorageTree.cs +++ b/src/Nethermind/Nethermind.State.Flat/ScopeProvider/FlatStorageTree.cs @@ -36,7 +36,8 @@ public FlatStorageTree( ConcurrencyController concurrencyQuota, Hash256 storageRoot, Address address, - ILogManager logManager) + ILogManager logManager, + ITrieNodeReadObserver? trieReadObserver = null) { _scope = scope; _trieCacheWarmer = trieCacheWarmer; @@ -45,7 +46,7 @@ public FlatStorageTree( _addressHash = address.ToAccountPath.ToHash256(); _selfDestructKnownStateIdx = bundle.DetermineSelfDestructSnapshotIdx(address); - StorageTrieStoreAdapter storageTrieAdapter = new(bundle, concurrencyQuota, _addressHash); + StorageTrieStoreAdapter storageTrieAdapter = new(bundle, concurrencyQuota, _addressHash, trieReadObserver); StorageTrieStoreWarmerAdapter warmerStorageTrieAdapter = new(bundle, _addressHash); _tree = new StorageTree(storageTrieAdapter, storageRoot, logManager) diff --git a/src/Nethermind/Nethermind.State.Flat/ScopeProvider/FlatWorldStateManager.cs b/src/Nethermind/Nethermind.State.Flat/ScopeProvider/FlatWorldStateManager.cs index b5d991e769b6..eb35ae30d77d 100644 --- a/src/Nethermind/Nethermind.State.Flat/ScopeProvider/FlatWorldStateManager.cs +++ b/src/Nethermind/Nethermind.State.Flat/ScopeProvider/FlatWorldStateManager.cs @@ -9,6 +9,7 @@ using Nethermind.State.Flat.Persistence; using Nethermind.State.Flat.Sync.Snap; using Nethermind.State.SnapServer; +using Nethermind.Trie; using Nethermind.Trie.Pruning; namespace Nethermind.State.Flat.ScopeProvider; @@ -23,9 +24,12 @@ public class FlatWorldStateManager( Func overridableWorldScopeFactory, [KeyFilter(DbNames.Code)] IDb codeDb, IFlatStateRootIndex flatStateRootIndex, - ILogManager logManager) + ILogManager logManager, + ITrieNodeReadObserver? mainTrieReadObserver = null) : IWorldStateManager, IDisposable { + // The read observer is threaded into the main world state only: read-only/resettable scopes + // (RPC envs, prewarming) never feed witness capture and stay observer-free. private readonly FlatScopeProvider _mainWorldState = new( codeDb, flatDbManager, @@ -33,7 +37,8 @@ public class FlatWorldStateManager( trieWarmer, ResourcePool.Usage.MainBlockProcessing, logManager, - isReadOnly: false); + isReadOnly: false, + trieReadObserver: mainTrieReadObserver); private readonly FlatTrieVerifier _trieVerifier = new(flatDbManager, persistence, logManager); diff --git a/src/Nethermind/Nethermind.State.Flat/ScopeProvider/FlatWorldStateScope.cs b/src/Nethermind/Nethermind.State.Flat/ScopeProvider/FlatWorldStateScope.cs index 31c87adcfb8a..169030de3bc8 100644 --- a/src/Nethermind/Nethermind.State.Flat/ScopeProvider/FlatWorldStateScope.cs +++ b/src/Nethermind/Nethermind.State.Flat/ScopeProvider/FlatWorldStateScope.cs @@ -39,6 +39,7 @@ public sealed class FlatWorldStateScope : IWorldStateScopeProvider.IScope, ITrie private volatile int _hintSequenceId = 0; private int _outstandingWarmups = 0; private StateId _currentStateId; + private readonly ITrieNodeReadObserver? _trieReadObserver; internal volatile bool _pausePrewarmer = false; private CancellationTokenSource? _hintBalCts; @@ -55,16 +56,18 @@ public FlatWorldStateScope( ITrieWarmer trieCacheWarmer, ILogManager logManager, Lazy? warmReadPool = null, - bool isReadOnly = false) + bool isReadOnly = false, + ITrieNodeReadObserver? trieReadObserver = null) { _currentStateId = currentStateId; _snapshotBundle = snapshotBundle; CodeDb = codeDb; _commitTarget = commitTarget; + _trieReadObserver = trieReadObserver; _concurrencyQuota = new ConcurrencyController(Environment.ProcessorCount); // Used during tree commit. _stateTree = new( - new StateTrieStoreAdapter(snapshotBundle, _concurrencyQuota), + new StateTrieStoreAdapter(snapshotBundle, _concurrencyQuota, trieReadObserver), logManager ) { @@ -351,7 +354,8 @@ private FlatStorageTree CreateStorageTreeImpl(Address address) _concurrencyQuota, storageRoot, address, - _logManager); + _logManager, + _trieReadObserver); return storage; } diff --git a/src/Nethermind/Nethermind.State.Flat/ScopeProvider/StateTrieStoreAdapter.cs b/src/Nethermind/Nethermind.State.Flat/ScopeProvider/StateTrieStoreAdapter.cs index ec6e03100a1f..4d65b5688cf9 100644 --- a/src/Nethermind/Nethermind.State.Flat/ScopeProvider/StateTrieStoreAdapter.cs +++ b/src/Nethermind/Nethermind.State.Flat/ScopeProvider/StateTrieStoreAdapter.cs @@ -11,17 +11,25 @@ namespace Nethermind.State.Flat.ScopeProvider; internal sealed class StateTrieStoreAdapter( SnapshotBundle bundle, - ConcurrencyController concurrencyQuota + ConcurrencyController concurrencyQuota, + ITrieNodeReadObserver? readObserver = null ) : AbstractMinimalTrieStore { public override TrieNode FindCachedOrUnknown(in TreePath path, Hash256 hash) { TrieNode node = bundle.FindStateNodeOrUnknown(path, hash); - return node.Keccak != hash ? throw new NodeHashMismatchException($"Node hash mismatch. Path: {path}. Hash: {node.Keccak} vs Requested: {hash}") : node; + if (node.Keccak != hash) throw new NodeHashMismatchException($"Node hash mismatch. Path: {path}. Hash: {node.Keccak} vs Requested: {hash}"); + if (readObserver is { IsActive: true } observer && node.NodeType != NodeType.Unknown && node.FullRlp.IsNotNull) + observer.OnTrieNodeRead(hash, node.FullRlp.ToArray()!); + return node; } - public override byte[]? TryLoadRlp(in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) => - bundle.TryLoadStateRlp(path, hash, flags); + public override byte[]? TryLoadRlp(in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) + { + byte[]? rlp = bundle.TryLoadStateRlp(path, hash, flags); + if (rlp is not null && readObserver is { IsActive: true } observer) observer.OnTrieNodeRead(hash, rlp); + return rlp; + } public override ICommitter BeginCommit(TrieNode? root, WriteFlags writeFlags = WriteFlags.None) => new Committer(bundle, concurrencyQuota); @@ -29,7 +37,7 @@ public override ICommitter BeginCommit(TrieNode? root, WriteFlags writeFlags = W public override ITrieNodeResolver GetStorageTrieNodeResolver(Hash256? address) { if (address is null) return this; - return new StorageTrieStoreAdapter(bundle, concurrencyQuota, address); + return new StorageTrieStoreAdapter(bundle, concurrencyQuota, address, readObserver); } private class Committer(SnapshotBundle bundle, ConcurrencyController concurrencyQuota) : AbstractMinimalCommitter(concurrencyQuota) @@ -65,17 +73,25 @@ public override ITrieNodeResolver GetStorageTrieNodeResolver(Hash256? address) internal sealed class StorageTrieStoreAdapter( SnapshotBundle bundle, ConcurrencyController concurrencyQuota, - Hash256AsKey addressHash + Hash256AsKey addressHash, + ITrieNodeReadObserver? readObserver = null ) : AbstractMinimalTrieStore { public override TrieNode FindCachedOrUnknown(in TreePath path, Hash256 hash) { TrieNode node = bundle.FindStorageNodeOrUnknown(addressHash, path, hash); - return node.Keccak != hash ? throw new NodeHashMismatchException($"Node hash mismatch. Address {addressHash.Value}. Path: {path}. Hash: {node.Keccak} vs Requested: {hash}") : node; + if (node.Keccak != hash) throw new NodeHashMismatchException($"Node hash mismatch. Address {addressHash.Value}. Path: {path}. Hash: {node.Keccak} vs Requested: {hash}"); + if (readObserver is { IsActive: true } observer && node.NodeType != NodeType.Unknown && node.FullRlp.IsNotNull) + observer.OnTrieNodeRead(hash, node.FullRlp.ToArray()!); + return node; } - public override byte[]? TryLoadRlp(in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) => - bundle.TryLoadStorageRlp(addressHash, in path, hash, flags); + public override byte[]? TryLoadRlp(in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) + { + byte[]? rlp = bundle.TryLoadStorageRlp(addressHash, in path, hash, flags); + if (rlp is not null && readObserver is { IsActive: true } observer) observer.OnTrieNodeRead(hash, rlp); + return rlp; + } public override ICommitter BeginCommit(TrieNode? root, WriteFlags writeFlags = WriteFlags.None) => new Committer(bundle, addressHash, concurrencyQuota); diff --git a/src/Nethermind/Nethermind.Taiko.Test/CertainBatchLookupTests.cs b/src/Nethermind/Nethermind.Taiko.Test/CertainBatchLookupTests.cs index aa74701d1249..d2377bb6e175 100644 --- a/src/Nethermind/Nethermind.Taiko.Test/CertainBatchLookupTests.cs +++ b/src/Nethermind/Nethermind.Taiko.Test/CertainBatchLookupTests.cs @@ -325,12 +325,13 @@ private static TaikoEngineRpcModule CreateRpcModule(IL1OriginStore l1OriginStore Substitute.For, IReadOnlyList>>(), Substitute.For(), Substitute.For>(), - Substitute.For, IReadOnlyList>>(), + Substitute.For, IReadOnlyList>>(), Substitute.For>>(), Substitute.For?>>(), Substitute.For?>>(), Substitute.For, IReadOnlyList>>(), Substitute.For(), + Substitute.For(), Substitute.For(), specProvider, null!, diff --git a/src/Nethermind/Nethermind.Taiko.Test/TxPoolContentListsTests.cs b/src/Nethermind/Nethermind.Taiko.Test/TxPoolContentListsTests.cs index 61f21c949ab3..9ade1d6f118c 100644 --- a/src/Nethermind/Nethermind.Taiko.Test/TxPoolContentListsTests.cs +++ b/src/Nethermind/Nethermind.Taiko.Test/TxPoolContentListsTests.cs @@ -253,12 +253,13 @@ private static TaikoEngineRpcModule CreateRpcModule( Substitute.For, IReadOnlyList>>(), Substitute.For(), Substitute.For>(), - Substitute.For, IReadOnlyList>>(), + Substitute.For, IReadOnlyList>>(), Substitute.For>>(), Substitute.For?>>(), Substitute.For?>>(), Substitute.For, IReadOnlyList>>(), Substitute.For(), + Substitute.For(), Substitute.For(), Substitute.For(), null!, diff --git a/src/Nethermind/Nethermind.Taiko/Rpc/TaikoEngineRpcModule.cs b/src/Nethermind/Nethermind.Taiko/Rpc/TaikoEngineRpcModule.cs index 0151fe385a5e..ea4671c80d52 100644 --- a/src/Nethermind/Nethermind.Taiko/Rpc/TaikoEngineRpcModule.cs +++ b/src/Nethermind/Nethermind.Taiko/Rpc/TaikoEngineRpcModule.cs @@ -52,6 +52,7 @@ public class TaikoEngineRpcModule(IAsyncHandler getPa IAsyncHandler?> getBlobsHandlerV4, IHandler, IReadOnlyList> getPayloadBodiesByHashV2Handler, IGetPayloadBodiesByRangeV2Handler getPayloadBodiesByRangeV2Handler, + INewPayloadWithWitnessHandler newPayloadWithWitnessHandler, IEngineRequestsTracker engineRequestsTracker, ISpecProvider specProvider, GCKeeper gcKeeper, @@ -79,6 +80,7 @@ public class TaikoEngineRpcModule(IAsyncHandler getPa getBlobsHandlerV4, getPayloadBodiesByHashV2Handler, getPayloadBodiesByRangeV2Handler, + newPayloadWithWitnessHandler, engineRequestsTracker, specProvider, gcKeeper, diff --git a/src/Nethermind/Nethermind.Trie/ITrieNodeReadObserver.cs b/src/Nethermind/Nethermind.Trie/ITrieNodeReadObserver.cs new file mode 100644 index 000000000000..9067cd211398 --- /dev/null +++ b/src/Nethermind/Nethermind.Trie/ITrieNodeReadObserver.cs @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Crypto; + +namespace Nethermind.Trie; + +/// +/// Side-channel observer for raw trie node reads on the live block-processing path. Trie-store +/// adapters consult it where they resolve persisted nodes (e.g. during state-root recomputation, +/// where branch collapse resolves sibling nodes that never surface at the world-state level). +/// +/// +/// +/// lets call sites skip RLP materialization entirely when nothing is +/// recording, keeping the disarmed cost to a property read per node access. +/// +/// +/// Generic by design: it names what it does (observe node reads), not who uses it. Currently fired +/// only by the flat backend's trie adapters — flat's live read path is not an , +/// so it cannot be decorated; the patricia backend captures the equivalent reads via a +/// WitnessCapturingTrieStore decorator instead. +/// +/// +public interface ITrieNodeReadObserver +{ + bool IsActive { get; } + + void OnTrieNodeRead(Hash256 hash, byte[] rlp); +}